github-linguist 1.0.0

data/lib/linguist/mime.rb

@@ -0,0 +1,91 @@
+require 'mime/types'
+require 'yaml'
+
+class MIME::Type
+  attr_accessor :override
+end
+
+# Register additional mime type extensions
+#
+# Follows same format as mime-types data file
+#   https://github.com/halostatue/mime-types/blob/master/lib/mime/types.rb.data
+File.read(File.expand_path("../mimes.yml", __FILE__)).lines.each do |line|
+  # Regexp was cargo culted from mime-types lib
+  next unless line =~ %r{^
+    #{MIME::Type::MEDIA_TYPE_RE}
+    (?:\s@([^\s]+))?
+    (?:\s:(#{MIME::Type::ENCODING_RE}))?
+  }x
+
+  mediatype  = $1
+  subtype    = $2
+  extensions = $3
+  encoding   = $4
+
+  # Lookup existing mime type
+  mime_type = MIME::Types["#{mediatype}/#{subtype}"].first ||
+    # Or create a new instance
+    MIME::Type.new("#{mediatype}/#{subtype}")
+
+  if extensions
+    extensions.split(/,/).each do |extension|
+      mime_type.extensions << extension
+    end
+  end
+
+  if encoding
+    mime_type.encoding = encoding
+  end
+
+  mime_type.override = true
+
+  # Kind of hacky, but we need to reindex the mime type after making changes
+  MIME::Types.add_type_variant(mime_type)
+  MIME::Types.index_extensions(mime_type)
+end
+
+module Linguist
+  module Mime
+    # Internal: Look up mime type for extension.
+    #
+    # ext - The extension String. May include leading "."
+    #
+    # Examples
+    #
+    #   Mime.mime_for('.html')
+    #   # => 'text/html'
+    #
+    #   Mime.mime_for('txt')
+    #   # => 'text/plain'
+    #
+    # Return mime type String otherwise falls back to 'text/plain'.
+    def self.mime_for(ext)
+      mime_type = lookup_mime_type_for(ext)
+      mime_type ? mime_type.to_s : 'text/plain'
+    end
+
+    # Internal: Lookup mime type for extension or mime type
+    #
+    # ext_or_mime_type - A file extension ".txt" or mime type "text/plain".
+    #
+    # Returns a MIME::Type
+    def self.lookup_mime_type_for(ext_or_mime_type)
+      ext_or_mime_type ||= ''
+
+      if ext_or_mime_type =~ /\w+\/\w+/
+        guesses = ::MIME::Types[ext_or_mime_type]
+      else
+        guesses = ::MIME::Types.type_for(ext_or_mime_type)
+      end
+
+      # Use custom override first
+      guesses.detect { |type| type.override } ||
+
+        # Prefer text mime types over binary
+        guesses.detect { |type| type.ascii? } ||
+
+        # Otherwise use the first guess
+        guesses.first
+    end
+  end
+end

data/lib/linguist/mimes.yml

@@ -0,0 +1,62 @@
+# Additional types to add to MIME::Types
+#
+# MIME types are used to set the Content-Type of raw binary blobs. All text
+# blobs are served as text/plain regardless of their type to ensure they
+# open in the browser rather than downloading.
+#
+# The encoding helps determine whether a file should be treated as plain
+# text or binary. By default, a mime type's encoding is base64 (binary).
+# These types will show a "View Raw" link. To force a type to render as
+# plain text, set it to 8bit for UTF-8. text/* types will be treated as
+# text by default.
+#
+#   <type> @<extensions> :<encoding>
+#
+# type       - mediatype/subtype
+# extensions - comma seperated extension list
+# encoding   - base64 (binary), 7bit (ASCII), 8bit (UTF-8), or
+#              quoted-printable (Printable ASCII).
+#
+# Follows same format as mime-types data file
+#   https://github.com/halostatue/mime-types/blob/master/lib/mime/types.rb.data
+#
+# Any additions or modifications (even trivial) should have corresponding
+# test change in `test/test_mime.rb`.
+
+# TODO: Lookup actual types
+application/octet-stream @a,blend,gem,graffle,ipa,lib,mcz,nib,o,ogv,otf,pfx,pigx,plgx,psd,sib,spl,sqlite3,swc,ucode,xpi
+
+# Please keep this list alphabetized
+application/java-archive @ear,war
+application/netcdf :8bit
+application/ogg @ogg
+application/postscript :base64
+application/vnd.adobe.air-application-installer-package+zip @air
+application/vnd.mozilla.xul+xml :8bit
+application/vnd.oasis.opendocument.presentation @odp
+application/vnd.oasis.opendocument.spreadsheet @ods
+application/vnd.oasis.opendocument.text @odt
+application/vnd.openofficeorg.extension @oxt
+application/vnd.openxmlformats-officedocument.presentationml.presentation @pptx
+application/x-chrome-extension @crx
+application/x-iwork-keynote-sffkey @key
+application/x-iwork-numbers-sffnumbers @numbers
+application/x-iwork-pages-sffpages @pages
+application/x-ms-xbap @xbap :8bit
+application/x-parrot-bytecode @pbc
+application/x-shockwave-flash @swf
+application/x-silverlight-app @xap
+application/x-supercollider @sc :8bit
+application/x-troff-ms :8bit
+application/x-wais-source :8bit
+application/xaml+xml @xaml :8bit
+application/xslt+xml @xslt :8bit
+image/x-icns @icns
+text/cache-manifest @manifest
+text/plain @cu,cxx
+text/x-logtalk @lgt
+text/x-nemerle @n
+text/x-nimrod @nim
+text/x-ocaml @ml,mli,mll,mly,sig,sml
+text/x-rust @rs,rc
+text/x-scheme @rkt,scm,sls,sps,ss

data/lib/linguist/pathname.rb

@@ -0,0 +1,92 @@
+require 'linguist/language'
+require 'linguist/mime'
+require 'pygments'
+
+module Linguist
+  # Similar to ::Pathname, Linguist::Pathname wraps a path string and
+  # provides helpful query methods. Its useful when you only have a
+  # filename but not a blob and need to figure out the language of the file.
+  class Pathname
+    # Public: Initialize a Pathname
+    #
+    # path - A filename String. The file may or maybe actually exist.
+    #
+    # Returns a Pathname.
+    def initialize(path)
+      @path = path
+    end
+
+    # Public: Get the basename of the path
+    #
+    # Examples
+    #
+    #   Pathname.new('sub/dir/file.rb').basename
+    #   # => 'file.rb'
+    #
+    # Returns a String.
+    def basename
+      File.basename(@path)
+    end
+
+    # Public: Get the extname of the path
+    #
+    # Examples
+    #
+    #   Pathname.new('.rb').extname
+    #   # => '.rb'
+    #
+    #   Pathname.new('file.rb').extname
+    #   # => '.rb'
+    #
+    # Returns a String.
+    def extname
+      File.extname(@path)
+    end
+
+    # Public: Get the language of the path
+    #
+    # The path extension name is the only heuristic used to detect the
+    # language name.
+    #
+    # Examples
+    #
+    #   Pathname.new('file.rb').language
+    #   # => Language['Ruby']
+    #
+    # Returns a Language or nil if none was found.
+    def language
+      @language ||= Language.find_by_filename(@path)
+    end
+
+    # Internal: Get the lexer of the path
+    #
+    # Returns a Lexer.
+    def lexer
+      language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
+    end
+
+    # Public: Get the mime type
+    #
+    # Examples
+    #
+    #   Pathname.new('index.html').mime_type
+    #   # => 'text/html'
+    #
+    # Returns a mime type String.
+    def mime_type
+      @mime_type ||= Mime.mime_for(extname)
+    end
+
+    # Public: Return self as String
+    #
+    # Returns a String
+    def to_s
+      @path.dup
+    end
+
+    def eql?(other)
+      other.is_a?(self.class) && @path == other.to_s
+    end
+    alias_method :==, :eql?
+  end
+end

data/lib/linguist/popular.yml

@@ -0,0 +1,29 @@
+# Popular languages appear at the top of language dropdowns
+#
+# This file should only be edited by GitHub staff
+
+- ActionScript
+- Bash
+- C
+- C#
+- C++
+- CSS
+- Common Lisp
+- Diff
+- Emacs Lisp
+- Erlang
+- HTML
+- Haskell
+- Java
+- JavaScript
+- Lua
+- Objective-C
+- PHP
+- Perl
+- Python
+- Ruby
+- SQL
+- Scala
+- Scheme
+- TeX
+- XML

data/lib/linguist/repository.rb

@@ -0,0 +1,95 @@
+require 'linguist/file_blob'
+
+module Linguist
+  # A Repository is an abstraction of a Grit::Repo or a basic file
+  # system tree. It holds a list of paths pointing to Blobish objects.
+  #
+  # Its primary purpose is for gathering language statistics across
+  # the entire project.
+  class Repository
+    # Public: Initialize a new Repository from a File directory
+    #
+    # base_path - A path String
+    #
+    # Returns a Repository
+    def self.from_directory(base_path)
+      new Dir["#{base_path}/**/*"].
+        select { |f| File.file?(f) }.
+        map { |path| FileBlob.new(path, base_path) }
+    end
+
+    # Public: Initialize a new Repository
+    #
+    # enum - Enumerator that responds to `each` and
+    #        yields Blob objects
+    #
+    # Returns a Repository
+    def initialize(enum)
+      @enum = enum
+      @computed_stats = false
+      @language = @size = nil
+      @sizes = Hash.new { 0 }
+    end
+
+    # Public: Returns a breakdown of language stats.
+    #
+    # Examples
+    #
+    #   # => { Language['Ruby'] => 46319,
+    #          Language['JavaScript'] => 258 }
+    #
+    # Returns a Hash of Language keys and Integer size values.
+    def languages
+      compute_stats
+      @sizes
+    end
+
+    # Public: Get primary Language of repository.
+    #
+    # Returns a Language
+    def language
+      compute_stats
+      @language
+    end
+
+    # Public: Get the total size of the repository.
+    #
+    # Returns a byte size Integer
+    def size
+      compute_stats
+      @size
+    end
+
+    # Internal: Compute language breakdown for each blob in the Repository.
+    #
+    # Returns nothing
+    def compute_stats
+      return if @computed_stats
+
+      @enum.each do |blob|
+        # Skip binary file extensions
+        next if blob.binary_mime_type?
+
+        # Skip vendored or generated blobs
+        next if blob.vendored? || blob.generated? || blob.language.nil?
+
+        # Only include programming languages
+        if blob.language.type == :programming
+          @sizes[blob.language.group] += blob.size
+        end
+      end
+
+      # Compute total size
+      @size = @sizes.inject(0) { |s,(k,v)| s + v }
+
+      # Get primary language
+      if primary = @sizes.max_by { |(_, size)| size }
+        @language = primary[0]
+      end
+
+      @computed_stats = true
+
+      nil
+    end
+  end
+end

data/lib/linguist/vendor.yml

@@ -0,0 +1,96 @@
+# Vendored files and directories are excluded from language
+# statistics.
+#
+# Lines in this file are Regexps that are matched against the file
+# pathname.
+#
+# Please add additional test coverage to
+# `test/test_blob.rb#test_vendored` if you make any changes.
+
+## Vendor Conventions ##
+
+# Caches
+- cache/
+
+# C deps
+#  https://github.com/joyent/node
+- ^deps/
+- ^tools/
+
+# Node depedencies
+- node_modules/
+
+# Vendored depedencies
+- vendor/
+
+
+## Commonly Bundled JavaScript frameworks ##
+
+# jQuery
+- (^|/)jquery([^.]*)(\.min)?\.js$
+- (^|/)jquery\-\d\.\d(\.\d)?(\.min)?\.js$
+
+# Prototype
+- (^|/)prototype(.*)\.js$
+- (^|/)effects\.js$
+- (^|/)controls\.js$
+- (^|/)dragdrop\.js$
+
+# MooTools
+- (^|/)mootools([^.]*)\d+\.\d+.\d+([^.]*)\.js$
+
+# Dojo
+- (^|/)dojo\.js$
+
+# MochiKit
+- (^|/)MochiKit\.js$
+
+# YUI
+- (^|/)yahoo-([^.]*)\.js$
+- (^|/)yui([^.]*)\.js$
+
+# LESS css
+- (^|/)less([^.]*)(\.min)?\.js$
+- (^|/)less\-\d+\.\d+\.\d+(\.min)?\.js$
+
+# WYS editors
+- (^|/)ckeditor\.js$
+- (^|/)tiny_mce([^.]*)\.js$
+- (^|/)tiny_mce/(langs|plugins|themes|utils)
+
+# MathJax
+- (^|/)MathJax/
+
+## Python ##
+
+# Fabric
+- ^fabfile\.py$
+
+# WAF
+- ^waf$
+
+
+## Obj-C ##
+
+# Sparkle
+- (^|/)Sparkle/
+
+## .NET ##
+
+# Visual Studio IntelliSense
+- -vsdoc\.js$
+
+# jQuery validation plugin (MS bundles this with asp.net mvc)
+- (^|/)jquery([^.]*)\.validate(\.min)?\.js$
+
+# Microsoft Ajax
+- (^|/)[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$
+
+# NuGet
+- ^[Pp]ackages/
+
+# ExtJS
+- (^|/)extjs/
+
+# Samples folders
+- ^[Ss]amples/