ol-github-linguist 2.4.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,56 @@
1
+ require 'linguist/blob_helper'
2
+
3
+ module Linguist
4
+ # A FileBlob is a wrapper around a File object to make it quack
5
+ # like a Grit::Blob. It provides the basic interface: `name`,
6
+ # `data`, and `size`.
7
+ class FileBlob
8
+ include BlobHelper
9
+
10
+ # Public: Initialize a new FileBlob from a path
11
+ #
12
+ # path - A path String that exists on the file system.
13
+ # base_path - Optional base to relativize the path
14
+ #
15
+ # Returns a FileBlob.
16
+ def initialize(path, base_path = nil)
17
+ @path = path
18
+ @name = base_path ? path.sub("#{base_path}/", '') : path
19
+ end
20
+
21
+ # Public: Filename
22
+ #
23
+ # Examples
24
+ #
25
+ # FileBlob.new("/path/to/linguist/lib/linguist.rb").name
26
+ # # => "/path/to/linguist/lib/linguist.rb"
27
+ #
28
+ # FileBlob.new("/path/to/linguist/lib/linguist.rb",
29
+ # "/path/to/linguist").name
30
+ # # => "lib/linguist.rb"
31
+ #
32
+ # Returns a String
33
+ attr_reader :name
34
+
35
+ # Public: Read file permissions
36
+ #
37
+ # Returns a String like '100644'
38
+ def mode
39
+ File.stat(@path).mode.to_s(8)
40
+ end
41
+
42
+ # Public: Read file contents.
43
+ #
44
+ # Returns a String.
45
+ def data
46
+ File.read(@path)
47
+ end
48
+
49
+ # Public: Get byte size
50
+ #
51
+ # Returns an Integer.
52
+ def size
53
+ File.size(@path)
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,162 @@
1
+ module Linguist
2
+ class Generated
3
+ # Public: Is the blob a generated file?
4
+ #
5
+ # name - String filename
6
+ # data - String blob data. A block also maybe passed in for lazy
7
+ # loading. This behavior is deprecated and you should always
8
+ # pass in a String.
9
+ #
10
+ # Return true or false
11
+ def self.generated?(name, data)
12
+ new(name, data).generated?
13
+ end
14
+
15
+ # Internal: Initialize Generated instance
16
+ #
17
+ # name - String filename
18
+ # data - String blob data
19
+ def initialize(name, data)
20
+ @name = name
21
+ @extname = File.extname(name)
22
+ @_data = data
23
+ end
24
+
25
+ attr_reader :name, :extname
26
+
27
+ # Lazy load blob data if block was passed in.
28
+ #
29
+ # Awful, awful stuff happening here.
30
+ #
31
+ # Returns String data.
32
+ def data
33
+ @data ||= @_data.respond_to?(:call) ? @_data.call() : @_data
34
+ end
35
+
36
+ # Public: Get each line of data
37
+ #
38
+ # Returns an Array of lines
39
+ def lines
40
+ # TODO: data should be required to be a String, no nils
41
+ @lines ||= data ? data.split("\n", -1) : []
42
+ end
43
+
44
+ # Internal: Is the blob a generated file?
45
+ #
46
+ # Generated source code is supressed in diffs and is ignored by
47
+ # language statistics.
48
+ #
49
+ # Please add additional test coverage to
50
+ # `test/test_blob.rb#test_generated` if you make any changes.
51
+ #
52
+ # Return true or false
53
+ def generated?
54
+ name == 'Gemfile.lock' ||
55
+ minified_javascript? ||
56
+ compiled_coffeescript? ||
57
+ xcode_project_file? ||
58
+ generated_net_docfile? ||
59
+ generated_parser?
60
+ end
61
+
62
+ # Internal: Is the blob an XCode project file?
63
+ #
64
+ # Generated if the file extension is an XCode project
65
+ # file extension.
66
+ #
67
+ # Returns true of false.
68
+ def xcode_project_file?
69
+ ['.xib', '.nib', '.storyboard', '.pbxproj', '.xcworkspacedata', '.xcuserstate'].include?(extname)
70
+ end
71
+
72
+ # Internal: Is the blob minified JS?
73
+ #
74
+ # Consider JS minified if the average line length is
75
+ # greater then 100c.
76
+ #
77
+ # Returns true or false.
78
+ def minified_javascript?
79
+ return unless extname == '.js'
80
+ if lines.any?
81
+ (lines.inject(0) { |n, l| n += l.length } / lines.length) > 100
82
+ else
83
+ false
84
+ end
85
+ end
86
+
87
+ # Internal: Is the blob of JS generated by CoffeeScript?
88
+ #
89
+ # CoffeScript is meant to output JS that would be difficult to
90
+ # tell if it was generated or not. Look for a number of patterns
91
+ # output by the CS compiler.
92
+ #
93
+ # Return true or false
94
+ def compiled_coffeescript?
95
+ return false unless extname == '.js'
96
+
97
+ # CoffeeScript generated by > 1.2 include a comment on the first line
98
+ if lines[0] =~ /^\/\/ Generated by /
99
+ return true
100
+ end
101
+
102
+ if lines[0] == '(function() {' && # First line is module closure opening
103
+ lines[-2] == '}).call(this);' && # Second to last line closes module closure
104
+ lines[-1] == '' # Last line is blank
105
+
106
+ score = 0
107
+
108
+ lines.each do |line|
109
+ if line =~ /var /
110
+ # Underscored temp vars are likely to be Coffee
111
+ score += 1 * line.gsub(/(_fn|_i|_len|_ref|_results)/).count
112
+
113
+ # bind and extend functions are very Coffee specific
114
+ score += 3 * line.gsub(/(__bind|__extends|__hasProp|__indexOf|__slice)/).count
115
+ end
116
+ end
117
+
118
+ # Require a score of 3. This is fairly arbitrary. Consider
119
+ # tweaking later.
120
+ score >= 3
121
+ else
122
+ false
123
+ end
124
+ end
125
+
126
+ # Internal: Is this a generated documentation file for a .NET assembly?
127
+ #
128
+ # .NET developers often check in the XML Intellisense file along with an
129
+ # assembly - however, these don't have a special extension, so we have to
130
+ # dig into the contents to determine if it's a docfile. Luckily, these files
131
+ # are extremely structured, so recognizing them is easy.
132
+ #
133
+ # Returns true or false
134
+ def generated_net_docfile?
135
+ return false unless extname.downcase == ".xml"
136
+ return false unless lines.count > 3
137
+
138
+ # .NET Docfiles always open with <doc> and their first tag is an
139
+ # <assembly> tag
140
+ return lines[1].include?("<doc>") &&
141
+ lines[2].include?("<assembly>") &&
142
+ lines[-2].include?("</doc>")
143
+ end
144
+
145
+ # Internal: Is the blob of JS a parser generated by PEG.js?
146
+ #
147
+ # PEG.js-generated parsers are not meant to be consumed by humans.
148
+ #
149
+ # Return true or false
150
+ def generated_parser?
151
+ return false unless extname == '.js'
152
+
153
+ # PEG.js-generated parsers include a comment near the top of the file
154
+ # that marks them as such.
155
+ if lines[0..4].join('') =~ /^(?:[^\/]|\/[^\*])*\/\*(?:[^\*]|\*[^\/])*Generated by PEG.js/
156
+ return true
157
+ end
158
+
159
+ false
160
+ end
161
+ end
162
+ end
@@ -0,0 +1,483 @@
1
+ # require 'escape_utils'
2
+ require 'cgi'
3
+ #require 'pygments'
4
+ require 'yaml'
5
+
6
+ require 'linguist/classifier'
7
+ require 'linguist/samples'
8
+
9
+ module Linguist
10
+ # Language names that are recognizable by GitHub. Defined languages
11
+ # can be highlighted, searched and listed under the Top Languages page.
12
+ #
13
+ # Languages are defined in `lib/linguist/languages.yml`.
14
+ class Language
15
+ @languages = []
16
+ @index = {}
17
+ @name_index = {}
18
+ @alias_index = {}
19
+ @extension_index = Hash.new { |h,k| h[k] = [] }
20
+ @filename_index = Hash.new { |h,k| h[k] = [] }
21
+
22
+ # Valid Languages types
23
+ TYPES = [:data, :markup, :programming]
24
+
25
+ # Internal: Create a new Language object
26
+ #
27
+ # attributes - A hash of attributes
28
+ #
29
+ # Returns a Language object
30
+ def self.create(attributes = {})
31
+ language = new(attributes)
32
+
33
+ @languages << language
34
+
35
+ # All Language names should be unique. Raise if there is a duplicate.
36
+ if @name_index.key?(language.name)
37
+ raise ArgumentError, "Duplicate language name: #{language.name}"
38
+ end
39
+
40
+ # Language name index
41
+ @index[language.name] = @name_index[language.name] = language
42
+
43
+ language.aliases.each do |name|
44
+ # All Language aliases should be unique. Raise if there is a duplicate.
45
+ if @alias_index.key?(name)
46
+ raise ArgumentError, "Duplicate alias: #{name}"
47
+ end
48
+
49
+ @index[name] = @alias_index[name] = language
50
+ end
51
+
52
+ language.extensions.each do |extension|
53
+ if extension !~ /^\./
54
+ raise ArgumentError, "Extension is missing a '.': #{extension.inspect}"
55
+ end
56
+
57
+ @extension_index[extension] << language
58
+ end
59
+
60
+ language.filenames.each do |filename|
61
+ @filename_index[filename] << language
62
+ end
63
+
64
+ language
65
+ end
66
+
67
+ # Public: Detects the Language of the blob.
68
+ #
69
+ # name - String filename
70
+ # data - String blob data. A block also maybe passed in for lazy
71
+ # loading. This behavior is deprecated and you should always
72
+ # pass in a String.
73
+ # mode - Optional String mode (defaults to nil)
74
+ #
75
+ # Returns Language or nil.
76
+ def self.detect(name, data, mode = nil)
77
+ # A bit of an elegant hack. If the file is exectable but extensionless,
78
+ # append a "magic" extension so it can be classified with other
79
+ # languages that have shebang scripts.
80
+ if File.extname(name).empty? && mode && (mode.to_i(8) & 05) == 05
81
+ name += ".script!"
82
+ end
83
+
84
+ possible_languages = find_by_filename(name)
85
+
86
+ if possible_languages.length > 1
87
+ data = data.call() if data.respond_to?(:call)
88
+ if data.nil? || data == ""
89
+ nil
90
+ elsif result = Classifier.classify(Samples::DATA, data, possible_languages.map(&:name)).first
91
+ Language[result[0]]
92
+ end
93
+ else
94
+ possible_languages.first
95
+ end
96
+ end
97
+
98
+ # Public: Get all Languages
99
+ #
100
+ # Returns an Array of Languages
101
+ def self.all
102
+ @languages
103
+ end
104
+
105
+ # Public: Look up Language by its proper name.
106
+ #
107
+ # name - The String name of the Language
108
+ #
109
+ # Examples
110
+ #
111
+ # Language.find_by_name('Ruby')
112
+ # # => #<Language name="Ruby">
113
+ #
114
+ # Returns the Language or nil if none was found.
115
+ def self.find_by_name(name)
116
+ @name_index[name]
117
+ end
118
+
119
+ # Public: Look up Language by one of its aliases.
120
+ #
121
+ # name - A String alias of the Language
122
+ #
123
+ # Examples
124
+ #
125
+ # Language.find_by_alias('cpp')
126
+ # # => #<Language name="C++">
127
+ #
128
+ # Returns the Lexer or nil if none was found.
129
+ def self.find_by_alias(name)
130
+ @alias_index[name]
131
+ end
132
+
133
+ # Public: Look up Languages by filename.
134
+ #
135
+ # filename - The path String.
136
+ #
137
+ # Examples
138
+ #
139
+ # Language.find_by_filename('foo.rb')
140
+ # # => [#<Language name="Ruby">]
141
+ #
142
+ # Returns all matching Languages or [] if none were found.
143
+ def self.find_by_filename(filename)
144
+ basename, extname = File.basename(filename), File.extname(filename)
145
+ @filename_index[basename] + @extension_index[extname]
146
+ end
147
+
148
+ # Public: Look up Language by its name or lexer.
149
+ #
150
+ # name - The String name of the Language
151
+ #
152
+ # Examples
153
+ #
154
+ # Language['Ruby']
155
+ # # => #<Language name="Ruby">
156
+ #
157
+ # Language['ruby']
158
+ # # => #<Language name="Ruby">
159
+ #
160
+ # Returns the Language or nil if none was found.
161
+ def self.[](name)
162
+ @index[name]
163
+ end
164
+
165
+ # Public: A List of popular languages
166
+ #
167
+ # Popular languages are sorted to the top of language chooser
168
+ # dropdowns.
169
+ #
170
+ # This list is configured in "popular.yml".
171
+ #
172
+ # Returns an Array of Lexers.
173
+ def self.popular
174
+ @popular ||= all.select(&:popular?).sort_by { |lang| lang.name.downcase }
175
+ end
176
+
177
+ # Public: A List of non-popular languages
178
+ #
179
+ # Unpopular languages appear below popular ones in language
180
+ # chooser dropdowns.
181
+ #
182
+ # This list is created from all the languages not listed in "popular.yml".
183
+ #
184
+ # Returns an Array of Lexers.
185
+ def self.unpopular
186
+ @unpopular ||= all.select(&:unpopular?).sort_by { |lang| lang.name.downcase }
187
+ end
188
+
189
+ # Public: A List of languages with assigned colors.
190
+ #
191
+ # Returns an Array of Languages.
192
+ def self.colors
193
+ @colors ||= all.select(&:color).sort_by { |lang| lang.name.downcase }
194
+ end
195
+
196
+ # Public: A List of languages compatible with Ace.
197
+ #
198
+ # Returns an Array of Languages.
199
+ def self.ace_modes
200
+ @ace_modes ||= all.select(&:ace_mode).sort_by { |lang| lang.name.downcase }
201
+ end
202
+
203
+ # Internal: Initialize a new Language
204
+ #
205
+ # attributes - A hash of attributes
206
+ def initialize(attributes = {})
207
+ # @name is required
208
+ @name = attributes[:name] || raise(ArgumentError, "missing name")
209
+
210
+ # Set type
211
+ @type = attributes[:type] ? attributes[:type].to_sym : nil
212
+ if @type && !TYPES.include?(@type)
213
+ raise ArgumentError, "invalid type: #{@type}"
214
+ end
215
+
216
+ @color = attributes[:color]
217
+
218
+ # Set aliases
219
+ @aliases = [default_alias_name] + (attributes[:aliases] || [])
220
+
221
+ # Lookup Lexer object
222
+ #@lexer = Pygments::Lexer.find_by_name(attributes[:lexer] || name) ||
223
+ # raise(ArgumentError, "#{@name} is missing lexer")
224
+
225
+ @ace_mode = attributes[:ace_mode]
226
+ @wrap = attributes[:wrap] || false
227
+
228
+ # Set legacy search term
229
+ @search_term = attributes[:search_term] || default_alias_name
230
+
231
+ # Set extensions or default to [].
232
+ @extensions = attributes[:extensions] || []
233
+ @filenames = attributes[:filenames] || []
234
+
235
+ unless @primary_extension = attributes[:primary_extension]
236
+ raise ArgumentError, "#{@name} is missing primary extension"
237
+ end
238
+
239
+ # Prepend primary extension unless its already included
240
+ if primary_extension && !extensions.include?(primary_extension)
241
+ @extensions = [primary_extension] + extensions
242
+ end
243
+
244
+ # Set popular, and searchable flags
245
+ @popular = attributes.key?(:popular) ? attributes[:popular] : false
246
+ @searchable = attributes.key?(:searchable) ? attributes[:searchable] : true
247
+
248
+ # If group name is set, save the name so we can lazy load it later
249
+ if attributes[:group_name]
250
+ @group = nil
251
+ @group_name = attributes[:group_name]
252
+
253
+ # Otherwise we can set it to self now
254
+ else
255
+ @group = self
256
+ end
257
+ end
258
+
259
+ # Public: Get proper name
260
+ #
261
+ # Examples
262
+ #
263
+ # # => "Ruby"
264
+ # # => "Python"
265
+ # # => "Perl"
266
+ #
267
+ # Returns the name String
268
+ attr_reader :name
269
+
270
+ # Public: Get type.
271
+ #
272
+ # Returns a type Symbol or nil.
273
+ attr_reader :type
274
+
275
+ # Public: Get color.
276
+ #
277
+ # Returns a hex color String.
278
+ attr_reader :color
279
+
280
+ # Public: Get aliases
281
+ #
282
+ # Examples
283
+ #
284
+ # Language['C++'].aliases
285
+ # # => ["cpp"]
286
+ #
287
+ # Returns an Array of String names
288
+ attr_reader :aliases
289
+
290
+ # Deprecated: Get code search term
291
+ #
292
+ # Examples
293
+ #
294
+ # # => "ruby"
295
+ # # => "python"
296
+ # # => "perl"
297
+ #
298
+ # Returns the name String
299
+ attr_reader :search_term
300
+
301
+ # Public: Get Lexer
302
+ #
303
+ # Returns the Lexer
304
+ attr_reader :lexer
305
+
306
+ # Public: Get Ace mode
307
+ #
308
+ # Examples
309
+ #
310
+ # # => "text"
311
+ # # => "javascript"
312
+ # # => "c_cpp"
313
+ #
314
+ # Returns a String name or nil
315
+ attr_reader :ace_mode
316
+
317
+ # Public: Should language lines be wrapped
318
+ #
319
+ # Returns true or false
320
+ attr_reader :wrap
321
+
322
+ # Public: Get extensions
323
+ #
324
+ # Examples
325
+ #
326
+ # # => ['.rb', '.rake', ...]
327
+ #
328
+ # Returns the extensions Array
329
+ attr_reader :extensions
330
+
331
+ # Deprecated: Get primary extension
332
+ #
333
+ # Defaults to the first extension but can be overriden
334
+ # in the languages.yml.
335
+ #
336
+ # The primary extension can not be nil. Tests should verify this.
337
+ #
338
+ # This attribute is only used by app/helpers/gists_helper.rb for
339
+ # creating the language dropdown. It really should be using `name`
340
+ # instead. Would like to drop primary extension.
341
+ #
342
+ # Returns the extension String.
343
+ attr_reader :primary_extension
344
+
345
+ # Public: Get filenames
346
+ #
347
+ # Examples
348
+ #
349
+ # # => ['Rakefile', ...]
350
+ #
351
+ # Returns the extensions Array
352
+ attr_reader :filenames
353
+
354
+ # Public: Get URL escaped name.
355
+ #
356
+ # Examples
357
+ #
358
+ # "C%23"
359
+ # "C%2B%2B"
360
+ # "Common%20Lisp"
361
+ #
362
+ # Returns the escaped String.
363
+ def escaped_name
364
+ # EscapeUtils.escape_url(name).gsub('+', '%20')
365
+ CGI.escape(name).gsub('+', '%20')
366
+ end
367
+
368
+ # Internal: Get default alias name
369
+ #
370
+ # Returns the alias name String
371
+ def default_alias_name
372
+ name.downcase.gsub(/\s/, '-')
373
+ end
374
+
375
+ # Public: Get Language group
376
+ #
377
+ # Returns a Language
378
+ def group
379
+ @group ||= Language.find_by_name(@group_name)
380
+ end
381
+
382
+ # Public: Is it popular?
383
+ #
384
+ # Returns true or false
385
+ def popular?
386
+ @popular
387
+ end
388
+
389
+ # Public: Is it not popular?
390
+ #
391
+ # Returns true or false
392
+ def unpopular?
393
+ !popular?
394
+ end
395
+
396
+ # Public: Is it searchable?
397
+ #
398
+ # Unsearchable languages won't by indexed by solr and won't show
399
+ # up in the code search dropdown.
400
+ #
401
+ # Returns true or false
402
+ def searchable?
403
+ @searchable
404
+ end
405
+
406
+ # Public: Highlight syntax of text
407
+ #
408
+ # text - String of code to be highlighted
409
+ # options - A Hash of options (defaults to {})
410
+ #
411
+ # Returns html String
412
+ def colorize(text, options = {})
413
+ lexer.highlight(text, options = {})
414
+ end
415
+
416
+ # Public: Return name as String representation
417
+ def to_s
418
+ name
419
+ end
420
+
421
+ def ==(other)
422
+ eql?(other)
423
+ end
424
+
425
+ def eql?(other)
426
+ equal?(other)
427
+ end
428
+
429
+ def hash
430
+ name.hash
431
+ end
432
+
433
+ def inspect
434
+ "#<#{self.class} name=#{name}>"
435
+ end
436
+ end
437
+
438
+ extensions = Samples::DATA['extnames']
439
+ filenames = Samples::DATA['filenames']
440
+ popular = YAML.load_file(File.expand_path("../popular.yml", __FILE__))
441
+
442
+ YAML.load_file(File.expand_path("../languages.yml", __FILE__)).each do |name, options|
443
+ options['extensions'] ||= []
444
+ options['filenames'] ||= []
445
+
446
+ if extnames = extensions[name]
447
+ extnames.each do |extname|
448
+ if !options['extensions'].include?(extname)
449
+ options['extensions'] << extname
450
+ else
451
+ warn "#{name} #{extname.inspect} is already defined in samples/. Remove from languages.yml."
452
+ end
453
+ end
454
+ end
455
+
456
+ if fns = filenames[name]
457
+ fns.each do |filename|
458
+ if !options['filenames'].include?(filename)
459
+ options['filenames'] << filename
460
+ else
461
+ warn "#{name} #{filename.inspect} is already defined in samples/. Remove from languages.yml."
462
+ end
463
+ end
464
+ end
465
+
466
+ Language.create(
467
+ :name => name,
468
+ :color => options['color'],
469
+ :type => options['type'],
470
+ :aliases => options['aliases'],
471
+ :lexer => options['lexer'],
472
+ :ace_mode => options['ace_mode'],
473
+ :wrap => options['wrap'],
474
+ :group_name => options['group'],
475
+ :searchable => options.key?('searchable') ? options['searchable'] : true,
476
+ :search_term => options['search_term'],
477
+ :extensions => options['extensions'].sort,
478
+ :primary_extension => options['primary_extension'],
479
+ :filenames => options['filenames'],
480
+ :popular => popular.include?(name)
481
+ )
482
+ end
483
+ end