comment_extractor 1.0.1

data/lib/comment_extractor/extractor/erlang.rb

@@ -0,0 +1,12 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Erlang < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  shebang /escript/
+  filename /\.(?:erl|es)$/
+  filetype 'erlang'
+
+  define_default_bracket
+  comment start_with: /%+/
+end

data/lib/comment_extractor/extractor/fortran.rb

@@ -0,0 +1,11 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Fortran < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.(?:f|f90|F|F90)$/
+  filetype 'fortran'
+
+  define_default_bracket
+  comment start_with: '!'
+end

data/lib/comment_extractor/extractor/go.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Go < CommentExtractor::Extractor
+  filename /\.go$/
+  filetype 'go'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/h.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::H < CommentExtractor::Extractor
+  filename /\.h$/
+  filetype 'h'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/haml.rb

@@ -0,0 +1,49 @@
+# Copyright (c) 2006-2009 Hampton Catlin and Nathan Weizenbaum and @alpaca-tc
+
+require 'haml'
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Haml < CommentExtractor::Extractor
+  filename /\.haml$/
+  filetype 'haml'
+
+  # [review] - incompleted method
+  def scan
+    options = ::Haml::Options.new
+    parser = ::Haml::Parser.new(self.content, options)
+    parsered = parser.parse
+    parsered.children.each do |node|
+      detect_comment_from_node(node)
+    end
+  end
+
+  private
+
+  def detect_comment_from_node(node)
+    case node.type
+    when :haml_comment
+      identify_single_line_comment_from_comment_tag(node)
+    when :tag, :script
+      identify_single_line_comment_from_tag(node)
+    end
+
+    node.children.each do |child|
+      detect_comment_from_node(child)
+    end
+  end
+
+  def identify_single_line_comment_from_tag(node)
+    # [todo] - refactoring
+    if /#(?<comment>[^#]*)$/ =~ (node.value[:text])
+      code_objects << build_comment(node.line, comment) unless comment.empty?
+    end
+
+    if /#(?<comment>[^#]*)$/ =~ node.value[:value]
+      code_objects << build_comment(node.line, comment) unless comment.empty?
+    end
+  end
+
+  def identify_single_line_comment_from_comment_tag(node)
+    code_objects << build_comment(node.line, node.value[:text])  unless node.value[:text].empty?
+  end
+end

data/lib/comment_extractor/extractor/haskell.rb

@@ -0,0 +1,12 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Haskell < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.hs$/
+  filetype 'haskell'
+
+  define_default_bracket
+  comment start_with: '--'
+  comment start_with: '{-', end_with: '-}', type: BLOCK_COMMENT
+end

data/lib/comment_extractor/extractor/hpp.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Hpp < CommentExtractor::Extractor
+  filename /\.hpp$/
+  filetype 'hpp'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/html.rb

@@ -0,0 +1,13 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Html < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.html$/
+  filetype 'html'
+
+  define_default_bracket
+  define_ignore_patterns /<\s*script[^>]*>.*?<\/script\s*>/mi
+
+  comment start_with: '<!--', end_with: '-->', type: BLOCK_COMMENT
+end

data/lib/comment_extractor/extractor/java.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Java < CommentExtractor::Extractor
+  filename /\.java$/
+  filetype 'java'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/java_script.rb

@@ -0,0 +1,12 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::JavaScript < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+
+  filename /\.js$/
+  filetype 'javascript'
+  shebang /.*(?:js|node)$/
+
+  define_regexp_bracket
+end

data/lib/comment_extractor/extractor/lisp.rb

@@ -0,0 +1,11 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Lisp < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.el$/
+  filetype 'lisp'
+
+  define_bracket '"'
+  comment start_with: /;+/
+end

data/lib/comment_extractor/extractor/lua.rb

@@ -0,0 +1,12 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Lua < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.lua$/
+  filetype 'lua'
+
+  define_default_bracket
+  comment start_with: '--\[\[', end_with: /\s*\]\]/, type: BLOCK_COMMENT
+  comment start_with: '--'
+end

data/lib/comment_extractor/extractor/m.rb

@@ -0,0 +1,9 @@
+require 'comment_extractor/extractor'
+
+# Objective-C
+class CommentExtractor::Extractor::M < CommentExtractor::Extractor
+  filename /\.m$/
+  filetype 'm'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/markdown.rb

@@ -0,0 +1,7 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Markdown < CommentExtractor::Extractor
+  disable!
+  filename /\.(?:md|markdown|mkd)$/
+  filetype 'markdown'
+end

data/lib/comment_extractor/extractor/mm.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Mm < CommentExtractor::Extractor
+  filename /\.mm$/
+  filetype 'mm'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/perl.rb

@@ -0,0 +1,12 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Perl < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.(?:pm|pl)$/
+  filetype 'perl'
+
+  define_default_bracket
+  comment start_with: /^=pod/, end_with: /^=cut/, type: BLOCK_COMMENT
+  comment start_with: '#'
+end

data/lib/comment_extractor/extractor/php.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Php < CommentExtractor::Extractor
+  filename /\.php$/
+  filetype 'php'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/python.rb

@@ -0,0 +1,13 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Python < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.py$/
+  filetype 'py'
+
+  define_default_bracket
+  comment start_with: '"""', end_with: '"""', type: BLOCK_COMMENT
+  comment start_with: '"""', end_with: '"""', type: BLOCK_COMMENT
+  comment start_with: '#'
+end

data/lib/comment_extractor/extractor/ruby.rb

@@ -0,0 +1,40 @@
+require 'rdoc'
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Ruby < CommentExtractor::Extractor
+  include CommentExtractor::CodeObject::Comment::Type
+
+  filename /(?:Rakefile|Gemfile|\.rb|\.gemspec|Guardfile|config.ru)$/
+  filetype 'ruby'
+  shebang /.*ruby$/
+
+  class Options < Hash
+    def tab_width
+      2
+    end
+  end
+
+  def scan
+    tokens = RDoc::RubyLex.tokenize(content, Options.new)
+
+    tokens.each do |token|
+      case token
+      when RDoc::RubyToken::TkRD_COMMENT # =begin ... =end
+        token.value.split("\n").each_with_index do |comment, index|
+          line_no = token.line_no + 1 + index
+          add_comment(line_no, comment, type: BLOCK_COMMENT)
+        end
+      when RDoc::RubyToken::TkCOMMENT # # ...
+        comment = token.value.sub(/^\s*#\s?/, '')
+        add_comment(token.line_no, comment, type: ONE_LINER_COMMENT)
+      end
+    end
+  end
+
+  private
+
+  def add_comment(line, comment, **metadata)
+    comment_object = build_comment(line, comment, **metadata)
+    code_objects << comment_object
+  end
+end

data/lib/comment_extractor/extractor/sass.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Sass < CommentExtractor::Extractor
+  filename /\.sass$/
+  filetype 'sass'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/scala.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Scala < CommentExtractor::Extractor
+  filename /\.scala$/
+  filetype 'scala'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/scss.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Scss < CommentExtractor::Extractor
+  filename /\.scss$/
+  filetype 'scss'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/shell.rb

@@ -0,0 +1,11 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Shell < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /(?:\.(?:zsh|bash|sh)|zshrc|bashrc)$/
+  filetype %w[bash sh zsh]
+
+  define_default_bracket
+  comment start_with: '#'
+end

data/lib/comment_extractor/extractor/sqf.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Sqf < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+
+  filename /\.sqf$/
+  filetype 'sqf'
+end

data/lib/comment_extractor/extractor/sql.rb

@@ -0,0 +1,12 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Sql < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.sql$/
+  filetype 'sql'
+
+  define_default_bracket
+  comment start_with: '--'
+  comment start_with: '/\*', end_with: '\*/', type: BLOCK_COMMENT
+end

data/lib/comment_extractor/extractor/sqs.rb

@@ -0,0 +1,7 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Sqs < CommentExtractor::Extractor
+  disable!
+  filename /\.sqs$/
+  filetype 'sqs'
+end

data/lib/comment_extractor/extractor/tex.rb

@@ -0,0 +1,12 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Tex < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.tex$/
+  filetype 'tex'
+
+  define_bracket '{'
+  comment start_with: /(?<!\\)%/
+  define_ignore_patterns /\\%/
+end

data/lib/comment_extractor/extractor/text.rb

@@ -0,0 +1,10 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Text < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.txt$/
+  filetype 'text'
+
+  comment start_with: /#/
+end

data/lib/comment_extractor/extractor/yaml.rb

@@ -0,0 +1,12 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Yaml < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.yml$/
+  filetype 'yaml'
+
+  define_ignore_patterns /^\s*[^#]+$/
+  comment start_with: /^s*#/
+  comment start_with: /\s#(?=[^#]*)$/
+end

data/lib/comment_extractor/extractor.rb

@@ -0,0 +1,96 @@
+require 'strscan'
+require 'comment_extractor/code_object'
+require 'comment_extractor/code_objects'
+require 'comment_extractor/extractor/concerns/simple_extractor'
+require 'comment_extractor/extractor/concerns/slash_extractor'
+require 'comment_extractor/version'
+
+module CommentExtractor
+  class Extractor
+    class Error < RuntimeError; end
+    class SyntaxDefinitionError < RuntimeError; end
+
+    REGEXP = {
+      BREAK: /(?:\r?\n|\r)/,
+    }.freeze
+    SCHAME_ACCESSOR_NAMES = %i[shebang filetype filename]
+
+    attr_reader :content, :code_objects
+
+    def self.disabled?
+      @status == :disable
+    end
+
+    def self.disable!
+      @status = :disable
+    end
+
+    def self.schema_accessor(*keys)
+      keys.each do |key|
+        define_singleton_method key do |value = nil|
+        if value
+          self.schema[key] = value
+        else
+          self.schema[key]
+        end
+        end
+      end
+    end
+    schema_accessor *SCHAME_ACCESSOR_NAMES
+
+    def self.schema
+      @schema ||= {}
+    end
+
+    def initialize(content, code_objects = nil)
+      @content = content
+      @code_objects = code_objects || CodeObjects.new
+    end
+
+    # #extract_comments should retrun CodeObjects contains instance
+    # of CodeObject::Comment
+    def extract_comments
+      @extracted_comments ||= begin
+        scan
+        code_objects
+      end
+    end
+
+    protected
+
+    def scan
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def scanner
+      @scanner ||= build_scanner
+    end
+
+    def build_scanner
+      StringScanner.new(@content)
+    end
+
+    def build_comment(line, comment, **metadata)
+      metadata[:extractor] = self
+      CodeObject::Comment.new(line: line, value: comment, **metadata)
+    end
+
+    private
+
+    def raise_report
+      content = "Content:\n#{@content}"
+
+      raise SyntaxDefinitionError, <<-MSG.gsub(/^\s*/, '') + content
+      Error occurred.
+      Please report to <https://github.com/alpaca-tc/comment_extractor/issues>
+
+      - - -
+
+      CommentExtractor #{CommentExtractor::VERSION}
+
+      Date: #{Time.now}
+      Extractor: #{self.class}
+      MSG
+    end
+  end
+end

data/lib/comment_extractor/extractor_manager.rb

@@ -0,0 +1,158 @@
+require 'comment_extractor'
+require 'comment_extractor/extractor'
+require 'comment_extractor/file'
+
+using CommentExtractor::DetectableSchemeFile
+
+module CommentExtractor
+  module ExtractorManager
+    class << self
+      def default_extractors
+        %i[
+          C Cc Class Clojure Coffee Cpp
+          Cs Css Cxx D Erlang Fortran Go H Haml
+          Haskell Hpp Html Java JavaScript Lisp
+          Lua M Markdown Mm Perl Php Python
+          Ruby Sass Scala Scss Shell Sqf
+          Sql Sqs Tex Yaml
+        ]
+      end
+
+      def regist_extractor(klass_or_symbol)
+        @extractor_definitions = nil
+        extractor = klass_or_symbol.is_a?(Extractor) ? klass_or_symbol : nil
+        extractors[:"#{klass_or_symbol}"] = extractor
+
+        unless extractor
+          filename = "#{klass_or_symbol}".gsub(/\W/, '').gsub(/::/, '/').
+            gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+            gsub(/([a-z\d])([A-Z])/,'\1_\2').
+            tr("-", "_").
+            downcase
+          file_path = "comment_extractor/extractor/#{filename}"
+          Extractor.autoload klass_or_symbol, file_path
+        end
+      end
+
+      def can_extract(file_path)
+        return if File.binary?(file_path)
+
+        extractor = nil
+        if shebang = File.shebang(file_path)
+          extractor = find_extractor_by_shebang(shebang)
+        end
+
+        unless extractor
+          extractor = find_extractor_by_filename(file_path)
+        end
+
+        if ::CommentExtractor.configuration.use_default_extractor
+          extractor = default_extractor unless extractor
+        end
+
+        extractor
+      end
+
+      private
+
+      def defined_extractor_finders
+        @defined_extractor_finders ||= []
+      end
+
+      def define_extractor_finder_by(*keys)
+        defined_extractor_finders.concat(keys)
+
+        keys.each do |key|
+          define_singleton_method "find_extractor_by_#{key}" do |value|
+            find_extractor_by(key, value)
+          end
+        end
+      end
+
+      def initialize_extractors!(new_extractors = default_extractors)
+        new_extractors.each do |extractor|
+          self.regist_extractor(extractor)
+        end
+
+        self
+      end
+
+      def find_extractor_by(key, value)
+        case key
+        when :filename, :shebang
+          # Regexp optimization which can find value O(1)
+          if extractor_definitions[key][:regexp] =~ value
+            index = $~[1..-1].rindex($~[0])
+            extractor_definitions[key][:values][index]
+          end
+        when :filetype
+          extractor_definitions[:filetype][value]
+        end
+      end
+
+      def extractor_definitions
+        @extractor_definitions ||= build_extractor_definitions
+      end
+
+      def build_extractor_definitions
+        definitions = Hash.new { |h,k| h[k] = { regexp: nil, values: [] } }
+
+        finders = defined_extractor_finders.dup
+        finders.delete(:filetype)
+        definitions[:filetype] = build_filetype_extractor_definitions
+
+        finders.each do |finder|
+          regexp_keys = []
+          values = []
+
+          extractors.each do |name, value|
+            extractor = extractors[name] = value || Extractor.const_get(name)
+
+            next if extractor.disabled?
+
+            if schema = extractor.send(finder)
+              # [review] - Maybe my optimization way is not better
+              regexp_source = schema.is_a?(Regexp) ? schema.source : schema
+              regexp_keys << "(#{regexp_source})"
+              values << extractor
+            end
+          end
+
+          unless values.empty?
+            definitions[finder][:values] = values
+            definitions[finder][:regexp] = Regexp.new(regexp_keys.join('|'))
+          end
+        end
+
+        definitions
+      end
+
+      def build_filetype_extractor_definitions
+        definitions = Hash.new { |h,k| h[k] = [] }
+
+        extractors.each_with_object(definitions) do |(name, value), memo|
+          extractor = extractors[name] = value || Extractor.const_get(name)
+          filetypes = *extractor.filetype
+          filetypes.each { |filetype| memo[filetype] = extractor }
+        end
+      end
+
+      def extractors
+        return @extractors if @extractors
+        @extractors = {}
+        initialize_extractors!
+
+        @extractors
+      end
+
+      private
+
+      def default_extractor
+        ::CommentExtractor.configuration.default_extractor
+      end
+    end
+
+    # define :find_extractor_by_shebang, :find_extractor_by_filename
+    define_extractor_finder_by *Extractor::SCHAME_ACCESSOR_NAMES
+  end
+end