comment_extractor 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 81a66d3881e6e790f7b4fbd5626661f6b6b7ebce
+  data.tar.gz: eef85195b570687ca4e6eafbbe841de336884233
+SHA512:
+  metadata.gz: 918fd7799b29841b8d1ca3a23329ba8c59f48635806dfb29b68e993ec85f20150fe33170db4eb3c228b2d7326503ac76726fca9bbe06f364eac489e81c84d0d2
+  data.tar.gz: ee4337e67ac7c371d0cad3b83991b379b9d0dcc97e4c709935750b82fc277b75107d4874272876c9954a30dc893f7be16b901e93ca5a6242afe0610d5b11cfe6

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) <2014> <Ishii Hiroyuki>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,132 @@
+# comment\_extractor
+
+[![Gem Version](https://badge.fury.io/rb/comment_extractor.png)](http://badge.fury.io/rb/comment\_extractor)
+[![Build Status](https://travis-ci.org/alpaca-tc/comment_extractor.png?branch=v1.0.0)](https://travis-ci.org/alpaca-tc/comment\_parser)
+[![Coverage Status](https://coveralls.io/repos/alpaca-tc/comment_extractor/badge.png?branch=v1.0.0)](https://coveralls.io/r/alpaca-tc/comment\_extractor?branch=v1.0.0)
+
+## Description
+
+comment\_extractor extracts the comment out from a source code.
+
+## Installation
+
+CommentExtractor has been tested with ruby 2.1.
+
+```sh
+git clone https://github.com/alpaca-tc/comment_extractor
+cd comment_extractor
+rake install
+```
+
+## Usage
+
+### Parser
+
+Given a file path to `Parser.for`, it finds Extractor and returns an instance of self which is initialized by extractor. Getting the comments from file by using it.
+
+```ruby
+require 'comment_extractor'
+
+path = 'path/to/file'
+if parser = CommentExtractor::Parser.for(path)
+  comments = parser.parse
+  comemnts.is_a?(CommentExtractor::CodeObjects)
+
+  comment = comments.first
+  comment.file  #=> 'path/to/file'
+  comment.line  #=> 1
+  comment.value #=> 'I am a comment'
+end
+```
+
+### Extractor
+
+#### You can use Extractor directly.
+
+```ruby
+require 'comment_extractor'
+
+file_path = 'path/to/file.rb'
+manager = CommentExtractor::ExtractorManager
+if extractor = manager.can_extract(file_path)
+  content = File.read(file_path)
+  comments = extractor.new(content).extract_comments
+  comemnts.is_a?(CommentExtractor::CodeObjects)
+end
+
+# Other way to find extractor
+extractor = manager.find_extractor_by_shebang('#! /usr/local/bin/ruby')
+extractor = manager.find_extractor_by_filename('path/to/file.rb')
+extractor = manager.find_extractor_by_filetype('ruby')
+```
+
+#### How to use extractor of specific filetype.
+
+```ruby
+require 'comment_extractor/extractor/d'
+
+content = File.read('path/to/file.d')
+comments = CommentExtractor::Extractor::D.new(content).extract_comments
+```
+
+### Supported FileTypes
+
+- **Bash / Zsh**
+- **C / C++**
+- **Class**
+- **C#**
+- **Clojure**
+- **Coffee-Script**
+- **D**
+- **EmacsLisp**
+- **Erlang**
+- **Fortran**
+- **Go**
+- **Haml**
+- **Haskell**
+- **HTML**
+- **Java**
+- **JavaScript**
+- **Tex**
+- **Lua**
+- **PHP**
+- **Perl**
+- **Python**
+- **Ruby**
+- **SASS**
+- **SCSS**
+- **SQF**
+- **SQL**
+- **Scala**
+
+### TODO
+
+- Markdown
+- SQS; I can not implement it because I do not know the syntax of sqs.
+
+### Create a new Extractor
+
+If you see something missing from the supported file type, please either file an issue or submit a pull request:)
+And I would be glad if I could have you send the new filetype's source code via an issues.
+
+```ruby
+# lib/comment_extractor/extractor/file_type.rb
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::FileType < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  shebang /ruby$/            # (Optional)
+  filename /\.(extention)$/  # (Required)
+  filetype 'filetype'        # (Required) file type name. g.c 'ruby', 'python'
+
+  # define_ignore_patterns(*given regexp)
+
+  # define_bracket('"')   #=> define_ignore_patterns(/".*?(?<!\\)"/)
+  # define_regexp_bracket #=> define_ignore_patterns(%r!/(?=[^/])!, /(?<!\\)\//)
+
+  # define the rule of comment
+  comment start_with: /;+/
+  comment start_with: /;--/, end_with: /--\|/, type: BLOCK_COMMENT
+end
+```

data/bin/comment_parser_debug

@@ -0,0 +1,45 @@
+#! /usr/bin/env ruby
+
+$:.unshift File.expand_path('../../lib', __FILE__)
+require 'comment_extractor'
+
+class CommentExtractor::Debugger
+  def initialize(file_path)
+    @files = if File.file?(file_path)
+               [file_path]
+             elsif File.directory?(file_path)
+               Dir["#{file_path}/**/*"].select { |f| File.file?(f) }
+             end
+    @debug = Hash.new { |h,k| h[k] = [] }
+  end
+
+  def parse_all_files
+    @files.each do |file|
+      parse_file(file)
+    end
+
+    puts "Parser is not found"
+    puts @debug[:parser_not_found].join("\n")
+  end
+
+  def parse_file(file_path)
+    puts "Open: #{file_path}"
+
+    if parser = CommentExtractor::Parser.for(file_path)
+      puts "Use: #{parser.extractor.class}"
+      comments = parser.extract_comments
+    else
+      @debug[:parser_not_found] << file_path
+    end
+  end
+end
+
+CommentExtractor.configure do |c|
+  c.use_default_extractor = false
+end
+
+file_path = ARGV.first
+exit unless file_path
+
+debugger = CommentExtractor::Debugger.new(file_path)
+debugger.parse_all_files

data/lib/comment_extractor/code_object/comment.rb

@@ -0,0 +1,19 @@
+require 'comment_extractor/code_object'
+
+module CommentExtractor
+  class CodeObject
+    class Comment < CommentExtractor::CodeObject
+      attr_accessor :line
+
+      module Type
+        ONE_LINER_COMMENT = :one_liner_comment
+        BLOCK_COMMENT = :block_comment
+      end
+
+      def initialize(line: line, **values)
+        super(**values)
+        @line = line
+      end
+    end
+  end
+end

data/lib/comment_extractor/code_object.rb

@@ -0,0 +1,12 @@
+module CommentExtractor
+  class CodeObject
+    attr_accessor :metadata, :value
+
+    def initialize(value: nil, **others)
+      @value = value
+      @metadata = others
+    end
+  end
+end
+
+require 'comment_extractor/code_object/comment'

data/lib/comment_extractor/code_objects.rb

@@ -0,0 +1,46 @@
+require 'delegate'
+
+module CommentExtractor
+  class CodeObjects < DelegateClass(Array)
+    attr_accessor :file
+
+    def initialize(file: nil)
+      @file = file
+      super([])
+    end
+
+    def <<(code_object)
+      super(initialize_code_object(code_object))
+    end
+
+    def push(*code_object_array)
+      arguments = code_object_array.map { |v| initialize_code_object(v) }
+      super(*arguments)
+    end
+
+    def concat(*code_object_arrays)
+      arguments = code_object_arrays.flatten.map { |v| initialize_code_object(v) }
+      super(arguments)
+    end
+
+    def inspect
+      attributes = instance_variables.map { |v| "@#{v}=#{instance_variable_get(v)}" }
+      attributes = attributes.empty? ? '' : " #{attributes.join(', ')}"
+      object_id = '0x%x' % (self.object_id << 1)
+      "#<#{self.class}:#{object_id}#{attributes}>"
+    end
+    alias :to_s :inspect
+
+    private
+
+    def initialize_code_object(code_object)
+      unless code_object.is_a?(CodeObject)
+        message = "no implicit conversion of #{code_object.class} into #{CodeObject}"
+        raise TypeError, message
+      end
+
+      code_object.metadata[:parent] = self
+      code_object
+    end
+  end
+end

data/lib/comment_extractor/configuration.rb

@@ -0,0 +1,50 @@
+require 'comment_extractor/extractor_manager'
+require 'comment_extractor/extractor/text'
+
+module CommentExtractor
+  class Configuration
+    @@required_attributes = {}
+
+    def initialize(attributes = {})
+      attributes.each do |key, value|
+        method_name = "#{key}="
+        send(method_name, value) if respond_to?(method_name)
+      end
+
+      @@required_attributes.each_key do |key|
+        raise "Unable to initialize #{key} without attribute" unless self.send(key)
+      end
+
+      self.extractors = ExtractorManager.default_extractors
+      self.default_extractor = Extractor::Text
+      self.use_default_extractor = true
+    end
+
+    def self.add_setting(name, opts={})
+      attr_accessor name
+
+      define_predicate_for(name) if opts.delete(:predicate)
+      define_required_attribute(name) if opts.delete(:required)
+    end
+
+    private
+
+    def self.define_required_attribute(*names)
+      names.each do |name|
+        @@required_attributes[name] = nil
+      end
+    end
+
+    def self.define_predicate_for(*names)
+      names.each do |name|
+        define_method "#{name}?" do
+          !!send(name)
+        end
+      end
+    end
+
+    add_setting :extractors
+    add_setting :default_extractor
+    add_setting :use_default_extractor
+  end
+end

data/lib/comment_extractor/encoding.rb

@@ -0,0 +1,40 @@
+module CommentExtractor
+  module Encoding
+    def self.read_file(file_path, encoding = ::Encoding.default_external)
+      content = File.open(file_path, 'rb') { |f| f.read }
+      self.encode(content)
+    end
+
+    def self.encode(content, encoding = ::Encoding.default_external)
+      windows_platforms = Regexp.new(%w[mingw mswin].join('|'))
+      content.gsub!("\r\n", "\n") if RUBY_PLATFORM =~ windows_platforms
+
+      original_encoding = content.encoding
+
+      if strip_bom(content) # When the content contains bom, it is UTF-8
+        content.force_encoding(::Encoding::UTF_8)
+        content.encode!(encoding)
+      else
+        content.force_encoding(encoding)
+      end
+
+      unless content.valid_encoding?
+        content.force_encoding(original_encoding)
+        content.encode!(encoding)
+      end
+
+      unless content.valid_encoding?
+        raise "Unable to convert #{file_path} to #{encoding}"
+      end
+
+      content
+    end
+
+    private
+
+    def self.strip_bom(content)
+      bom_regexp = /\A\xef\xbb\xbf/
+      content.sub!(bom_regexp, '')
+    end
+  end
+end

data/lib/comment_extractor/extractor/c.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::C < CommentExtractor::Extractor
+  filename /\.c$/
+  filetype 'c'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/cc.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Cc < CommentExtractor::Extractor
+  filename /\.cc$/
+  filetype 'cc'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/class.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Class < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+
+  filename /\.class$/
+  filetype 'class'
+end

data/lib/comment_extractor/extractor/clojure.rb

@@ -0,0 +1,11 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Clojure < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.clj$/
+  filetype 'clojure'
+
+  define_default_bracket
+  comment start_with: /;+/
+end

data/lib/comment_extractor/extractor/coffee.rb

@@ -0,0 +1,13 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Coffee < CommentExtractor::Extractor
+  include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+  filename /\.coffee$/
+  filetype 'coffee'
+
+  define_default_bracket
+  define_regexp_bracket
+  comment start_with: '###', end_with: '###', type: BLOCK_COMMENT
+  comment start_with: '#'
+end

data/lib/comment_extractor/extractor/concerns/simple_extractor.rb

@@ -0,0 +1,189 @@
+require 'comment_extractor/smart_string_scanner'
+
+using CommentExtractor::SmartStringScanner
+
+class CommentExtractor::Extractor
+  module Concerns
+    module SimpleExtractor
+      include CommentExtractor::CodeObject::Comment::Type
+
+      def self.included(k)
+        k.class_eval do |klass|
+          extend ClassMethods
+        end
+      end
+
+      def self.attr_definition(*keys)
+        keys.each do |key|
+          define_method key do
+            self.class.instance_variable_get("@#{key}") || []
+          end
+        end
+      end
+      attr_definition :brackets, :comment_regexp,
+        :ignore_patterns, :complicate_conditions
+
+      module ClassMethods
+        include CommentExtractor::CodeObject::Comment::Type
+
+        def included(k)
+          self.instance_variables.each do |key|
+            k.instance_variable_set(key, self.instance_variable_get(key))
+          end
+        end
+
+        def comment(start_with: nil, end_with: nil, type: ONE_LINER_COMMENT)
+          @comment_regexp ||= []
+          raise ArgumentError unless [type, start_with].all?
+
+          definition = { start_with: build_regexp(start_with), type: type, end_with: end_with }
+
+          if type == BLOCK_COMMENT
+            definition[:end_with] = build_regexp(end_with, Regexp::MULTILINE)
+          end
+          @comment_regexp << definition
+        end
+
+        def define_ignore_patterns(*patterns)
+          @ignore_patterns ||= []
+          @ignore_patterns += patterns
+        end
+
+        def define_bracket(bracket, options = 0)
+          start_regexp = build_regexp(bracket)
+          stop_regexp = if bracket.is_a?(Regexp)
+                          join_regexp(/(?<!\\)/, bracket)
+                        else
+                          /(?<!\\)#{bracket}/
+                        end
+          stop_regexp = Regexp.new(stop_regexp.source, options)
+          append_bracket(start_regexp, stop_regexp)
+        end
+
+        def define_regexp_bracket
+          append_bracket(%r!/(?=[^/])!, /(?<!\\)\//)
+        end
+
+        def define_default_bracket
+          define_bracket('"', Regexp::MULTILINE)
+          define_bracket("'", Regexp::MULTILINE)
+        end
+
+        def append_bracket(start_with, end_with)
+          @brackets ||= []
+          @brackets << { start_with: start_with, end_with: end_with }
+        end
+
+        def define_complicate_condition(&proc_object)
+          @complicate_conditions ||= []
+          @complicate_conditions << proc_object
+        end
+
+        private
+
+        def join_regexp(*regexp)
+          # [review] - Should I ignore regexp options?
+          Regexp.new(regexp.map { |v| v.source }.inject(:+))
+        end
+
+        def build_regexp(str_or_reg, type = 0)
+          str_or_reg = str_or_reg.source if str_or_reg.respond_to?(:source)
+          Regexp.new(str_or_reg, type)
+        end
+      end
+
+      def scan
+        until scanner.eos?
+          case
+          when scan_ignore_patterns
+            next
+          when scan_complicate_conditions
+            next
+          when scan_comment
+            next
+          when scan_bracket
+            next
+          when scanner.scan(CommentExtractor::Extractor::REGEXP[:BREAK])
+            next
+          when scanner.scan(/./)
+            next
+          else
+            raise_report
+          end
+        end
+      end
+
+      private
+
+      def scan_complicate_conditions
+        complicate_conditions.each do |proc_object|
+          return if self.instance_eval(&proc_object)
+        end
+
+        nil
+      end
+
+      def scan_bracket
+        brackets.each do |definition|
+          start_with = definition[:start_with]
+          end_with = definition[:end_with]
+          next unless scanner.scan(start_with)
+
+          new_regexp = Regexp.new(/.*?/.source + end_with.source, end_with.options)
+          return scanner.scan(new_regexp)
+        end
+
+        nil
+      end
+
+      def scan_ignore_patterns
+        ignore_patterns.each do |pattern|
+          return true if scanner.scan(pattern)
+        end
+
+        nil
+      end
+
+      def scan_comment
+        comment_regexp.each do |definition|
+          next unless scanner.scan(definition[:start_with])
+
+          result = case definition[:type]
+                   when ONE_LINER_COMMENT
+                     identify_single_line_comment
+                   when BLOCK_COMMENT
+                     identify_multi_line_comment(definition[:end_with])
+                   else
+                     raise_report
+                   end
+
+          return result
+        end
+
+        nil
+      end
+
+      def identify_single_line_comment
+        line_number = scanner.current_line
+        comment = scanner.scan(/^.*$/)
+        metadata = { type: ONE_LINER_COMMENT }
+        comment_object = build_comment(line_number, comment, **metadata)
+
+        code_objects << comment_object
+      end
+
+      def identify_multi_line_comment(regexp)
+        line_no = scanner.current_line
+        stop_regexp = Regexp.new(/.*?/.source + regexp.source, regexp.options)
+        comment_block = scanner.scan(stop_regexp)
+
+        remove_tail_regexp = Regexp.new(regexp.source + /$/.source)
+        comments = comment_block.sub(remove_tail_regexp, '').split("\n")
+        comments.each_with_index do |comment, index|
+          metadata = { type: BLOCK_COMMENT }
+          code_objects << build_comment(line_no + index, comment, metadata)
+        end
+      end
+    end
+  end
+end

data/lib/comment_extractor/extractor/concerns/slash_extractor.rb

@@ -0,0 +1,16 @@
+require 'comment_extractor/extractor'
+require 'comment_extractor/extractor/concerns/simple_extractor'
+
+module CommentExtractor
+  class Extractor
+    module Concerns
+      module SlashExtractor
+        include CommentExtractor::Extractor::Concerns::SimpleExtractor
+
+        define_default_bracket
+        comment start_with: /\/\//
+        comment start_with: /\/\*/, end_with: /\*\//, type: BLOCK_COMMENT
+      end
+    end
+  end
+end

data/lib/comment_extractor/extractor/cpp.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Cpp < CommentExtractor::Extractor
+  filename /\.cpp$/
+  filetype 'cpp'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/cs.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Cs < CommentExtractor::Extractor
+  filename /\.cs$/
+  filetype 'cs'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/css.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Css < CommentExtractor::Extractor
+  filename /\.css$/
+  filetype 'css'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/cxx.rb

@@ -0,0 +1,8 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::Cxx < CommentExtractor::Extractor
+  filename /\.cxx$/
+  filetype 'cxx'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end

data/lib/comment_extractor/extractor/d.rb

@@ -0,0 +1,9 @@
+require 'comment_extractor/extractor'
+
+class CommentExtractor::Extractor::D < CommentExtractor::Extractor
+  shebang /\/dmd/
+  filename /\.d$/
+  filetype 'd'
+
+  include CommentExtractor::Extractor::Concerns::SlashExtractor
+end