issuesrc 0.0.3

data/lib/issuesrc.rb

@@ -0,0 +1,211 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+
+require 'issuesrc/version'
+require 'issuesrc/config'
+require 'issuesrc/tag_extractor'
+require 'issuesrc/event_loop'
+
+module Issuesrc
+  DEFAULT_SOURCER = 'github'
+  DEFAULT_ISSUER = 'github'
+  DEFAULT_TAG_FINDERS = ['blunt']
+
+  SOURCERS = {
+    'git' => ['sourcers/git_sourcer', 'GitSourcer'],
+    'github' => ['sourcers/github_sourcer', 'GithubSourcer'],
+  }
+
+  ISSUERS = {
+    'github' => ['issuers/github_issuer', 'GithubIssuer'],
+  }
+
+  TAG_FINDERS = {
+    'blunt' => ['tag_finders/blunt_tag_finder', 'BluntTagFinder'],
+  }
+
+  # Run issuesrc.
+  def self.run(args, config)
+    program = Class.new { include Issuesrc }.new
+    program.set_config(args, config)
+    program.init_files_offsets()
+
+    event_loop = Issuesrc::SequentialEventLoop.new()
+    sourcer = program.load_sourcer()
+    tag_finders = program.load_tag_finders()
+    issuer = program.load_issuer(event_loop)
+
+    issues = issuer.async_load_issues()
+
+    created_tags, updated_tags, closed_issues = [], [], []
+
+    sourcer.retrieve_files().each do |file|
+      if Issuesrc::Config::option_from_args(:verbose, args)
+        puts file.path
+      end
+
+      tag_finder = program.select_tag_finder_for(file, tag_finders)
+      if tag_finder.nil?
+        next
+      end
+
+      tags = []
+      tag_finder.find_tags(file) { |tag| tags << tag }
+
+      tags_by_issue_id, new_tags = program.classify_tags(tags, file)
+
+      new_tags.each do |tag|
+        created_tags << tag
+        issuer.async_create_issue(tag) do |tag|
+          program.save_tag_in_file(tag)
+        end
+      end
+
+      issuer.async_update_or_close_issues(issues, tags_by_issue_id) do
+      |issue_id, tag, action|
+        case action
+        when :updated
+          program.save_tag_in_file(tag)
+          updated_tags << tag
+        when :closed
+          closed_issues << issue_id
+        end
+      end
+    end
+
+    event_loop.wait_for_pending()
+
+    if sourcer.respond_to? :finish
+      sourcer.finish(created_tags, updated_tags, closed_issues)
+    end
+  end
+
+  def set_config(args, config)
+    @args = args
+    @config = config
+  end
+
+  def init_files_offsets
+    @files_offsets = {}
+  end
+
+  # Creates the instance of the sourcer that should be used for the current
+  # execution of issuesrc. It looks first at the :sourcer command line
+  # argument, then `[sourcer] sourcer = ...` from the config file. If those are
+  # not present, `DEFAULT_SOURCER` will be used. If the selected sourcer is not
+  # implemented, ie. is not a key of `SOURCERS`, the execution will fail.
+  def load_sourcer
+    path, cls = load_component(
+      ['sourcer', 'sourcer'],
+      :sourcer,
+      DEFAULT_SOURCER,
+      SOURCERS)
+    do_require(path)
+    make_sourcer(cls)
+  end
+
+  def make_sourcer(cls)
+    Issuesrc::Sourcers.const_get(cls).new(@args, @config)
+  end
+
+  # Like `load_sourcer`, but for the issuer. It first looks at :issuer from
+  # the command line arguments, then `[issuer] issuer = ...` from the config
+  # file.
+  def load_issuer(event_loop)
+    path, cls = load_component(
+      ['issuer', 'issuer'],
+      :issuer,
+      DEFAULT_ISSUER,
+      ISSUERS)
+    do_require(path)
+    make_issuer(cls, event_loop)
+  end
+
+  def make_issuer(cls, event_loop)
+    Issuesrc::Issuers.const_get(cls).new(@args, @config, event_loop)
+  end
+
+  def load_component(config_key, arg_key, default, options)
+    type = Config::option_from_both(arg_key, config_key, @args, @config)
+    if type.nil?
+      type = default
+    end
+    load_component_by_type(type, options)
+  end
+
+  def load_component_by_type(type, options)
+    if !options.include?(type)
+      exec_fail 'Unrecognized sourcer type: #{type}'
+    end
+
+    options[type]
+  end
+
+  # Like `load_sourcer` but for the tag finders. It only looks at
+  # `[tag_finders] tag_finders = [...]` from the config file.
+  def load_tag_finders
+    tag_finders = Config::option_from_config(
+      ['tag_finders', 'tag_finders'], @config)
+    if tag_finders.nil?
+      tag_finders = DEFAULT_TAG_FINDERS
+    end
+    load_tag_finders_by_types(tag_finders)
+  end
+
+  def load_tag_finders_by_types(types)
+    tag_extractor = Issuesrc::TagExtractor.new(@args, @config)
+    tag_finders = []
+    types.each do |type|
+      path, cls = load_component_by_type(type, TAG_FINDERS)
+      do_require(path)
+      tag_finders << make_tag_finder(cls, tag_extractor)
+    end
+    tag_finders
+  end
+
+  def make_tag_finder(cls, tag_extractor)
+    Issuesrc::TagFinders.const_get(cls).new(tag_extractor, @args, @config)
+  end
+
+  def select_tag_finder_for(file, tag_finders)
+    ret = nil
+    tag_finders.each do |tag_finder|
+      if tag_finder.accepts? file
+        ret = tag_finder
+        break
+      end
+    end
+    ret
+  end
+
+  def classify_tags(tags, file)
+    tags_by_issue = {}
+    new_tags = []
+    tags.each do |tag|
+      if tag.issue_id.nil?
+        new_tags << tag
+      else
+        tags_by_issue[tag.issue_id] = tag
+      end
+    end
+    [tags_by_issue, new_tags]
+  end
+
+  def save_tag_in_file(tag)
+    offsets = @files_offsets.fetch(tag.file.path, [])
+    offsets = tag.write_in_file(offsets)
+    @files_offsets[tag.file.path] = offsets
+  end
+
+  def self.exec_fail(feedback)
+    raise IssuesrcError, feedback
+  end
+
+  def do_require(path)
+    require path
+  end
+
+  class IssuesrcError < Exception; end
+end

data/lib/sourcers/git_sourcer.rb

@@ -0,0 +1,238 @@
+require 'tmpdir'
+require 'open3'
+require 'find'
+require 'issuesrc/file'
+require 'issuesrc/config'
+require 'ptools'
+
+module Issuesrc
+  module Sourcers
+    class GitSourcer
+      def initialize(args, config)
+        if @url.nil?
+          @url = try_find_repo_url(args, config)
+        end
+
+        if @url.nil?
+          @path = try_find_repo_path(args, config)
+        end
+
+        init(args, config)
+      end
+
+      def initialize_with_url(url, args, config)
+        @url = url
+        init(args, config)
+      end
+
+      def initialize_with_path(path, args, config)
+        @path = path
+        init(args, config)
+      end
+
+      def retrieve_files()
+        dir = nil
+        if !@url.nil?
+          tmp_dir = Dir::mktmpdir
+          @tmp_dir = tmp_dir
+          dir = clone_repo(tmp_dir)
+          @path = dir
+        else
+          dir = @path
+        end
+
+        Enumerator.new do |enum|
+          find_all_files(dir) do |file|
+            enum << file
+          end
+        end
+      end
+
+      def finish(created_tags, updated_tags, closed_issues)
+        if created_tags.empty? && closed_issues.empty?
+          return
+        end
+
+        msg = make_commit_message(created_tags, updated_tags, closed_issues)
+        $stderr.puts msg
+
+        if @commit_when_done
+          make_commit(msg)
+        end
+        
+        if @push_when_done
+          push_repo()
+        end
+
+        if @tmp_dir
+          FileUtils.remove_entry_secure @tmp_dir
+        end
+      end
+
+      private
+      def init(args, config)
+        init_exclude(config)
+        @commit_when_done = decide_commit_when_done(args, config)
+        @push_when_done = decide_push_when_done(args, config)
+      end
+
+      def init_exclude(config)
+        @exclude = Issuesrc::Config::option_from_config(
+          ['sourcer', 'exclude_files'], config)
+        if @exclude.nil?
+          @exclude = []
+        end
+      end
+
+
+      def try_find_repo_url(args, config)
+        Issuesrc::Config::option_from_both(:repo_url, ['git', 'repo'],
+                                           args, config)
+      end
+
+      def try_find_repo_path(args, config)
+        Issuesrc::Config::option_from_both(:repo_path, ['git', 'repo_path'],
+                                           args, config)
+      end
+
+      def decide_commit_when_done(args, config)
+        decide_when_done(:commit_when_done, 'commit_when_done', args, config)
+      end
+
+      def decide_push_when_done(args, config)
+        decide_when_done(:push_when_done, 'push_when_done', args, config)
+      end
+
+      def decide_when_done(args_key, config_key, args, config)
+        opt = Issuesrc::Config::option_from_both(
+          args_key, ['git', config_key], args, config)
+        if opt.nil?
+          opt = is_downloaded?
+        end
+        opt
+      end
+
+      def is_downloaded?
+        !@url.nil?
+      end
+
+      def clone_repo(dir)
+        out, err, status = Open3.capture3 "git clone #{@url} #{dir}/repo"
+        if status != 0
+          raise err
+        end
+        repo_dir = dir + '/repo'
+        change_branch_in_clone(repo_dir)
+        repo_dir
+      end
+
+      def find_all_files(dir)
+        set_branch_from_clone(dir)
+        Find.find(dir) do |path|
+          if FileTest.directory?(path) || File.binary?(path)
+            if File.basename(path) == '.git'
+              Find.prune
+            end
+            next
+          end
+
+          excluded = false
+          @exclude.each do |exc|
+            if File.fnmatch?(exc, path_in_repo(path))
+              excluded = true
+              next
+            end
+          end
+          if excluded
+            next
+          end
+
+          yield Issuesrc::GitFile.new(path, path_in_repo(path), @branch)
+        end
+      end
+
+      def path_in_repo(path)
+        path[@path.length + 1..-1]
+      end
+
+      def change_branch_in_clone(repo_dir)
+        if @branch.nil?
+          return
+        end
+
+        pwd = Dir::pwd
+        Dir::chdir(repo_dir)
+        out, err, status = Open3.capture3 "git checkout #{@branch}"
+        if status != 0
+          Dir::chdir(pwd)
+          raise err
+        end
+        Dir::chdir(pwd)
+      end
+
+      def set_branch_from_clone(repo_dir)
+        pwd = Dir::pwd
+        Dir::chdir(repo_dir)
+        out, err, status = Open3.capture3 "git rev-parse --abbrev-ref HEAD"
+        if status != 0
+          Dir::chdir(pwd)
+          raise err
+        end
+        @branch = out[0..-2]
+        Dir::chdir(pwd)
+      end
+
+      def make_commit_message(created_tags, updated_tags, closed_issues)
+        s = "Issuesrc: Synchronize issues from source code."
+        created_issues = created_tags.map { |x| x.issue_id }
+        s << make_commit_message_issues("Opens", created_issues)
+        s << make_commit_message_issues("Fixes", closed_issues)
+      end
+
+      def make_commit_message_issues(action, issue_ids)
+        if issue_ids.empty?
+          return ""
+        end
+
+        s = "\n\n"
+        parts = []
+        issue_ids.each do |issue_id|
+          parts << "#{action} \##{issue_id}"
+        end
+        s << parts.join("\n")
+        s
+      end
+
+      def make_commit(msg)
+        prev_dir = Dir::pwd
+        Dir::chdir(@path)
+        Open3.popen3("git commit -a --file -") do |fin, fout, ferr, proc|
+          fin.write(msg + "\n")
+          fin.close
+          out = fout.read 
+          err = ferr.read
+          status = proc.value
+          fout.close
+          ferr.close
+          if status.to_i != 0
+            Dir::chdir(prev_dir)
+            puts "error committing: #{out} #{err}"
+            next
+          end
+        end
+        Dir::chdir(prev_dir)
+      end
+
+      def push_repo
+        prev_dir = Dir::pwd
+        Dir::chdir(@path)
+        out, err, status = Open3.capture3 "git push"
+        if status != 0
+          Dir::chdir(prev_dir)
+          raise err
+        end
+        Dir::chdir(prev_dir)
+      end
+    end
+  end
+end

data/lib/sourcers/github_sourcer.rb

@@ -0,0 +1,26 @@
+require 'sourcers/git_sourcer'
+require 'issuesrc/config'
+
+module Issuesrc
+  module Sourcers
+    class GithubSourcer < GitSourcer
+      def initialize(args, config)
+        begin
+          user, repo = try_find_repo(args, config)
+          url = "git@github.com:#{user}/#{repo}.git"
+          GitSourcer.instance_method(:initialize_with_url).bind(self).call(
+            url, args, config)
+        rescue Exception => e
+          super args, config
+        end
+      end
+
+      private
+      def try_find_repo(args, config)
+        repo_arg = Issuesrc::Config::option_from_both(
+          :repo, ['github', 'repo'], args, config)
+        repo_arg.split('/')
+      end
+    end
+  end
+end

data/lib/tag_finders/blunt_tag_finder.rb

@@ -0,0 +1,186 @@
+require 'issuesrc/tag'
+
+module Issuesrc
+  module TagFinders
+    class BluntTagFinder
+      DEFAULT_COMMENT_MARKERS = [['//', "\n"], ['/*', '*/']]
+      DEFAULT_STRING_MARKERS = ['"', "'"]
+
+      COMMENTS_BY_LANG = {
+        'php' => [['//', "\n"], ['/*', '*/'], ['#', "\n"]],
+        'html' => [['<!--', '-->']],
+        'sql' => [['--', "\n"]],
+        'sh' => [['#', "\n"]],
+        'hs' => [['--', "\n"], ['{-', '-}']],
+        'py' => [['#', "\n"]],
+        'rb' => [['#', "\n"]],
+        'clj' => [[';', "\n"]],
+        'coffee' => [['#', "\n"]],
+      }
+
+      STRINGS_BY_LANG = {
+        'go' => ['"', "'", '`'],
+        'rs' => ['"'],
+        'hs' => ['"'],
+      }
+
+      def initialize(tag_extractor, args, config)
+        @tag_extractor = tag_extractor
+      end
+
+      def accepts?(file)
+        true
+      end
+
+      def find_tags(file)
+        find_comments(file) do |comment, nline, pos|
+          # A tag extractor extracts from a single line, whereas comments may
+          # span several lines.
+          nline_offset = 0
+          comment.split("\n").each do |line|
+            tag_data = @tag_extractor.extract(line)
+            if !tag_data.nil?
+              yield Issuesrc::Tag.new(
+                tag_data['type'],
+                tag_data['issue_id'],
+                tag_data['author'],
+                tag_data['title'],
+                file,
+                nline + nline_offset,
+                pos + tag_data['begin_pos'],
+                pos + tag_data['end_pos']
+              )
+            end
+            pos += line.length + 1
+            nline_offset += 1
+          end
+        end
+      end
+
+      private
+      def find_comments(file)
+        comment_markers, string_markers = decide_markers(file)
+        body = file.body.read.force_encoding('BINARY') # TODO(#26): Use less memory here.
+        comment_finder = CommentFinder.new(body, comment_markers, string_markers)
+        pos = 0
+
+        comment_finder.each do |comment, nline, pos|
+          yield comment, nline, pos
+        end
+      end
+
+      def decide_markers(file)
+        [
+          COMMENTS_BY_LANG.fetch(file.type, DEFAULT_COMMENT_MARKERS),
+          STRINGS_BY_LANG.fetch(file.type, DEFAULT_STRING_MARKERS),
+        ]
+      end
+
+      class CommentFinder
+        def initialize(body, comment_markers, string_markers)
+          @body = body
+          @comment_markers = comment_markers
+          @string_markers = string_markers
+
+          @pos = 0
+          @nline = 1
+        end
+
+        def each
+          state = :init
+          while !@body.empty?
+            case state
+            when :init
+              consumed, state = consume_init()
+              @pos += consumed
+            when :string
+              consumed, state = consume_string()
+              @pos += consumed
+            when :comment
+              nline = @nline
+              lex, consumed, state = read_comment()
+              yield [lex, nline, @pos]
+              @pos += consumed
+            end
+          end
+        end
+
+        private
+        def consume_init
+          consumed = 0
+          next_state = nil
+          while !@body.empty?
+            next_state = peek_next_state()
+            if next_state != :init
+              break
+            end
+            consumed += consume_body(1)
+          end
+          [consumed, next_state]
+        end
+
+        def consume_string
+          lex, consumed, state = read_delimited(@string_markers)
+          [consumed, state]
+        end
+
+        def read_comment
+          read_delimited(@comment_markers)
+        end
+
+        def read_delimited(markers)
+          consumed = state_boundary(markers, :begin)
+          lex = @body[0...consumed]
+          consume_body(consumed)
+
+          consumed_end = state_boundary(markers, :end)
+          while !@body.empty? && consumed_end.nil?
+            lex << @body[0]
+            consumed += consume_body(1)
+            consumed_end = state_boundary(markers, :end)
+          end
+
+          if !consumed_end.nil?
+            lex << @body[0...consumed_end]
+            consumed += consume_body(consumed_end)
+          end
+
+          [lex, consumed, peek_next_state()]
+        end
+
+        def peek_next_state()
+          if !state_boundary(@comment_markers, :begin).nil?
+            :comment
+          elsif !state_boundary(@string_markers, :begin).nil?
+            :string
+          else
+            :init
+          end
+        end
+
+        def state_boundary(markers, begin_or_end)
+          if @body.nil?
+            nil
+          end
+
+          markers.each do |marker|
+            if marker.instance_of? Array 
+              marker = marker[begin_or_end == :begin ? 0 : 1]
+            end
+            if @body.start_with? marker
+              return marker.length
+            end
+          end
+          nil
+        end
+
+        def consume_body(n)
+          l = @body.length
+          @nline += @body[0...n].count "\n"
+          @body = @body[n..-1] || ''
+          l - @body.length
+        end
+      end
+    end
+  end
+end