issuesrc 0.0.3

data/lib/issuers/github_issuer.rb

@@ -0,0 +1,247 @@
+require 'issuesrc/config'
+require 'em-http-request'
+require 'json'
+require 'set'
+
+module Issuesrc
+  module Issuers
+    DEFAULT_LABEL = 'issuesrc'
+
+    class Issues
+      def initialize(queue)
+        @queue = queue
+        @queue_done = false
+        @cache = []
+      end
+
+      def each
+        i = 0
+        while i < @cache.length
+          yield @cache[i]
+          i += 1
+        end
+
+        while !@queue_done
+          @queue.pop do |issue_page|
+            if issue_page == :end
+              @queue_done = true
+              next
+            end
+            issue_page.each do |issue|
+              yield issue unless issue.include? 'pull_request'
+            end
+          end
+        end
+      end
+
+    end
+
+    class GithubIssuer
+      def initialize(args, config, event_loop)
+        @user, @repo = find_repo(args, config)
+        @token = try_find_token(args, config)
+        @event_loop = event_loop
+
+        @issuesrc_label = try_find_issuesrc_label(args, config)
+      end
+
+      def async_load_issues()
+        queue = EM::Queue.new
+        async_load_issues_pages(queue, 1)
+        Issues.new(queue)
+      end
+
+      def async_create_issue(tag, &block)
+        save_tag(tag, :post, "/repos/#{@user}/#{@repo}/issues", &block)
+      end
+
+      def async_update_issue(issue_id, tag, &block)
+        save_tag(tag, :patch, "/repos/#{@user}/#{@repo}/issues/#{issue_id}",
+                 &block)
+      end
+
+      def async_close_issue(issue_id)
+        ghreq(:patch, "/repos/#{@user}/#{@repo}/issues/#{issue_id}", {
+          'state' => 'closed'
+        }) do |req|
+          yield if block_given?
+        end
+      end
+
+      def async_update_or_close_issues(prev_issues, tags_by_issue_id, &block)
+        updated = Set.new
+
+        prev_issues.each do |issue|
+          issue_id = issue['number'].to_s
+
+          if tags_by_issue_id.include? issue_id
+            updated.add issue_id
+            tag = tags_by_issue_id[issue_id]
+            make_sure_issue_exists_and_then do |exists|
+              if exists
+                async_update_issue(issue_id, tag) do |tag|
+                  if !block.nil?
+                    yield issue_id, tag, :updated
+                  end
+                end
+              end
+            end
+          else
+            async_close_issue(issue_id) do
+              yield issue_id, nil, :closed
+            end
+          end
+        end
+
+        tags_by_issue_id.each do |issue_id, tag|
+          if updated.include? issue_id
+            next
+          end
+          make_sure_issue_exists_and_then do |exists|
+            if exists
+              async_update_issue(issue_id, tag) do |tag|
+                if !block.nil?
+                  yield issue_id, tag, :updated
+                end
+              end
+            end
+          end
+        end
+      end
+
+      def make_sure_issue_exists_and_then
+        # TODO(#21)
+        yield true
+      end
+
+      private
+      def find_repo(args, config)
+        repo_arg = Issuesrc::Config::option_from_both(
+          :repo, ['github', 'repo'], args, config, :require => true)
+        repo_arg.split('/')
+      end
+
+      def try_find_token(args, config)
+        Issuesrc::Config::option_from_both(
+          :github_token, ['github', 'auth_token'], args, config)
+      end
+
+      def try_find_issuesrc_label(args, config)
+        label = Issuesrc::Config::option_from_both(
+          :issuesrc_label, ['issuer', 'issuesrc_label'], args, config)
+        if label.nil?
+          label = DEFAULT_LABEL
+        end
+        label
+      end
+
+      def async_load_issues_pages(queue, from_page)
+        concurrent_pages = 1  # TODO(#22): Configurable?
+
+        waiting_for = concurrent_pages
+        end_reached = false
+
+        (from_page...from_page + concurrent_pages).each do |page|
+          ghreq(
+            :get,
+            "repos/#{@user}/#{@repo}/issues?filter=all&page=#{page}" +
+            "&labels=#{DEFAULT_LABEL}"
+          ) do |req|
+            st = req.response_header.status
+            if st < 200 or st >= 300
+              end_reached = true
+              queue.push(:end)
+            end
+
+            if end_reached
+              next
+            end
+
+            page_issues = JSON.parse(req.response)
+            if page_issues.length == 0
+              end_reached = true
+              queue.push(:end)
+            else
+              queue.push(page_issues)
+            end
+
+            waiting_for -= 1
+            if waiting_for == 0 and !end_reached
+              async_load_issues_pages(queue, from_page + concurrent_pages)
+            end
+          end
+        end
+      end
+
+      def save_tag(tag, method, url, &block)
+        title = gen_issue_title(tag)
+        body = gen_issue_body(tag)
+        params = {
+          'title' => title,
+          'labels' => [@issuesrc_label, tag.label],
+          'body' => body,
+          'state' => 'open',
+        }
+        if !tag.author.nil?
+          params['assignee'] = tag.author
+        end
+        ghreq(method, url, params) do |req|
+          # TODO(#23): Error handling.
+          new_tag_data = JSON.parse(req.response)
+          tag.issue_id = new_tag_data['number'].to_s
+
+          if !block.nil?
+            block.call tag
+          end
+        end
+      end
+
+      def gen_issue_title(tag)
+        title = tag.title
+        if title.nil? || title.length == 0
+          title = "#{tag.label} at #{tag.file.path_in_repo}:#{tag.line}"
+        end
+        title
+      end
+
+      def gen_issue_body(tag)
+        body = ""
+        if tag.file.instance_of? Issuesrc::GitFile
+          body = "https://github.com/#{@user}/#{@repo}" +
+                 "/blob/#{tag.file.branch}/#{tag.file.path_in_repo}" +
+                 "\#L#{tag.line}"
+        else
+          body = tag.file.path
+        end
+      end
+
+      def ghreq(method, url, params=nil)
+        if url[0] == ?/
+          url = url[1..-1]
+        end
+
+        req_data = {
+          :head => {
+            'Accept' => 'application/vnd.github.v3+json'
+          }
+        }
+        if !@token.nil?
+          req_data[:head]['Authorization'] = "token #{@token}"
+        end
+        if !params.nil?
+          req_data[:head]['Content-Type'] = 'application/json; charset=utf-8'
+          req_data[:body] = JSON.generate(params)
+        end
+
+        @event_loop.async_http_request(
+          method,
+          "https://api.github.com/#{url}",
+          req_data
+        ) do |req|
+          yield req if block_given?
+        end
+      end
+    end
+  end
+end
+

data/lib/issuesrc/config.rb

@@ -0,0 +1,42 @@
+module Issuesrc
+  module Config
+    def self.report_missing_in_config(option)
+      Issuesrc::exec_fail "Missing config entry: #{option}"
+    end
+
+    def self.option_from_both(option_args, option_config, args, config,
+                              flags = {})
+      value = option_from_args(option_args, args)
+      if value.nil?
+        value = option_from_config(option_config, config)
+      end
+      option_from_check_require(
+        "#{option_args} or #{option_config.join('.')}", value, flags)
+    end
+
+    def self.option_from_args(option, args, flags = {})
+      value = args.fetch(option, nil)
+      option_from_check_require(option, value, flags)
+    end
+
+    def self.option_from_config(option, config, flags = {})
+      value = config
+      option.each do |part|
+        if !value.include?(part)
+          value = nil
+          break
+        end
+        value = value[part]
+      end
+      option_from_check_require(option.join('.'), value, flags)
+    end
+
+    private
+    def self.option_from_check_require(option, value, flags)
+      if value.nil? && flags.include?(:require)
+        report_missing_in_config option
+      end
+      value
+    end
+  end
+end

data/lib/issuesrc/event_loop.rb

@@ -0,0 +1,91 @@
+require 'em-http-request'
+require 'net/http'
+
+module Issuesrc
+  class EventLoop
+    def async_http_request(method, url, opts, &callback)
+      @http_channel.push([method, url, opts, callback])
+    end
+
+    def wait_for_pending()
+      if @waiting == 0 || !@thread.alive?
+        return
+      end
+      @wakeup_when_done << Thread.current
+      Thread.stop
+    end
+
+    def initialize()
+      @pending = []
+      @http_channel = EM::Channel.new
+      @waiting = 0
+      @wakeup_when_done = []
+
+      @thread = Thread.new do
+        begin
+          EM.run do
+            handle_http_channel()
+          end
+        rescue
+          # TODO(#24): Proper error handling here.
+          wakeup_waiters()
+        end
+      end
+    end
+
+    private
+    def done_request()
+      @waiting -= 1
+      if @waiting > 0
+        return
+      end
+      wakeup_waiters()
+    end
+
+    def wakeup_waiters()
+      @wakeup_when_done.each do |t|
+        t.wakeup
+      end
+    end
+
+    def handle_http_channel()
+      @http_channel.subscribe do |msg|
+        method, url, opts, callback = msg
+        req = EM::HttpRequest.new(url).send(method, opts)
+        # TODO(#25): Err handling.
+        req.callback do
+          callback.call req
+          done_request()
+        end
+        req.errback do
+          callback.call req
+          done_request()
+        end
+      end
+    end
+  end
+
+  class SequentialEventLoop < EventLoop
+    def async_http_request(method, url, opts, &callback)
+      @waiting += 1
+      if @busy
+        @pending << [method, url, opts, callback]
+        return
+      end
+
+      @busy = true
+      @http_channel.push([method, url, opts, lambda do |req|
+        if !callback.nil?
+          callback.call req
+        end
+        @busy = false
+        if @pending.length > 0
+          method, url, opts, callback = @pending[0]
+          @pending = @pending[1..-1]
+          @waiting -= 1
+          async_http_request(method, url, opts, &callback)
+        end
+      end])
+    end
+  end
+end

data/lib/issuesrc/file.rb

@@ -0,0 +1,40 @@
+module Issuesrc
+  class FSFile
+    attr_reader :type
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+      @type = File.extname(path).slice(1..-1)
+    end
+
+    def body
+      File.open(@path, 'r')
+    end
+
+    def replace_at(pos, old_content_length, new_content)
+      fbody = body.read
+      fbody = replace_in_string(fbody, pos, old_content_length, new_content)
+      f = File.open(@path, 'wb')
+      f.write(fbody)
+      f.close()
+    end
+
+    private
+    def replace_in_string(s, pos, deleted_length, new_content)
+      (s[0...pos] || '') + new_content + (s[pos + deleted_length..-1] || '')
+    end
+  end
+
+  class GitFile < FSFile
+    attr_reader :repo
+    attr_reader :path_in_repo
+    attr_reader :branch
+
+    def initialize(path, path_in_repo, branch)
+      super(path)
+      @path_in_repo = path_in_repo
+      @branch = branch
+    end
+  end
+end

data/lib/issuesrc/tag.rb

@@ -0,0 +1,61 @@
+module Issuesrc
+  class Tag
+    attr_reader :label
+    attr_accessor :issue_id
+    attr_reader :author
+    attr_reader :title
+    attr_reader :file
+    attr_reader :line
+    attr_reader :begin_pos
+    attr_reader :end_pos
+
+    def initialize(label, issue_id, author, title, file, line, 
+                   begin_pos, end_pos)
+      @label = label
+      @issue_id = issue_id.nil? || issue_id.empty? ? nil : issue_id
+      @author = author.nil? || author.empty? ? nil : author
+      @title = title
+      @file = file
+      @line = line
+      @begin_pos = begin_pos
+      @end_pos = end_pos
+    end
+
+    def to_s
+      ret = ""
+      ret << @label
+      if !@issue_id.nil? || !@author.nil?
+        ret << '('
+        if !@author.nil?
+          ret << @author
+        end
+        if !@issue_id.nil?
+          ret << '#' << @issue_id
+        end
+        ret << ')'
+      end
+      if !@title.nil?
+        ret << ': ' << @title.strip
+      end
+      ret
+    end
+
+    def write_in_file(offsets)
+      total_offset = 0
+      offsets.each do |pos, offset|
+        if pos <= @begin_pos
+          total_offset += offset
+        end
+      end
+      old_begin_pos = @begin_pos
+      @begin_pos += total_offset
+      @end_pos += total_offset
+      file.replace_at(@begin_pos, @end_pos-@begin_pos, to_s())
+      new_end_pos = @begin_pos + to_s.length
+      offset = new_end_pos - @end_pos
+      @end_pos = new_end_pos
+      offsets << [old_begin_pos, offset]
+      offsets
+    end
+  end
+end

data/lib/issuesrc/tag_extractor.rb

@@ -0,0 +1,58 @@
+require 'issuesrc/config'
+
+module Issuesrc
+  TAG_EXTRACTORS = [
+    {
+      'regexp' => /(?<type>TODO|FIXME|BUG)\s*(\(\s*(?<author>[^)#\s]+)?\s*(#\s*(?<issue_id>[^)\s]+))?\s*\))?\s*:?\s*(?<title>[^\s].*)?/,
+    }
+  ]
+
+  class TagExtractor
+    def initialize(args, config)
+      @extractors = Issuesrc::Config.option_from_config(
+        ['tags', 'extractors'], config)
+      if @extractors.nil?
+        @extractors = TAG_EXTRACTORS.clone
+      end
+
+      more = Issuesrc::Config.option_from_config(
+        ['tags', 'additional_extractors'], config)
+      if !more.nil?
+        @extractors.merge! more
+      end
+    end
+
+    def extract(source)
+      @extractors.each do |extr|
+        tag = try_extractor(extr, source)
+        if !tag.nil?
+          return tag
+        end
+      end
+      nil
+    end
+
+    private
+    def try_extractor(extractor, source)
+      tag = nil
+      if extractor.include? 'regexp'
+        tag = try_regexp_extractor(extractor, source)
+      end
+      tag
+    end
+
+    def try_regexp_extractor(extractor, source)
+      m = extractor['regexp'].match(source)
+      if m.nil?
+        return m
+      end
+      ret = {}
+      m.names.each do |name|
+        ret[name] = m[name]
+      end
+      ret['begin_pos'] = m.begin(0)
+      ret['end_pos'] = ret['begin_pos'] + m.to_s.length
+      ret
+    end
+  end
+end

data/lib/issuesrc/version.rb

@@ -0,0 +1,3 @@
+module Issuesrc
+  VERSION = "0.0.3"
+end