fluent-plugin-github-activities 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f6761994ec06ed9516a6b7c9abe3ec44f77bfdbe
+  data.tar.gz: f8709e6fa06915f13ea6a3890382f84769c38465
+SHA512:
+  metadata.gz: 5e1251b591cfdc869b64f10f5f726cc5e637edb21be2c77350460d46ded5ef9c01e5a9d4160998aa983ab05eeac62ad82477d3b356c65fb828e83ee393fe41d7
+  data.tar.gz: cd5c50121621d5100c2b3f6aa0070fe306aaf4d72a1615b3d5da220b819549975b2d324a97850fcb654ce7aa04343a8f0712ead41dcd0b3570e138b4b3ddbec7

data/Gemfile

@@ -0,0 +1,22 @@
+# -*- mode: ruby; coding: utf-8 -*-
+#
+# This file is part of fluent-plugin-github-activities.
+#
+# fluent-plugin-github-activities is free software: you can
+# redistribute it and/or modify it under the terms of the GNU Lesser
+# General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# fluent-plugin-github-activities is distributed in the hope that
+# it will be useful, but WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with fluent-plugin-github-activities.  If not, see
+# <http://www.gnu.org/licenses/>.
+
+source "https://rubygems.org/"
+
+gemspec

data/README.md

@@ -0,0 +1,87 @@
+# fluent-plugin-github-activities
+
+Provides ability to watch public activities on GitHub.
+This crawls GitHub activities of specified users and forward each activity as a record.
+
+## Supported activity types
+
+ * Activities related to commits
+   * `push`
+   * `commit` (See also following notes)
+   * `commit-comment`
+ * Activities related to repositories
+   * `fork`
+   * `branch`
+   * `tag`
+ * Activities related to issues
+   * `issue-open`
+   * `issue-close`
+   * `issue-reopen`
+   * `issue-assign`
+   * `issue-unassign`
+   * `issue-label`
+   * `issue-unlabel`
+ * Activities related to pull requests
+   * `pull-request`
+   * `pull-request-merged`
+   * `pull-request-cancelled`
+   * `pull-request-reopen`
+   * `pull-request-comment`
+
+Forwarded message is same to an activity provided by GitHub.
+See also [the API documentations of GitHub activity events](https://developer.github.com/v3/activity/events/).
+
+Notes:
+
+ * Because a "push" activity doesn't include full information of each commit, commits are separately forwarded [commits](https://developer.github.com/v3/git/commits/) as pseudo `commit` activities.
+ * All forwarded records have an extra property `$github-activities-related-avatar`.
+   It will be useful to get the URI of the avatar image easily, for both activity events and commits.
+ * Unsupported activities are also forwarded with their raw event type like `StatusEvent`.
+
+
+## Configurations
+
+~~~
+<source>
+  type github-activities
+
+  # Authentication settings.
+  # They are optional but strongly recommended to be configured,
+  # because there is a rate limit: 60requests/hour by default.
+  # By an authenticated crawler, you can crawl 5000requests/hour
+  # (means about 80requests/minute).
+  basic_username your-user-name-of-github
+  basic_password your-password-of-github
+
+  # Interval seconds for requests. This is `1` by default.
+  interval 1
+
+  # Path to a file to store timestamp of last crawled activity
+  # for each user. If you don't specify this option, same records
+  # can be forwarded after the fluentd is restarted.
+  pos_file /tmp/github-activities.json
+
+  # Base tag of forwarded records. It will be used as
+  # <base_tag>.<activity type>, like: "github-activity.push",
+  # "github-activity.StatusEvent", etc.
+  base_tag github-activity.
+
+  # The lisf of target users' account IDs on the GitHub to be crawled.
+  users ashie,co-me,cosmo0920,hayamiz,hhatto,kenhys,kou
+  # External list is also available.
+  #users_list /path/to/list/of/users
+
+  # Merged pull requests will provide push and commit activities,
+  # so you possibly see same commits twice when a pull request by
+  # a known user (in the list above) is merged by another known user.
+  # To avoid such annoying duplicated records, they are ignored by
+  # default. If you hope those records are also forwarded, set this
+  # option `true` manually.
+  #include_commits_from_pull_request true
+
+  # Pull requests can include commits by unknown users (out of the
+  # list above) and the crawler ignores such users' commits by default.
+  # To include those commit activities, set this option `true` manually.
+  #include_foreign_commits true
+</source>
+~~~

data/fluent-plugin-github-activities.gemspec

@@ -0,0 +1,43 @@
+# -*- mode: ruby; coding: utf-8 -*-
+#
+# This file is part of fluent-plugin-github-activities.
+#
+# fluent-plugin-github-activities is free software: you can
+# redistribute it and/or modify it under the terms of the GNU Lesser
+# General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# fluent-plugin-github-activities is distributed in the hope that
+# it will be useful, but WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with fluent-plugin-github-activities.  If not, see
+# <http://www.gnu.org/licenses/>.
+
+Gem::Specification.new do |spec|
+  spec.name = "fluent-plugin-github-activities"
+  spec.version = "0.1.0"
+  spec.authors = ["YUKI Hiroshi"]
+  spec.email = ["yuki@clear-code.com"]
+  spec.summary = "Fluentd plugin to crawl public activities on the GitHub."
+  spec.description = "This provides ability you fluentd to crawl public " +
+                       "activities of users."
+  spec.homepage = "https://github.com/groonga/fluent-plugin-groonga"
+  spec.license = "LGPL-3.0"
+
+  spec.files = ["README.md", "Gemfile", "#{spec.name}.gemspec"]
+  spec.files += Dir.glob("lib/**/*.rb")
+  spec.test_files += Dir.glob("test/**/*")
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency("fluentd")
+
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("bundler")
+  spec.add_development_dependency("packnga", ">= 1.0.1")
+  spec.add_development_dependency("test-unit")
+  spec.add_development_dependency("test-unit-notify")
+end

data/lib/fluent/plugin/github-activities.rb

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of fluent-plugin-github-activities.
+#
+# fluent-plugin-github-activities is free software: you can
+# redistribute it and/or modify it under the terms of the GNU Lesser
+# General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# fluent-plugin-github-activities is distributed in the hope that
+# it will be useful, but WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with fluent-plugin-github-activities.  If not, see
+# <http://www.gnu.org/licenses/>.
+
+require "fluent/plugin/github-activities/crawler"
+
+module Fluent
+  module GithubActivities
+    TYPE_EVENTS = :events
+    TYPE_COMMIT = :commit
+  end
+end

data/lib/fluent/plugin/github-activities/crawler.rb

@@ -0,0 +1,368 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of fluent-plugin-github-activities.
+#
+# fluent-plugin-github-activities is free software: you can
+# redistribute it and/or modify it under the terms of the GNU Lesser
+# General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# fluent-plugin-github-activities is distributed in the hope that
+# it will be useful, but WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with fluent-plugin-github-activities.  If not, see
+# <http://www.gnu.org/licenses/>.
+
+require "uri"
+require "net/https"
+require "json"
+require "pathname"
+
+require "fluent/plugin/github-activities/safe_file_writer"
+
+module Fluent
+  module GithubActivities
+    class Crawler
+      class EmptyRequestQueue < StandardError
+      end
+
+      NO_INTERVAL = 0
+      DEFAULT_INTERVAL = 1
+
+      DEFAULT_LAST_EVENT_TIMESTAMP = -1
+
+      RELATED_USER_IMAGE_KEY = "$github-activities-related-avatar"
+
+      attr_writer :on_emit
+      attr_reader :request_queue, :interval_for_next_request
+
+      def initialize(options={})
+        @username = options[:username]
+        @password = options[:password]
+
+        @watching_users = options[:watching_users] || []
+
+        @include_commits_from_pull_request = options[:include_commits_from_pull_request]
+        @include_foreign_commits = options[:include_foreign_commits]
+
+        @positions = {}
+        @pos_file = options[:pos_file]
+        @pos_file = Pathname(@pos_file) if @pos_file
+        load_positions
+
+        @avatars = {}
+
+        @request_queue = options[:request_queue] || []
+
+        @default_interval = options[:default_interval] || DEFAULT_INTERVAL
+
+        @watching_users.each do |user|
+          fetch_avatar(user)
+          reserve_user_events(user)
+        end
+      end
+
+      def process_request
+        raise EmptyRequestQueue.new if @request_queue.empty?
+
+        request = @request_queue.shift
+        $log.info("GithubActivities::Crawler: processing request: #{request.inspect}")
+        if request[:process_after] and
+             Time.now.to_i < request[:process_after]
+          @request_queue.push(request)
+          @interval_for_next_request = NO_INTERVAL
+          return false
+        end
+
+        uri = request_uri(request)
+        extra_headers = extra_request_headers(request)
+
+        $log.info("GithubActivities::Crawler: requesting to #{uri.inspect}")
+        response = http_get(uri, extra_headers)
+        $log.info("GithubActivities::Crawler: response: #{response.inspect}")
+
+        case response
+        when Net::HTTPSuccess
+          body = JSON.parse(response.body)
+          $log.info("GithubActivities::Crawler: request type: #{request[:type]}")
+          case request[:type]
+          when TYPE_EVENTS
+            events = body
+            $log.info("GithubActivities::Crawler: events size: #{events.size}")
+            process_user_events(request[:user], events)
+            reserve_user_events(request[:user], :previous_response => response)
+            save_user_position(request[:user], :entity_tag => response["ETag"])
+          when TYPE_COMMIT
+            process_commit(body, request[:push])
+          end
+        when Net::HTTPNotModified
+          case request[:type]
+          when TYPE_EVENTS
+            reserve_user_events(request[:user],
+                                :previous_response => response,
+                                :previous_entity_tag => extra_headers["If-None-Match"])
+          end
+          @interval_for_next_request = NO_INTERVAL
+          return true
+        end
+        @interval_for_next_request = @default_interval
+        return true
+      end
+
+      def request_uri(request)
+        uri = nil
+        case request[:type]
+        when TYPE_EVENTS
+          uri = user_activities(request[:user])
+        else
+          uri = request[:uri]
+        end
+      end
+
+      def extra_request_headers(request)
+        headers = {}
+        if request[:previous_entity_tag]
+          headers["If-None-Match"] = request[:previous_entity_tag]
+        elsif request[:type] == TYPE_EVENTS and @positions[request[:user]]
+          entity_tag = @positions[request[:user]]["entity_tag"]
+          headers["If-None-Match"] = entity_tag if entity_tag
+        end
+        headers
+      end
+
+      def reserve_user_events(user, options={})
+        request = {
+          :type => TYPE_EVENTS,
+          :user => user,
+        }
+        response = options[:previous_response]
+        if response
+          now = options[:now] || Time.now
+          interval = response["X-Poll-Interval"].to_i
+          time_to_process = now.to_i + interval
+          request[:previous_entity_tag] = response["ETag"] ||
+                                            options[:previous_entity_tag]
+          request[:process_after] = time_to_process
+        end
+        @request_queue.push(request)
+      end
+
+      def process_user_events(user, events)
+        last_event_timestamp = DEFAULT_LAST_EVENT_TIMESTAMP
+        if @positions[user] and @positions[user]["last_event_timestamp"]
+          last_event_timestamp = @positions[user]["last_event_timestamp"]
+        end
+        events.each do |event|
+          timestamp = Time.parse(event["created_at"]).to_i
+          next if timestamp <= last_event_timestamp
+          process_user_event(user, event)
+          save_user_position(user, :last_event_timestamp => timestamp)
+        end
+      end
+
+      def process_user_event(user, event)
+        # see also: https://developer.github.com/v3/activity/events/types/
+        event[RELATED_USER_IMAGE_KEY] = @avatars[user]
+        case event["type"]
+        when "PushEvent"
+          process_push_event(event)
+        when "CommitCommentEvent"
+          emit("commit-comment", event)
+        when "IssuesEvent"
+          process_issue_event(event)
+        when "IssueCommentEvent"
+          process_issue_or_pull_request_comment_event(event)
+        when "ForkEvent"
+          emit("fork", event)
+        when "PullRequestEvent"
+          process_pull_request_event(event)
+        when "CreateEvent"
+          process_create_event(event)
+        else
+          emit(event["type"], event)
+        end
+      end
+
+      def process_push_event(event)
+        payload = event["payload"]
+        commit_refs = payload["commits"]
+        if !@include_commits_from_pull_request and
+             push_event_from_merged_pull_request?(event)
+          return
+        end
+        commit_refs.reverse.each do |commit_ref|
+          @request_queue.unshift(:type => TYPE_COMMIT,
+                                 :uri  => commit_ref["url"],
+                                 :push => event)
+        end
+        # emit("push", event)
+      end
+
+      def process_commit(commit, push_event)
+        user = commit["author"]["login"]
+        fetch_avatar(user)
+
+        if @include_foreign_commits or watching_user?(user)
+          commit[RELATED_USER_IMAGE_KEY] = @avatars[user]
+          emit("commit", commit)
+        end
+
+        commit_refs = push_event["payload"]["commits"]
+        target_commit_ref = commit_refs.find do |commit_ref|
+          commit_ref["url"] == commit["url"]
+        end
+        target_commit_ref["commit"] = commit if target_commit_ref
+
+        completely_fetched = commit_refs.all? do |commit_ref|
+          commit_ref["commit"]
+        end
+        emit("push", push_event) if completely_fetched
+      end
+
+      def watching_user?(user)
+        @watching_users.include(user)
+      end
+
+      def process_issue_event(event)
+        payload = event["payload"]
+        case payload["action"]
+        when "opened"
+          emit("issue-open", event)
+        when "closed"
+          emit("issue-close", event)
+        when "reopened"
+          emit("issue-reopen", event)
+        when "assigned"
+          emit("issue-assign", event)
+        when "unassigned"
+          emit("issue-unassign", event)
+        when "labeled"
+          emit("issue-label", event)
+        when "unlabeled"
+          emit("issue-unlabel", event)
+        end
+      end
+
+      def process_pull_request_event(event)
+        payload = event["payload"]
+        case payload["action"]
+        when "opened"
+          emit("pull-request", event)
+        when "closed"
+          if payload["pull_request"]["merged"]
+            emit("pull-request-merged", event)
+          else
+            emit("pull-request-cancelled", event)
+          end
+        when "reopened"
+          emit("pull-request-reopen", event)
+        end
+      end
+
+      MERGE_COMMIT_MESSAGE_PATTERN = /\AMerge pull request #\d+ from [^\/]+\/[^\/]+\n\n/
+
+      def push_event_from_merged_pull_request?(event)
+        payload = event["payload"]
+        inserted_requests = []
+        commit_refs = payload["commits"]
+        if MERGE_COMMIT_MESSAGE_PATTERN =~ commit_refs.last["message"]
+          true
+        else
+          false
+        end
+      end
+
+      def process_issue_or_pull_request_comment_event(event)
+        payload = event["payload"]
+        if payload["issue"]["pull_request"]
+          emit("pull-request-comment", event)
+          # emit("pull-request.cancel", event)
+        else
+          emit("issue-comment", event)
+        end
+      end
+
+      def process_create_event(event)
+        payload = event["payload"]
+        case payload["ref_type"]
+        when "branch"
+          emit("branch", event)
+        when "tag"
+          emit("tag", event)
+        end
+      end
+
+      def fetch_avatar(user)
+        return if @avatars.key?(user)
+        response = http_get(user_info(user))
+        fetched_user_info = JSON.parse(response.body)
+        @avatars[user] = fetched_user_info["avatar_url"]
+      end
+
+      private
+      def user_activities(user)
+        "https://api.github.com/users/#{user}/events/public"
+      end
+
+      def user_info(user)
+        "https://api.github.com/users/#{user}"
+      end
+
+      def emit(tag, record)
+        $log.trace("GithubActivities::Crawler: emit => #{tag}, #{record.inspect}")
+        @on_emit.call(tag, record) if @on_emit
+      end
+
+      def http_get(uri, extra_headers={})
+        parsed_uri = URI(uri)
+        response = nil
+        http = Net::HTTP.new(parsed_uri.host, parsed_uri.port)
+        http.use_ssl = parsed_uri.is_a?(URI::HTTPS)
+        http.start do |http|
+          http_request = Net::HTTP::Get.new(parsed_uri.path, extra_headers)
+          if @username and @password
+            http_request.basic_auth(@username, @password)
+          end
+          response = http.request(http_request)
+        end
+        response
+      end
+
+      def load_positions
+        return unless @pos_file
+        return unless @pos_file.exist?
+
+        @positions = JSON.parse(@pos_file.read)
+      rescue
+        @positions = {}
+      end
+
+      def save_positions
+        return unless @pos_file
+        SafeFileWriter.write(@pos_file, JSON.pretty_generate(@positions))
+      end
+
+      def save_user_position(user, params)
+        @positions[user] ||= {}
+
+        if params[:entity_tag]
+          @positions[user]["entity_tag"] = params[:entity_tag]
+        end
+
+        if params[:last_event_timestamp] and
+             params[:last_event_timestamp] != DEFAULT_LAST_EVENT_TIMESTAMP
+          old_timestamp = @positions[user]["last_event_timestamp"]
+          if old_timestamp.nil? or old_timestamp < params[:last_event_timestamp]
+            @positions[user]["last_event_timestamp"] = params[:last_event_timestamp]
+          end
+        end
+
+        save_positions
+      end
+    end
+  end
+end