jira_cache 0.2.1

data/jira_cache.gemspec

@@ -0,0 +1,41 @@
+# coding: utf-8
+# frozen_string_literal: true
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "jira_cache/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "jira_cache"
+  spec.version       = JiraCache::VERSION
+  spec.authors       = ["Romain Champourlier"]
+  spec.email         = ["pro@rchampourlier.com"]
+  spec.summary       = "Fetches data from JIRA and caches it in a local database."
+  spec.homepage      = "https://github.com/rchampourlier/jira_cache"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org by setting "allowed_push_host", or
+  # delete this section to allow pushing this gem to any host.
+  unless spec.respond_to?(:metadata)
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = []
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "i18n"
+  spec.add_dependency "activesupport-inflector"
+  spec.add_dependency "pg"
+  spec.add_dependency "sequel"
+  spec.add_dependency "sequel_pg"
+  spec.add_dependency "rest-client"
+  spec.add_dependency "sinatra"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "awesome_print"
+  spec.add_development_dependency "dotenv"
+end

data/lib/jira_cache.rb

@@ -0,0 +1,53 @@
+require "jira_cache/version"
+require "jira_cache/sync"
+require "jira_cache/webhook_app"
+
+# JiraCache enables storing JIRA issues fetched from the API
+# in a local storage for easier and faster processing.
+#
+# This is the main module and it provides some high level
+# methods to either trigger a full project sync, a single
+# issue sync or start a Sinatra webhook app to trigger sync
+# on JIRA"s webhooks.
+module JiraCache
+
+  # Sync issues using the specified client. If a `project_key` is
+  # specified, only syncs the issues for the corresponding project.
+  def self.sync_issues(client: default_client, project_key: nil)
+    Sync.new(client).sync_issues(project_key: project_key)
+  end
+
+  def self.sync_issue(issue_key, client: default_client)
+    Sync.new(client).sync_issue(issue_key)
+  end
+
+  # @param client [JiraCache::Client]: defaults to a default
+  #   client using environment variables for domain, username
+  #   and password, a logger writing to STDOUT and a default
+  #   `JiraCache::Notifier` instance as notifier.
+  def self.webhook_app(client: default_client)
+    Sinatra.new(JiraCache::WebhookApp) do
+      set(:client, client)
+    end
+  end
+
+  def self.default_client
+    JiraCache::Client.new(
+      domain: ENV["JIRA_DOMAIN"],
+      username: ENV["JIRA_USERNAME"],
+      password: ENV["JIRA_PASSWORD"],
+      logger: default_logger,
+      notifier: default_notifier
+    )
+  end
+
+  def self.default_logger
+    logger = Logger.new(STDOUT)
+    logger.level = Logger::DEBUG
+    logger
+  end
+
+  def self.default_notifier(logger: default_logger)
+    JiraCache::Notifier.new(logger)
+  end
+end

data/lib/jira_cache/client.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+require "rest-client"
+require "base64"
+require "jira_cache/notifier"
+
+module JiraCache
+
+  # The JIRA API Client.
+  class Client
+    JIRA_MAX_RESULTS = 1000
+
+    EXPANDED_FIELDS = %w(
+      renderedFields
+      changelog
+    ).freeze
+    # Other possible fields: names, schema, operations, editmeta
+
+    attr_reader :logger, :notifier
+
+    # Returns a new instance of the client, configured with
+    # the specified parameters.
+    #
+    # @param domain [String] JIRA API domain (e.g. your-project.atlassian.net)
+    # @param username [String] JIRA user"s name, if required
+    # @param password [String] JIRA user"s password, if required
+    # @param logger [Logger] used to log message (defaults to a logger to STDOUT at
+    #   info level)
+    # @param notifier [Notifier] a notifier instance that will be used to publish
+    #   event notifications (see `JiraCache::Notifier` for more information)
+    #
+    def initialize(domain: ENV["JIRA_DOMAIN"],
+                   username: ENV["JIRA_USERNAME"],
+                   password: ENV["JIRA_PASSWORD"],
+                   notifier: default_notifier,
+                   logger: default_logger)
+      check_domain!(domain)
+      check_password!(username, password)
+      @domain = domain
+      @username = username
+      @password = password
+      @notifier = notifier
+      @logger = logger
+    end
+
+    # Fetches the issue represented by id_or_key from the
+    # client.
+    # If the data is already present in the cache,
+    # returns the cached version, unless if :allow_cache
+    # option is false.
+    def issue_data(id_or_key)
+      logger.info "Fetching data for issue #{id_or_key}"
+      issue_data = do_get("/issue/#{id_or_key}",
+        expand: EXPANDED_FIELDS.join(",")
+      ).to_hash
+      return nil if issue_not_found?(issue_data)
+      issue_data = complete_worklogs(id_or_key, issue_data)
+      begin
+        notifier.publish "fetched_issue", key: id_or_key, data: issue_data
+      rescue => e
+        logger.error "Notifier failed: #{e}"
+        logger.error e.backtrace
+      end
+      issue_data
+    end
+
+    def issue_keys_for_query(jql_query)
+      start_at = 0
+      issues = []
+      loop do
+        total, page_issues = issue_ids_in_limits(jql_query, start_at)
+        logger.info "Total number of issues: #{total}" if issues.length == 0
+        issues += page_issues
+        logger.info "  -- loaded #{page_issues.length} issues"
+        start_at = issues.length
+        break if issues.length == total
+      end
+      issues.collect { |issue| issue["key"] }
+    end
+
+    # Implementation methods
+    # ======================
+
+    # @return [total, issues]
+    #   - total: [Int] the total number of issues in the query results
+    #   - issues: [Array] array of issues in the response
+    #     (max `JIRA_MAX_RESULTS`)
+    def issue_ids_in_limits(jql_query, start_at)
+      results = do_get "/search",
+        jql: jql_query,
+        startAt: start_at,
+        fields: "id",
+        maxResults: JIRA_MAX_RESULTS
+      [results["total"], results["issues"]]
+    end
+
+    def issue_not_found?(issue_data)
+      return false if issue_data["errorMessages"].nil?
+      issue_data["errorMessages"].first == "Issue Does Not Exist"
+    end
+
+    def complete_worklogs(id_or_key, issue_data)
+      if incomplete_worklogs?(issue_data)
+        issue_data["fields"]["worklog"] = issue_worklog_content(id_or_key)
+      end
+      issue_data
+    end
+
+    def incomplete_worklogs?(issue_data)
+      worklog = issue_data["fields"]["worklog"]
+      worklog["total"].to_i > worklog["maxResults"].to_i
+    end
+
+    def issue_worklog_content(id_or_key)
+      do_get("/issue/#{id_or_key}/worklog").to_hash
+    end
+
+    def project_data(id)
+      do_get "/project/#{id}"
+    end
+
+    def projects_data
+      do_get "/project"
+    end
+
+    def do_get(path, params = {})
+      logger.debug "GET #{uri(path)} #{params}"
+      response = RestClient.get uri(path),
+        params: params,
+        content_type: "application/json"
+      begin
+        JSON.parse(response.body)
+      rescue JSON::ParseError
+        response.body
+      end
+    end
+
+    # Returns the JIRA API"s base URI (build using `config[:domain]`)
+    def uri(path)
+      "https://#{authorization_prefix}#{@domain}/rest/api/2#{path}"
+    end
+
+    def authorization_prefix
+      return "" if missing_credential?
+      "#{CGI.escape(@username)}:#{CGI.escape(@password)}@"
+    end
+
+    def default_logger
+      return @logger unless @logger.nil?
+      @logger = ::Logger.new(STDOUT)
+      @logger.level = ::Logger::FATAL
+      @logger
+    end
+
+    def default_notifier
+      return @notifier unless @notifier.nil?
+      @notifier = JiraCache::Notifier.new(@logger)
+    end
+
+    # Returns an hash of info on the client
+    def info
+      {
+        domain: @domain,
+        username: @username
+      }
+    end
+
+    private
+
+    def check_domain!(domain)
+      raise "Missing domain" if domain.nil? || domain.empty?
+    end
+
+    def check_password!(username, password)
+      unless (username.nil? || username.empty?)
+        raise "Missing password (mandatory if username given)" if password.nil? || password.empty?
+      end
+    end
+
+    def missing_credential?
+      return true if @username.nil? || @username.empty?
+      return true if @password.nil? || @password.empty?
+      false
+    end
+  end
+end

data/lib/jira_cache/data.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+require "sequel"
+
+module JiraCache
+  module Data
+    DATABASE_URL = ENV["DATABASE_URL"]
+    DB = Sequel.connect(DATABASE_URL)
+    DB.extension :pg_array, :pg_json
+  end
+end

data/lib/jira_cache/data/issue_repository.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+require "jira_cache/data"
+require "active_support/inflector"
+
+module JiraCache
+  module Data
+
+    # Superclass for repositories. Simply provide some shared
+    # methods.
+    class IssueRepository
+
+      # It inserts a new issue row with the specified data.
+      # If the issue already exists (checking on the "key"),
+      # the row is updated instead.
+      def self.insert(key:, data:, synced_at:, deleted_from_jira_at: nil)
+        attributes = {
+          key: key,
+          data: Sequel.pg_json(data),
+          synced_at: synced_at,
+          deleted_from_jira_at: deleted_from_jira_at
+        }
+        if exist_with_key?(key)
+          update_where({ key: key }, attributes)
+        else
+          table.insert row(attributes)
+        end
+      end
+
+      def self.find_by_key(key)
+        table.where(key: key).first
+      end
+
+      def self.exist_with_key?(key)
+        table.where(key: key).count != 0
+      end
+
+      def self.keys_in_project(project_key)
+        table.where("(data #>> '{fields,project,key}') = ?", project_key).select(:key).map(&:values).flatten
+      end
+
+      def self.keys_for_non_deleted_issues
+        table
+          .where("deleted_from_jira_at IS NULL")
+          .select(:key)
+          .map(&:values)
+          .flatten
+      end
+
+      def self.keys_for_deleted_issues
+        table
+          .where("deleted_from_jira_at IS NOT NULL")
+          .select(:key)
+          .map(&:values)
+          .flatten
+      end
+
+      def self.delete_where(where_data)
+        table.where(where_data).delete
+      end
+
+      def self.update_where(where_data, values)
+        table.where(where_data).update(values)
+      end
+
+      def self.first_where(where_data)
+        table.where(where_data).first
+      end
+
+      def self.index
+        table.entries
+      end
+
+      def self.count
+        table.count
+      end
+
+      def self.latest_sync_time
+        table.order(:synced_at).select(:synced_at).last&.dig(:synced_at)
+      end
+
+      def self.row(attributes, time = nil)
+        time ||= Time.now
+        attributes.merge(
+          created_at: time,
+          updated_at: time
+        )
+      end
+
+      def self.table
+        DB[:jira_cache_issues]
+      end
+    end
+  end
+end

data/lib/jira_cache/notifier.rb

@@ -0,0 +1,30 @@
+module JiraCache
+
+  # This notifiers simply logs messages using the specified
+  # logger.
+  #
+  # If you want to use this mechanism to trigger actions when
+  # events are triggered in JiraCache, you can use the
+  # `JiraCache::Client.set_notifier(notifier)` method and pass
+  # it an instance of a notifier class implementing the
+  # `#publish` method with the same signature as
+  # `JiraCache::Notifier#publish`.
+  class Notifier
+
+    # Initializes a notifier with the specified logger. The
+    # logger is used to log info messages when #publish
+    # is called.
+    def initialize(logger)
+      @logger = logger
+    end
+
+    # Simply logs the event name and data.
+    # @param event_name [String] e.g. "fetched_issue"
+    # @param data [Hash]
+    #   - :key [String] issue key
+    #   - :data [Hash] issue data
+    def publish(event_name, data = nil)
+      @logger.info "[#{event_name}] #{data[:key]}"
+    end
+  end
+end

data/lib/jira_cache/sync.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+require "jira_cache/data/issue_repository"
+require "jira_cache/client"
+
+module JiraCache
+
+  # Performs the sync between JIRA and the local database
+  # where the issues are cached.
+  #
+  # The issues are cached in the database through the
+  # Data::IssueRepository interface. It currently implements
+  # storage into a PostgreSQL database.
+  class Sync
+    attr_reader :client, :logger
+
+    def initialize(client)
+      @client = client
+      @logger = client.logger
+    end
+
+    # Fetches new and updated raw issues, save them
+    # to the `issues` collection. Also mark issues
+    # deleted from JIRA as such.
+    #
+    # @param project_key [String] the JIRA project key
+    def sync_issues(project_key: nil)
+      sync_start = Time.now
+
+      log "Determining which issues to fetch..."
+      remote = remote_keys(project_key: project_key)
+      log "  - #{remote.count} remote issues"
+
+      cached = cached_keys(project_key: project_key)
+      log "  - #{cached.count} cached issues"
+
+      missing = remote - cached
+      log "  => #{missing.count} missing issues"
+
+      updated = updated_keys(project_key: project_key)
+      log "  - #{updated.count} updated issues"
+
+      log "Fetching #{missing.count + updated.count} issues"
+      fetch_issues(missing + updated, sync_start)
+
+      deleted = cached - remote
+      mark_deleted(deleted)
+    end
+
+    def sync_issue(key, sync_time: Time.now)
+      data = client.issue_data(key)
+      Data::IssueRepository.insert(
+        key: key,
+        data: data,
+        synced_at: sync_time
+      )
+    end
+
+    # IMPLEMENTATION FUNCTIONS
+
+    def remote_keys(project_key: nil)
+      fetch_issue_keys(project_key: project_key)
+    end
+
+    def cached_keys(project_key: nil)
+      Data::IssueRepository.keys_in_project(project_key: project_key)
+    end
+
+    def updated_keys(project_key: nil)
+      time = latest_sync_time(project_key: project_key)
+      fetch_issue_keys(project_key: project_key, updated_since: time)
+    end
+
+    # Fetch from JIRA
+
+    # Fetch issue keys from JIRA using the specified `JiraCache::Client`
+    # instance, for the specified project, with an optional `updated_since`
+    # parameter.
+    #
+    # @param project_key [String]
+    # @param updated_since [Time]
+    # @return [Array] array of issue keys as strings
+    def fetch_issue_keys(project_key: nil, updated_since: nil)
+      query_items = []
+      query_items << "project = \"#{project_key}\"" unless project_key.nil?
+      query_items << "updatedDate > \"#{updated_since.strftime('%Y-%m-%d %H:%M')}\"" unless updated_since.nil?
+      query = query_items.join(" AND ")
+      client.issue_keys_for_query(query)
+    end
+
+    # @param issue_keys [Array] array of strings representing the JIRA keys
+    def fetch_issues(issue_keys, sync_time)
+      issue_keys.each do |issue_key|
+        sync_issue(issue_key, sync_time: sync_time)
+      end
+    end
+
+    def mark_deleted(issue_keys)
+      Data::IssueRepository.update_where({ key: issue_keys }, deleted_from_jira_at: Time.now)
+    end
+
+    def latest_sync_time(project_key)
+      Data::IssueRepository.latest_sync_time
+    end
+
+    def log(message)
+      return if logger.nil?
+      logger.info(message)
+    end
+  end
+end