overview 0.0.3.61 → 0.0.4.12

data/lib/changelog.rb

@@ -0,0 +1,455 @@
+module Changelog
+
+  require 'httparty'
+  require 'json'
+  require 'octokit'
+  require 'formatador'
+  require 'pp'
+  require 'yaml'
+  require 'logging'
+  require_relative 'helpers/ci'
+  require_relative 'helpers/git'
+  require_relative 'helpers/github'
+  require 'api_cache'
+  require 'moneta'
+  require 'faraday-http-cache'
+=begin
+TODO:
+
+More than I can comprehend...
+Keep all incoming data in original format, so that it can be reproccessed. e.g. Make calls. Store Data. Retrieve data. Process data. Display data. Deploy.
+Mark sprint.ly items as deployed if complete. This will eventually be in a separate script.
+Sprint.ly status
+Compelete refactor into a modern ruby CLI app.
+More info output
+Color output for terminal
+HTML output with embedded links
+Logic to be improved for push and pull rquests
+Deployment logic to work differently for feature branches... I think
+=end
+
+  class Base
+    def initialize
+      @logger = Logging.logger[self]
+      @logger.add_appenders \
+        Logging.appenders.stdout,
+        Logging.appenders.file('overview.log')
+      @logger.level = :info
+
+      APICache.store = Moneta.new(:YAML, :file => "#{self.class.name}_cache")
+      APICache.logger.level = Logger::DEBUG
+
+      #Caching for octokit
+      #store = Moneta.new(:YAML, :file => 'changelog_octokit.cache')
+      stack = Faraday::RackBuilder.new do |builder|
+        builder.use Faraday::HttpCache
+        builder.use Octokit::Response::RaiseError
+        #builder.use :store => store
+        builder.adapter Faraday.default_adapter
+      end
+
+      Octokit.middleware = stack
+    end
+  end
+
+  class Release < Base
+    def initialize(name, description, sha, date, status=nil, items=nil)
+      super()
+      status ||= "NA"
+      @name = name.delete("\n")
+
+      @description = description
+      @sha = sha
+      @date = date
+      @status = status
+      @items = items
+    end
+
+    def commits
+    end
+
+    def items
+      @items
+    end
+
+    def inspect
+      str = "<#{@name} at #{@date.strftime("%x")} (#{@items.length} items)>"
+      @items.each { |i| str.concat("<#{i.inspect}>") }
+      return str
+    end
+
+    def name
+      return @name
+    end
+
+    def date
+      return @date
+    end
+
+    def status
+      return @status
+    end
+
+    def released?
+      return self.status == "RELEASED"
+    end
+
+    def display
+
+    end
+  end
+
+  class Item
+    def initialize(title, id, description, sha, date, source, type=nil, status=nil, author=nil, environments=nil)
+      type ||="NA"
+      status ||="UNKNOWN"
+      author ||="UNKNOWN"
+      environments ||= []
+      @title = title.delete("\n")
+      @description = description
+      @sha = sha
+      @date = date
+      @id = id
+      @source = source
+      @type = type
+      @status = status
+      @author = author
+      @environments = environments
+    end
+
+    def inspect
+      "<#{@source} #{@type.upcase} ##{@id} : #{self.status} : #{@title}>"
+    end
+
+    def title
+      return @title
+    end
+
+    def date
+      return @date
+    end
+
+    def type
+      return @type
+    end
+
+    def author
+      return @author
+    end
+
+    def environments
+      return @environments.any? ? @environments : nil
+    end
+
+    def sha
+      return @sha
+    end
+
+    def id
+      return @id
+    end
+
+    def status
+      return @status.nil? ? "NA" : @status.upcase
+    end
+
+    def source
+      return @source
+    end
+
+    def sprintly?
+      return self.source.upcase == "SPRINT.LY"
+    end
+
+    def complete?
+      return %w(COMPLETED ACCEPTED).include?(self.status)
+    end
+
+    def built?
+      return self.environments.join.scan(/#(\d+)/).any? unless self.environments.nil?
+    end
+
+    def released?
+      return self.environments.join.scan(/v(\d+)/).any? unless self.environments.nil?
+    end
+  end
+
+  class Log < Base
+    def githubReleaseStatus(release=nil)
+      case
+        when release.nil?
+          status = "NA"
+        when release.draft
+          status = "DRAFT"
+        when release.prerelease
+          status = "PRE-RELEASE"
+        else
+          status = "RELEASED"
+      end
+      return status
+    end
+
+
+    def displayEnvironments(environments)
+      case
+        when environments.nil?
+          environments = ""
+        when environments.compact.length > 0
+          environments = " [" + environments.compact.join(", ") + "]"
+        else
+          environments = ""
+      end
+      return environments
+    end
+
+    def deploy(releases)
+      buildItems = []
+      allReleasedItems = []
+      releases.each { |r|
+        releaseItems = []
+        r.items.each { |i|
+          buildItems << i.id unless (i.built? || !i.complete?) #potential write each build for incomplete tasks or stories...not sure
+          releaseItems << i.id unless (i.released? || !i.complete? || !r.released? || allReleasedItems.include?(i.id))
+        }
+        allReleasedItems.concat(releaseItems)
+        releaseEnvironment = "Release #{r.name}"
+        sprintlyDeploy(releaseItems.uniq, releaseEnvironment) unless (releaseEnvironment.nil? || !releaseItems.any?)
+      }
+      buildEnvironment = "Build ##{ENV['TRAVIS_BUILD_NUMBER']}"
+      sprintlyDeploy(buildItems.uniq, buildEnvironment) unless (buildEnvironment.nil? || !buildItems.any?)
+    end
+
+#refactor to keep source data separate, and update environments on sprint.ly ticket so future checks return the new environemnt
+#deployments should also be batched, with one api call for all items of the build or release
+#@iterations = 0
+    def sprintlyDeploy(ids, environment)
+      #@iterations += 1
+      data = {:environment => environment, :numbers => ids.join(",")}
+      url = "https://sprint.ly/api/products/#{@product_id}/deploys.json"
+      response = HTTParty.post(url, {:basic_auth => @auth, :body => data})
+      item = JSON.parse(response.body)
+      #pp item
+      #exit unless @iterations < 2
+      #curl -u jono@overllc.com:WXKp2h3F38VUm2CmzxpqBYMDZBFvw84f --data "environment=test&numbers=22,8" https://sprint.ly/api/products/20064/deploys.json
+    end
+
+    def displayReleases(releases)
+      f = Formatador.new
+      types = releases.flat_map { |r|
+        r.items.flat_map { |i|
+          i.type
+        }
+      }.uniq
+      environments = releases.flat_map { |r|
+        r.items.flat_map { |i|
+          i.environments.flatten unless i.environments.nil?
+        }
+      }.uniq.compact
+      #puts environments.inspect
+      releases.each { |r|
+        f.display_line("#{r.name} (#{r.status.upcase}, #{displayDate(r.date)})")
+        f.indent {
+
+          types.each { |t|
+            items = r.items.select { |i| i.type == t }
+            f.display_line("#{t.upcase}") if items.length > 0
+            f.indent {
+              items.each { |i|
+                # i.sha[0..6]
+                #TODO fix this
+                bullet = ENV['TRAVIS_COMMIT'] == i.sha[0..6] ? "+" : "•"
+                f.display_line("#{bullet} ##{i.id}: #{i.title} (#{i.status.upcase}; #{i.author}; #{displayDate(i.date)})#{displayEnvironments(i.environments)}")
+
+              }
+            }
+          }
+        }
+
+      }
+    end
+
+    def idsFromCommitMessage(message)
+      commands = %w(close closed closes finish finished finishes fix fixed fixes breaks unfixes reopen reopens re-open re-opens addresses re ref references refs start starts see).collect { |x| "\\b#{x}\\b" }.join("|")
+      prefixes = %w(task issue defect bug item ticket).collect { |x| "\\b#{x}:\\b" }.join("|") + "|#"
+      re = Regexp.new(/(?:#{commands})\s(?:#{prefixes})(\d+)/)
+      #working re = Regexp.new(/(?:#{commands})\s(?:#{prefixes})(\d+)/)
+      #working re = Regexp.new(/(?:\bcloses\b|\bfixes\b)..(\d+)/)
+      #puts re.source
+
+      sprintlyIds = message.scan(re).flatten.compact
+      crashlyticsIds = message.scan(/(c|C):(?<crashlytics>\d+)/).flatten.compact
+      return sprintlyIds, crashlyticsIds
+    end
+
+    def crashlyticsItem(title, id, sha, date, author)
+      #puts "    " + item["type"].capitalize + " " + item["number"].to_s + ": " + item["title"]
+      return Item.new(title, id, "NA", sha, date, "Crashlytics", "crash", nil, author)
+    end
+
+#TODO - consider returning sprint.ly story rather than task
+    #TODO - cache raw response, not item
+    def sprintlyItem(id, sha, date, author)
+      url = "https://sprint.ly/api/products/" + @product_id + "/items/" + id + ".json"
+      APICache.get(url, :timeout => 30, :fail => []) do
+          response = HTTParty.get(url, :basic_auth => @auth)
+          item = JSON.parse(response.body)
+          Item.new(item["title"], item["number"], item["description"], sha, date, "Sprint.ly", item["type"], item["status"], author, item["deployed_to"])
+        end
+    end
+
+    def commitItem(commit)
+      APICache.get(commit.sha, :fail => []) do
+          Item.new(commit.commit.message.lines.first, commit.sha[0..6], commit.commit.message, commit.sha, commit.commit.author.date, "Github", "commit", "NA", commit.commit.author.name)
+      end
+    end
+
+
+
+    def itemsFromCommit(commit)
+      #pp commit
+      items = []
+      sprintlyIds, crashlyticsIds = idsFromCommitMessage(commit.commit.message)
+      #puts sprintlyIds.class
+      #puts commit.commit.message
+      unless (sprintlyIds.nil? || !sprintlyIds.any?) then
+        sprintlyIds.each { |id| items << sprintlyItem(id, commit.sha, commit.commit.author.date, commit.commit.author.name) }
+      end
+      unless (crashlyticsIds.nil? || !crashlyticsIds.any?) then
+        crashlyticsIds.each { |id| items << crashlyticsItem(commit.commit.message.lines.first, id, commit.sha, commit.commit.author.date, commit.commit.author.name) }
+      end
+      items << commitItem(commit) unless (!items.nil? && items.any?)
+      return items
+    end
+
+
+    def displayDate(date)
+      return date.strftime("%-d-%b-%Y")
+    end
+
+
+#*****
+#deploy items parsed from travis commit sha (as I can't check what was previosuly undeployed)
+#awaiting improvements from sprint.ly - due 19 Feb
+#*****
+
+#TRAVIS_COMMIT_RANGE
+
+    def getReleasesFromServices
+
+
+      #user = @github.client.user
+      #user.login
+
+
+      result = []
+
+      all_releases = @github.releases
+      #p all_releases.inspect
+      r = all_releases.first
+      #p r
+      p r.tag_name
+      ref = @github.ref(r.tag_name)
+      sha = ref.object.sha
+      commit = @github.commit(sha).commit
+      since = commit.author.date
+
+      commits = @github.commits_since(since)
+      commits.pop #remove commit from last release - not sure why this in necessary and recon that it is not accurate either
+
+      items = []
+      commits.each { |c| items.concat(itemsFromCommit(c)) }
+
+      release = Release.new("Next Version", "NA", sha, since, "NA", items)
+
+      result << release
+
+      releases = all_releases.take(ENV['NO_OF_RELEASES'].to_i)
+
+      releases.each_with_index { |r, index|
+
+        next_r = all_releases[index+1] unless index == all_releases.size - 1 #note we use all_releases for this
+        ref = @github.ref(r.tag_name)
+        sha = ref.object.sha
+        commit = @github.commit(sha).commit
+
+        to = commit.author.date
+
+        if next_r
+          next_ref = @github.ref(next_r.tag_name)
+          next_sha = next_ref.object.sha
+          next_commit = @github.commit(next_sha).commit
+          from = next_commit.author.date
+          commits = @github.client.commits_between(ENV['TRAVIS_REPO_SLUG'], from, to, ENV['TRAVIS_BRANCH'])
+        else
+          commits = @github.client.commits_before(ENV['TRAVIS_REPO_SLUG'], to, ENV['TRAVIS_BRANCH'])
+        end
+
+        items = []
+        commits.each { |c| items.concat(itemsFromCommit(c)) }
+
+        release = Release.new(r.tag_name, r.name, sha, to, githubReleaseStatus(r), items)
+
+        result << release
+      }
+      return result
+    end
+
+    def getReleasesFromYAML
+      return YAML.load_file(Dir.pwd + ENV['YAML_BACKUP']) if File.file?(Dir.pwd + ENV['YAML_BACKUP'])
+    end
+
+    def saveReleasesToYAML(releases)
+      File.open(Dir.pwd + ENV['YAML_BACKUP'], 'w') { |file| file.write(releases.to_yaml) }
+    end
+
+
+    def initialize
+
+      super
+      @commit = CI.commit || Git.commit_sha
+      @repo = CI.repo || Git.repo
+      @build_no = CI.build_no
+      @branch = CI.branch || Git.branch
+
+      warning = 'Unable to determine repo branch' if @branch.nil?
+      warning = 'Unable to determine SHA1' if @commit.nil?
+      warning = 'Unable to determine repo' if @repo.nil?
+      warning = 'Unable to determine GITHUB_TOKEN' if ENV['GITHUB_TOKEN'].nil?
+      warning = 'Unable to determine TRAVIS_TOKEN' if ENV['TRAVIS_TOKEN'].nil?
+      warning = 'Unable to determine SPRINTLY_USER' if ENV['SPRINTLY_USER'].nil?
+      warning = 'Unable to determine SPRINTLY_API_KEY' if ENV['SPRINTLY_API_KEY'].nil?
+
+      @logger.error warning if warning
+
+      ENV['NO_OF_RELEASES'] = '2'
+      ENV['YAML_BACKUP'] = '/releases.yml'
+      ENV['TRAVIS_COMMIT_RANGE'] = nil if ENV['TRAVIS_COMMIT_RANGE'].nil?
+
+      @auth = {:username => ENV['SPRINTLY_USER'], :password => ENV['SPRINTLY_API_KEY']}
+
+      @repo = CI.repo || Git.repo
+      @branch = CI.branch || Git.branch
+      #TODO abstract Github via CI class
+      @github = Github.new(@repo, @branch)
+
+      @product_id = @github.sprintly_product_id
+      if !@product_id
+        @logger.error "Unable to retrieve Sprint.ly product_id for #{repo}"
+        exit 1
+      end
+
+
+      #@releases = getReleasesFromYAML
+      @releases = getReleasesFromServices if @releases.nil?
+      #saveReleasesToYAML @releases unless @releases.nil?
+    end
+
+    def display(audience='production')
+      displayReleases @releases unless @releases.nil?
+    end
+
+    def commit
+      @commit
+    end
+  end
+end

data/lib/{appversion → helpers}/ci.rb

@@ -1,4 +1,3 @@
-module Overview
 
   class CI
     DEFAULT_BUILD_NO = '1'
@@ -25,6 +24,9 @@ module Overview
     def self.version_suffix
       ENV['VERSION_SUFFIX'] || ''
     end
+
+    def self.commit
+      ENV['TRAVIS_COMMIT'] || ENV['CIRCLE_SHA1']
+    end
   end
 
-end

data/lib/helpers/git.rb

@@ -0,0 +1,67 @@
+require_relative 'string.rb'
+
+  class Git
+    def self.remote
+      begin
+        `git remote -v show`.lines.first.strip.match(/github\.com[\/|:](.+)\.git/)[1]
+      rescue
+        $stderr.puts 'Unable to retrieve slug from >> git remote -v show'
+        exit 1
+      end
+    end
+    def self.repo
+      remote
+    end
+    def self.branch
+      `git rev-parse --abbrev-ref HEAD`.strip
+    end
+    def self.tag
+      (`git describe --tags --match 'v*' --abbrev=0 2>/dev/null` || 'HEAD').strip
+    end
+    def self.clean_tag(tag=self.tag)
+      tag.strip.sub('v','').split(/[\.,-]/).select { |e| e.is_number?}.first(3).join('.')
+    end
+    def self.commit_count
+      `git rev-list --count HEAD`.to_i
+    end
+    def self.commit_count_since_tag(tag)
+      `git rev-list --count ${tag}.. 2>/dev/null`.to_i
+    end
+    def self.installed?
+      system 'git --version >>/dev/null 2>&1'
+    end
+    def self.parse_deploy(commitMessage)
+      commands = %w(deploy DEPLOY force_deploy FORCE_DEPLOY).collect{ |x| "\\b#{x}\\b" }.join("|")
+      re = Regexp.new(/<(#{commands}):?\s*(.*?)(?:>)/)
+      deploy = commitMessage.scan(re).flatten.compact
+      return !deploy[0].nil? && !deploy[0].empty?, deploy[1] || ''
+    end
+    def self.parse_crashlytics(commitMessage)
+      return commitMessage.scan(/(c|C):(?<crashlytics>\d+)/).flatten.collect { |x| x.to_i }.uniq
+    end
+    def self.parse_sprintly(commitMessage)
+      commands = %w(close closed closes finish finished finishes fix fixed fixes breaks unfixes reopen reopens re-open re-opens addresses re ref references refs start starts see).collect{ |x| "\\b#{x}\\b" }.join("|")
+      prefixes = %w(task issue defect bug item ticket).collect{ |x| "\\b#{x}:\\b" }.join("|") + "|#"
+      re = Regexp.new(/(?:#{commands})\s(?:#{prefixes})(\d+)/)
+      return commitMessage.downcase.scan(re).flatten.collect{ |x| x.to_i }.uniq
+    end
+    def self.parse_commit_message(commitMessage)
+      h = Hash.new
+      deploy, message = self.parse_deploy(commitMessage)
+      sprintly_tickets = self.parse_sprintly(commitMessage)
+      crashlytics_ids = self.parse_crashlytics(commitMessage)
+      h["deploy?"] = deploy
+      h["message"] = message
+      h["sprintly"] = sprintly_tickets
+      h["crashlytics"] = crashlytics_ids
+      return h
+    end
+    def self.commit_sha
+      `git rev-parse HEAD`.strip
+    end
+    def self.commit_short_sha
+      `git rev-parse --short HEAD`.strip
+    end
+  end
+
+

data/lib/helpers/github.rb

@@ -0,0 +1,81 @@
+require 'api_cache'
+require 'moneta'
+require 'octokit'
+#TODO - this is actually a github class
+class Github
+  #TODO make this a singleton
+  #TODO add repo, and branch to cache keys
+  def initialize(repo, branch, github_token=ENV['GITHUB_TOKEN'])
+    #TODO: inherit from base class
+    @github_token = github_token
+    @repo = repo
+    @branch = branch
+    @logger = Logging.logger[self]
+    @logger.add_appenders \
+        Logging.appenders.stdout,
+        Logging.appenders.file('overview.log')
+    @logger.level = :info
+
+    APICache.store = Moneta.new(:YAML, :file => "#{self.class.name}_cache")
+    APICache.logger.level = Logger::INFO
+    #APICache.store = nil
+  end
+
+  def client
+    if !@client
+      @client = Octokit::Client.new(:access_token => @github_token)
+      @client.auto_paginate = true
+      @client.default_media_type = "application/vnd.github.moondragon+json"
+      @logger.info "Connected to Github" if @client
+    end
+    @client
+  end
+
+  def hooks
+      @hooks = client.hooks(@repo) unless @hooks
+      @logger.info "Retrieved hooks for #{@repo}" if @hooks
+      @hooks
+  end
+
+  def releases
+    #APICache.get("releases") do
+      client.releases(@repo)
+    #end
+  end
+
+  def sprintly_product_id
+    #TODO this could also return a CLI flag or env variable
+    APICache.get("sprintly_product_id_#{@repo}", :fail => [], :timeout => 30) do
+      @logger.info "Not using cache"
+      product_id = hooks.map { |h| h.config.product_id }.compact.first
+      @logger.info "Sprint.ly product_id = #{product_id}" if product_id
+      product_id
+    end
+  end
+
+  def ref(tag)
+    #APICache.get("ref_#{tag}") do
+      client.ref(@repo, "tags/" + tag)
+    #end
+  end
+
+  def commit(sha)
+    #APICache.get("commit_#{sha}") do
+      client.commit(@repo, sha)
+    #end
+  end
+
+  def commits_since(since)
+    #APICache.get("commit_since_#{since}") do
+      client.commits_since(@repo, since, @branch)
+    #end
+  end
+
+  def commits_between(from,to)
+
+  end
+  def commits_before(to)
+
+  end
+end
+

data/lib/overview/version.rb

@@ -1,3 +1,3 @@
 module Overview
-  VERSION = '0.0.3.61'
+  VERSION = '0.0.4.12'
 end

data/lib/overview.rb

@@ -1,9 +1,9 @@
 require_relative 'overview/version.rb'
-require_relative 'appversion/appversion.rb'
-require_relative 'appversion/git.rb'
-require_relative 'appversion/ci.rb'
-require_relative 'appversion/string.rb'
-
+require_relative 'helpers/git.rb'
+require_relative 'helpers/ci.rb'
+require_relative 'helpers/string.rb'
+require_relative 'appversion.rb'
+require_relative 'changelog.rb'
 
 # Add requires for other files you add to your project here, so
 # you just need to require this one file in your bin file

data/overview.gemspec

@@ -1,5 +1,5 @@
 # Ensure we require the local version and not one we might have installed already
-require File.join([File.dirname(__FILE__),'lib','overview','version.rb'])
+require File.join([File.dirname(__FILE__),'lib/overview','version.rb'])
 
 spec = Gem::Specification.new do |s| 
   s.name = 'overview'
@@ -8,7 +8,7 @@ spec = Gem::Specification.new do |s|
   s.email = 'jono@overllc.com'
   s.homepage = 'http://www.overllc.com'
   s.platform = Gem::Platform::RUBY
-  s.summary = "Over's custom CI / CD toolchain, integrating Sprint.ly, Github, Travis and ITC"
+  s.summary = "Over's custom CI / CD toolchain, integrating Sprint.ly, Github, Github and ITC"
   s.files = `git ls-files`.split("
 ")
   s.require_paths << 'lib'
@@ -26,5 +26,10 @@ spec = Gem::Specification.new do |s|
   s.add_development_dependency('gem-release')
   s.add_runtime_dependency('gli','2.13.0')
   s.add_runtime_dependency('octokit','3.8.0')
-
+  s.add_runtime_dependency('httparty')
+  s.add_runtime_dependency('formatador')
+  s.add_runtime_dependency('moneta')
+  s.add_runtime_dependency('api_cache')
+  s.add_runtime_dependency('logging')
+  s.add_runtime_dependency('faraday-http-cache')
 end