goapi 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: da6488566478de3c1f4bd2a82aaadb9a7d1c4aba
+  data.tar.gz: f50eed750d0ad1058e84fdb4e0a66128a66d81c6
+SHA512:
+  metadata.gz: 6e1e41bcbb6531fdcf2870c45615e8cddecec35b0c86aef79848ff4ccff0e2748ea75ace5ece5e34885aee681f3f260add2e63483ba61c63af474b7be05bde37
+  data.tar.gz: 897c445726b3234d8877510a7ff61bef37067b7238eccf86557363426630f15ac11be112820883b7de2e5a978e8b4898d2ff19fdd3ee87062de53a33298bc9d8

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in goapi.gemspec
+gemspec

data/README.md

@@ -0,0 +1,67 @@
+# Go API
+
+[Go](http://www.go.cd) API Ruby client.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'goapi'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install goapi
+
+## Usage
+
+### Initialize with Go server url and Basic Auth credentials
+
+    require 'goapi'
+
+    go = GoAPI.new('server-url', basic_auth: [username, password])
+
+### Fetch pipeline stages history
+
+    require 'goapi'
+    go = GoAPI.new('server-url', basic_auth: [username, password])
+    go.stages(pipeline_name, stage_name)
+
+### Fetch artifacts
+
+    require 'goapi'
+    go = GoAPI.new('server-url', basic_auth: [username, password])
+    stages = go.stages(pipeline_name, stage_name)
+    artifacts = go.artifacts(stages[0], job_name)
+    artifacts.map do |artifact|
+      if artifact.type == 'file'
+        go.artifact(artifact)
+      else # when type == 'folder'
+        ....
+      end
+    end
+
+More complex examples please checkout [examples](examples) directory in codebase
+
+## API design
+
+1. flat: one level API, you can find all APIs definition in class GoAPI.
+2. data: all APIs return data object only, they are:
+   1. Primitive type in Ruby
+   2. OpenStruct object, with primitive type values (rule #1 flat)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake false` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ThoughtWorksStudios/goapi.
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "goapi"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/agent_stats.rb

@@ -0,0 +1,31 @@
+
+server_url = <go server url>
+username = <username>
+password = <your password>
+pipeline_name = <pipeline name>
+stage_name = <stage name>
+
+lib_path = File.expand_path('../../lib', __FILE__)
+$:.unshift(lib_path)
+
+require 'goapi'
+
+go = GoAPI.new(server_url, basic_auth: [username, password])
+
+go.stages(pipeline_name, stage_name).select do |s|
+  s.result == 'Passed'
+end.map do |stage|
+  puts "fetch jobs' properties for #{go.stage_id(stage)}"
+  stage.jobs.map do |job|
+    go.job_properties(stage, job.name)
+  end
+end.flatten.map do |job|
+  [job.cruise_agent, job.cruise_job_duration.to_i]
+end.group_by do |tmp|
+  tmp[0]
+end.map do |agent, jobs|
+  times = jobs.map{|t| t[1]}
+  [agent, times.reduce(:+)/times.size, times.sort[times.size/2], times.size]
+end.sort_by{|r| r[1]}.reverse.each do |agent, avg, m, size|
+  puts "#{agent} ran #{size} jobs, avg time: #{avg.to_i} secs; median time: #{m.to_i} secs"
+end

data/examples/job_stats.rb

@@ -0,0 +1,26 @@
+
+server_url = <go server url>
+username = <username>
+password = <your password>
+pipeline_name = <pipeline name>
+stage_name = <stage name>
+
+lib_path = File.expand_path('../../lib', __FILE__)
+$:.unshift(lib_path)
+
+require 'goapi'
+
+go = GoAPI.new(server_url, basic_auth: [username, password])
+go.stages(pipeline_name, stage_name).select do |s|
+  s.result == 'Passed'
+end.map do |stage|
+  stage.jobs.map do |job|
+    properties = go.job_properties(stage, job.name)
+    OpenStruct.new(properties.to_h.merge(name: job.name))
+  end
+end.flatten.group_by(&:name).map do |name, jobs|
+  times = jobs.map(&:cruise_job_duration).map(&:to_i)
+  [name, times.reduce(:+)/times.size, times.sort[times.size/2], times.size]
+end.sort_by{|r| r[1]}.reverse.each do |name, avg, m, size|
+  puts "#{name} avg time: #{avg.to_i} secs; median time: #{m.to_i} secs"
+end

data/examples/junit_reports.rb

@@ -0,0 +1,88 @@
+
+# We have selenium test suite that ran 11 hours 45 minutes in sequence on a 2 CPUs 3.5 GB memory VM.
+# We configure 35 jobs on Go to run these tests. Tests are partitioned by [TLB] with a time based splitter.
+# [TLB] should partition tests equally on build.
+# But these jobs randomly ran from 18 minutes to 48 minutes. It seems really badly partitioned.
+#
+# This is one of scripts I wrote for myself to investigate the problem described above.
+# We use ci_reporter to generate junit reports on build.
+# This script fetches all junit reports on jobs of latest green stage.
+# And try partition tests based on test run times in 2 ways:
+#   1. random
+#   2. test run time
+#
+# [TLB]: https://github.com/test-load-balancer/tlb
+
+begin
+  require 'descriptive-statistics'
+rescue LoadError
+  puts "Need descriptive-statistics to run this example"
+  puts "You can install it by: gem install descriptive-statistics"
+  exit(1)
+end
+
+server_url = <go server url>
+username = <username>
+password = <your password>
+pipeline_name = <pipeline name>
+stage_name = <stage name>
+junit_report_file_name_regex = /TEST-[^.]+.xml/
+
+lib_path = File.expand_path('../../lib', __FILE__)
+$:.unshift(lib_path)
+
+require 'goapi'
+require 'rexml/document'
+
+class JunitReport < Struct.new(:name, :tests)
+  def self.parse(xml)
+    root = REXML::Document.new(xml).root
+    suite = root.attributes
+    tests = []
+    root.each_element('//testcase') do |e|
+      tests << OpenStruct.new(name: "#{suite['name']}##{e.attributes['name']}", time: e.attributes['time'].to_f)
+    end
+    new(suite['name'], tests)
+  end
+end
+
+go = GoAPI.new(server_url, basic_auth: [username, password])
+
+stage = go.stages(pipeline_name, stage_name).find do |s|
+  s.result == 'Passed'
+end
+
+require 'pstore'
+store = PStore.new('pstore')
+reports = store.transaction do
+  store['reports'] ||= stage.jobs.map do |job|
+    files = go.artifacts(stage, job.name).map{|a| a.type == 'folder' ? a.files : a}.flatten.select{|f| f.name =~ junit_report_file_name_regex}
+    puts "Job #{job.name}, #{files.size} reports"
+    files.map { |f| JunitReport.parse(go.artifact(f)) }
+  end.compact.flatten
+end
+
+print_stats = lambda do |partitions|
+  stats = DescriptiveStatistics::Stats.new(partitions.map {|p|p.map(&:time).reduce(:+).to_i})
+  puts "partitions: #{partitions.size}"
+  puts "mean: #{stats.mean}"
+  puts "median: #{stats.median}"
+  puts "max: #{stats.max}"
+  puts "min: #{stats.min}"
+  puts "standard deviation: #{stats.standard_deviation}"
+end
+
+tests = reports.map(&:tests).flatten
+puts tests.size
+puts tests.map(&:time).reduce(:+)/3600
+puts "Random partition result:"
+print_stats[tests.shuffle.each_slice(tests.size/35 + 1)]
+puts ''
+puts "Partitioned by test runtime"
+bins = Array.new(35) { [] }
+tests.sort_by(&:time).reverse.each do |test|
+  bin = bins.min_by{|bin| bin.empty? ? 0 : bin.map(&:time).reduce(:+)}
+  bin << test
+end
+print_stats[bins]
+

data/goapi.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'goapi/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "goapi"
+  spec.version       = Goapi::VERSION
+  spec.authors       = ["Xiao Li"]
+  spec.email         = ["swing1979@gmail.com"]
+
+  spec.summary       = %q{Go API ruby client.}
+  spec.homepage      = "https://github.com/ThoughtWorksStudios/goapi"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "json", "~> 1.8"
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/goapi.rb

@@ -0,0 +1,99 @@
+require "goapi/version"
+require "goapi/http"
+require 'logger'
+require 'json'
+require 'time'
+require 'uri'
+require 'ostruct'
+
+class GoAPI
+
+  class Stages
+    include Enumerable
+    def initialize(data)
+      @data = data
+    end
+
+    def each(&block)
+      @data.stages.each(&block)
+    end
+
+    def size
+      @data.stages
+    end
+  end
+
+  class << self
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+  end
+
+  # server: Go server url, for example: https://go01.domain.com
+  # credentials: 
+  #   only support basic auth right now, example: {basic_auth: [username, password]}
+  #   default value is nil
+  def initialize(server, credentials=nil)
+    @server, @http = server, Http.new(credentials)
+  end
+
+  def stages(pipeline_name, stage_name)
+    Stages.new(fetch(:api, :stages, pipeline_name, stage_name, :history))
+  end
+
+  # stage:
+  #   object has attribute methods: pipeline_name, pipeline_counter, name, counter
+  #   You can get this object by calling stages(pipeline_name, stage_name)
+  # job_name: job name
+  def job_properties(stage, job_name)
+    text = @http.get(url(:properties, stage.pipeline_name, stage.pipeline_counter, stage.name, stage.counter, job_name))[1]
+    names, values = text.split("\n")
+    values = values.split(',')
+    OpenStruct.new(Hash[names.split(',').each_with_index.map do |name, i|
+      [name, values[i]]
+    end])
+  end
+
+  # stage:
+  #   object has attribute methods: pipeline_name, pipeline_counter, name, counter
+  #   You can get this object by calling stages(pipeline_name, stage_name)
+  # job_name: job name
+  def artifacts(stage, job_name)
+    fetch(:files, stage.pipeline_name, stage.pipeline_counter, stage.name, stage.counter, "#{job_name}.json")
+  end
+
+  # artifact: object from artifacts(stage, job_name), it should have 2 attributes "type" and "url", and "type" should be "file"
+  def artifact(artifact)
+    raise "Only accept file type artifact" if artifact.type != 'file'
+    u = URI(artifact.url)
+    @http.get([@server, u.path].join('/'))[1]
+  end
+
+  # build stage uniq identifier for stage object
+  def stage_id(stage)
+    "#{stage.pipeline_name}/#{stage.pipeline_counter}/#{stage.name}/#{stage.counter}"
+  end
+
+  private
+  def fetch(*resources)
+    ostruct(JSON.parse(@http.get(url(*resources))[1]))
+  end
+
+  def url(*resources)
+    [@server, :go, *resources].join('/')
+  end
+
+  def ostruct(o)
+    case o
+    when Array
+      o.map(&method(:ostruct))
+    when Hash
+      r = o.map do |k, v|
+        [k, ostruct(v)]
+      end
+      OpenStruct.new(Hash[r])
+    else
+      o
+    end
+  end
+end

data/lib/goapi/http.rb

@@ -0,0 +1,57 @@
+require 'net/http'
+require 'net/https'
+
+class GoAPI
+  class Http
+    class Error < StandardError
+      def initialize(request_class, url, response)
+        super("error[#{request_class.name}][#{url}][#{response.code}]: #{response.body}")
+      end
+    end
+
+    def initialize(credentials)
+      @credentials = credentials || {}
+    end
+
+    def get(url)
+      process(Net::HTTP::Get, url)
+    end
+
+    def post(url, params)
+      process(Net::HTTP::Post, url, params)
+    end
+
+    def process(request_class, url, form_data={}, headers=nil)
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+
+      if uri.scheme == 'https'
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      request = request_class.new(uri.request_uri, form_data)
+
+      if headers
+        headers.each do |key, value|
+          request[key] = value
+        end
+      end
+
+      if @credentials[:basic_auth]
+        request.basic_auth *@credentials[:basic_auth]
+      end
+
+      response = http.request(request)
+      raise Error.new(request_class, url, response) if response.code.to_i >= 300
+
+      to_canonical_response(response)
+    end
+
+    def to_canonical_response(response)
+      headers = {}
+      response.each_header {|key, value|  headers[key] = value }
+      [response.code.to_i, response.body, headers]
+    end
+  end
+end

data/lib/goapi/version.rb

@@ -0,0 +1,3 @@
+module Goapi
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: goapi
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Xiao Li
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-08-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- swing1979@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- examples/agent_stats.rb
+- examples/job_stats.rb
+- examples/junit_reports.rb
+- goapi.gemspec
+- lib/goapi.rb
+- lib/goapi/http.rb
+- lib/goapi/version.rb
+homepage: https://github.com/ThoughtWorksStudios/goapi
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Go API ruby client.
+test_files: []