goapi 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: da6488566478de3c1f4bd2a82aaadb9a7d1c4aba
4
+ data.tar.gz: f50eed750d0ad1058e84fdb4e0a66128a66d81c6
5
+ SHA512:
6
+ metadata.gz: 6e1e41bcbb6531fdcf2870c45615e8cddecec35b0c86aef79848ff4ccff0e2748ea75ace5ece5e34885aee681f3f260add2e63483ba61c63af474b7be05bde37
7
+ data.tar.gz: 897c445726b3234d8877510a7ff61bef37067b7238eccf86557363426630f15ac11be112820883b7de2e5a978e8b4898d2ff19fdd3ee87062de53a33298bc9d8
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in goapi.gemspec
4
+ gemspec
@@ -0,0 +1,67 @@
1
+ # Go API
2
+
3
+ [Go](http://www.go.cd) API Ruby client.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'goapi'
11
+ ```
12
+
13
+ And then execute:
14
+
15
+ $ bundle
16
+
17
+ Or install it yourself as:
18
+
19
+ $ gem install goapi
20
+
21
+ ## Usage
22
+
23
+ ### Initialize with Go server url and Basic Auth credentials
24
+
25
+ require 'goapi'
26
+
27
+ go = GoAPI.new('server-url', basic_auth: [username, password])
28
+
29
+ ### Fetch pipeline stages history
30
+
31
+ require 'goapi'
32
+ go = GoAPI.new('server-url', basic_auth: [username, password])
33
+ go.stages(pipeline_name, stage_name)
34
+
35
+ ### Fetch artifacts
36
+
37
+ require 'goapi'
38
+ go = GoAPI.new('server-url', basic_auth: [username, password])
39
+ stages = go.stages(pipeline_name, stage_name)
40
+ artifacts = go.artifacts(stages[0], job_name)
41
+ artifacts.map do |artifact|
42
+ if artifact.type == 'file'
43
+ go.artifact(artifact)
44
+ else # when type == 'folder'
45
+ ....
46
+ end
47
+ end
48
+
49
+ More complex examples please checkout [examples](examples) directory in codebase
50
+
51
+ ## API design
52
+
53
+ 1. flat: one level API, you can find all APIs definition in class GoAPI.
54
+ 2. data: all APIs return data object only, they are:
55
+ 1. Primitive type in Ruby
56
+ 2. OpenStruct object, with primitive type values (rule #1 flat)
57
+
58
+ ## Development
59
+
60
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake false` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
61
+
62
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
63
+
64
+ ## Contributing
65
+
66
+ Bug reports and pull requests are welcome on GitHub at https://github.com/ThoughtWorksStudios/goapi.
67
+
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "goapi"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
@@ -0,0 +1,7 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+
5
+ bundle install
6
+
7
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,31 @@
1
+
2
+ server_url = <go server url>
3
+ username = <username>
4
+ password = <your password>
5
+ pipeline_name = <pipeline name>
6
+ stage_name = <stage name>
7
+
8
+ lib_path = File.expand_path('../../lib', __FILE__)
9
+ $:.unshift(lib_path)
10
+
11
+ require 'goapi'
12
+
13
+ go = GoAPI.new(server_url, basic_auth: [username, password])
14
+
15
+ go.stages(pipeline_name, stage_name).select do |s|
16
+ s.result == 'Passed'
17
+ end.map do |stage|
18
+ puts "fetch jobs' properties for #{go.stage_id(stage)}"
19
+ stage.jobs.map do |job|
20
+ go.job_properties(stage, job.name)
21
+ end
22
+ end.flatten.map do |job|
23
+ [job.cruise_agent, job.cruise_job_duration.to_i]
24
+ end.group_by do |tmp|
25
+ tmp[0]
26
+ end.map do |agent, jobs|
27
+ times = jobs.map{|t| t[1]}
28
+ [agent, times.reduce(:+)/times.size, times.sort[times.size/2], times.size]
29
+ end.sort_by{|r| r[1]}.reverse.each do |agent, avg, m, size|
30
+ puts "#{agent} ran #{size} jobs, avg time: #{avg.to_i} secs; median time: #{m.to_i} secs"
31
+ end
@@ -0,0 +1,26 @@
1
+
2
+ server_url = <go server url>
3
+ username = <username>
4
+ password = <your password>
5
+ pipeline_name = <pipeline name>
6
+ stage_name = <stage name>
7
+
8
+ lib_path = File.expand_path('../../lib', __FILE__)
9
+ $:.unshift(lib_path)
10
+
11
+ require 'goapi'
12
+
13
+ go = GoAPI.new(server_url, basic_auth: [username, password])
14
+ go.stages(pipeline_name, stage_name).select do |s|
15
+ s.result == 'Passed'
16
+ end.map do |stage|
17
+ stage.jobs.map do |job|
18
+ properties = go.job_properties(stage, job.name)
19
+ OpenStruct.new(properties.to_h.merge(name: job.name))
20
+ end
21
+ end.flatten.group_by(&:name).map do |name, jobs|
22
+ times = jobs.map(&:cruise_job_duration).map(&:to_i)
23
+ [name, times.reduce(:+)/times.size, times.sort[times.size/2], times.size]
24
+ end.sort_by{|r| r[1]}.reverse.each do |name, avg, m, size|
25
+ puts "#{name} avg time: #{avg.to_i} secs; median time: #{m.to_i} secs"
26
+ end
@@ -0,0 +1,88 @@
1
+
2
+ # We have selenium test suite that ran 11 hours 45 minutes in sequence on a 2 CPUs 3.5 GB memory VM.
3
+ # We configure 35 jobs on Go to run these tests. Tests are partitioned by [TLB] with a time based splitter.
4
+ # [TLB] should partition tests equally on build.
5
+ # But these jobs randomly ran from 18 minutes to 48 minutes. It seems really badly partitioned.
6
+ #
7
+ # This is one of scripts I wrote for myself to investigate the problem described above.
8
+ # We use ci_reporter to generate junit reports on build.
9
+ # This script fetches all junit reports on jobs of latest green stage.
10
+ # And try partition tests based on test run times in 2 ways:
11
+ # 1. random
12
+ # 2. test run time
13
+ #
14
+ # [TLB]: https://github.com/test-load-balancer/tlb
15
+
16
+ begin
17
+ require 'descriptive-statistics'
18
+ rescue LoadError
19
+ puts "Need descriptive-statistics to run this example"
20
+ puts "You can install it by: gem install descriptive-statistics"
21
+ exit(1)
22
+ end
23
+
24
+ server_url = <go server url>
25
+ username = <username>
26
+ password = <your password>
27
+ pipeline_name = <pipeline name>
28
+ stage_name = <stage name>
29
+ junit_report_file_name_regex = /TEST-[^.]+.xml/
30
+
31
+ lib_path = File.expand_path('../../lib', __FILE__)
32
+ $:.unshift(lib_path)
33
+
34
+ require 'goapi'
35
+ require 'rexml/document'
36
+
37
+ class JunitReport < Struct.new(:name, :tests)
38
+ def self.parse(xml)
39
+ root = REXML::Document.new(xml).root
40
+ suite = root.attributes
41
+ tests = []
42
+ root.each_element('//testcase') do |e|
43
+ tests << OpenStruct.new(name: "#{suite['name']}##{e.attributes['name']}", time: e.attributes['time'].to_f)
44
+ end
45
+ new(suite['name'], tests)
46
+ end
47
+ end
48
+
49
+ go = GoAPI.new(server_url, basic_auth: [username, password])
50
+
51
+ stage = go.stages(pipeline_name, stage_name).find do |s|
52
+ s.result == 'Passed'
53
+ end
54
+
55
+ require 'pstore'
56
+ store = PStore.new('pstore')
57
+ reports = store.transaction do
58
+ store['reports'] ||= stage.jobs.map do |job|
59
+ files = go.artifacts(stage, job.name).map{|a| a.type == 'folder' ? a.files : a}.flatten.select{|f| f.name =~ junit_report_file_name_regex}
60
+ puts "Job #{job.name}, #{files.size} reports"
61
+ files.map { |f| JunitReport.parse(go.artifact(f)) }
62
+ end.compact.flatten
63
+ end
64
+
65
+ print_stats = lambda do |partitions|
66
+ stats = DescriptiveStatistics::Stats.new(partitions.map {|p|p.map(&:time).reduce(:+).to_i})
67
+ puts "partitions: #{partitions.size}"
68
+ puts "mean: #{stats.mean}"
69
+ puts "median: #{stats.median}"
70
+ puts "max: #{stats.max}"
71
+ puts "min: #{stats.min}"
72
+ puts "standard deviation: #{stats.standard_deviation}"
73
+ end
74
+
75
+ tests = reports.map(&:tests).flatten
76
+ puts tests.size
77
+ puts tests.map(&:time).reduce(:+)/3600
78
+ puts "Random partition result:"
79
+ print_stats[tests.shuffle.each_slice(tests.size/35 + 1)]
80
+ puts ''
81
+ puts "Partitioned by test runtime"
82
+ bins = Array.new(35) { [] }
83
+ tests.sort_by(&:time).reverse.each do |test|
84
+ bin = bins.min_by{|bin| bin.empty? ? 0 : bin.map(&:time).reduce(:+)}
85
+ bin << test
86
+ end
87
+ print_stats[bins]
88
+
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'goapi/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "goapi"
8
+ spec.version = Goapi::VERSION
9
+ spec.authors = ["Xiao Li"]
10
+ spec.email = ["swing1979@gmail.com"]
11
+
12
+ spec.summary = %q{Go API ruby client.}
13
+ spec.homepage = "https://github.com/ThoughtWorksStudios/goapi"
14
+
15
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
16
+ spec.bindir = "exe"
17
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_dependency "json", "~> 1.8"
21
+
22
+ spec.add_development_dependency "bundler", "~> 1.10"
23
+ spec.add_development_dependency "rake", "~> 10.0"
24
+ end
@@ -0,0 +1,99 @@
1
+ require "goapi/version"
2
+ require "goapi/http"
3
+ require 'logger'
4
+ require 'json'
5
+ require 'time'
6
+ require 'uri'
7
+ require 'ostruct'
8
+
9
+ class GoAPI
10
+
11
+ class Stages
12
+ include Enumerable
13
+ def initialize(data)
14
+ @data = data
15
+ end
16
+
17
+ def each(&block)
18
+ @data.stages.each(&block)
19
+ end
20
+
21
+ def size
22
+ @data.stages
23
+ end
24
+ end
25
+
26
+ class << self
27
+ def logger
28
+ @logger ||= Logger.new(STDOUT)
29
+ end
30
+ end
31
+
32
+ # server: Go server url, for example: https://go01.domain.com
33
+ # credentials:
34
+ # only support basic auth right now, example: {basic_auth: [username, password]}
35
+ # default value is nil
36
+ def initialize(server, credentials=nil)
37
+ @server, @http = server, Http.new(credentials)
38
+ end
39
+
40
+ def stages(pipeline_name, stage_name)
41
+ Stages.new(fetch(:api, :stages, pipeline_name, stage_name, :history))
42
+ end
43
+
44
+ # stage:
45
+ # object has attribute methods: pipeline_name, pipeline_counter, name, counter
46
+ # You can get this object by calling stages(pipeline_name, stage_name)
47
+ # job_name: job name
48
+ def job_properties(stage, job_name)
49
+ text = @http.get(url(:properties, stage.pipeline_name, stage.pipeline_counter, stage.name, stage.counter, job_name))[1]
50
+ names, values = text.split("\n")
51
+ values = values.split(',')
52
+ OpenStruct.new(Hash[names.split(',').each_with_index.map do |name, i|
53
+ [name, values[i]]
54
+ end])
55
+ end
56
+
57
+ # stage:
58
+ # object has attribute methods: pipeline_name, pipeline_counter, name, counter
59
+ # You can get this object by calling stages(pipeline_name, stage_name)
60
+ # job_name: job name
61
+ def artifacts(stage, job_name)
62
+ fetch(:files, stage.pipeline_name, stage.pipeline_counter, stage.name, stage.counter, "#{job_name}.json")
63
+ end
64
+
65
+ # artifact: object from artifacts(stage, job_name), it should have 2 attributes "type" and "url", and "type" should be "file"
66
+ def artifact(artifact)
67
+ raise "Only accept file type artifact" if artifact.type != 'file'
68
+ u = URI(artifact.url)
69
+ @http.get([@server, u.path].join('/'))[1]
70
+ end
71
+
72
+ # build stage uniq identifier for stage object
73
+ def stage_id(stage)
74
+ "#{stage.pipeline_name}/#{stage.pipeline_counter}/#{stage.name}/#{stage.counter}"
75
+ end
76
+
77
+ private
78
+ def fetch(*resources)
79
+ ostruct(JSON.parse(@http.get(url(*resources))[1]))
80
+ end
81
+
82
+ def url(*resources)
83
+ [@server, :go, *resources].join('/')
84
+ end
85
+
86
+ def ostruct(o)
87
+ case o
88
+ when Array
89
+ o.map(&method(:ostruct))
90
+ when Hash
91
+ r = o.map do |k, v|
92
+ [k, ostruct(v)]
93
+ end
94
+ OpenStruct.new(Hash[r])
95
+ else
96
+ o
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,57 @@
1
+ require 'net/http'
2
+ require 'net/https'
3
+
4
+ class GoAPI
5
+ class Http
6
+ class Error < StandardError
7
+ def initialize(request_class, url, response)
8
+ super("error[#{request_class.name}][#{url}][#{response.code}]: #{response.body}")
9
+ end
10
+ end
11
+
12
+ def initialize(credentials)
13
+ @credentials = credentials || {}
14
+ end
15
+
16
+ def get(url)
17
+ process(Net::HTTP::Get, url)
18
+ end
19
+
20
+ def post(url, params)
21
+ process(Net::HTTP::Post, url, params)
22
+ end
23
+
24
+ def process(request_class, url, form_data={}, headers=nil)
25
+ uri = URI(url)
26
+ http = Net::HTTP.new(uri.host, uri.port)
27
+
28
+ if uri.scheme == 'https'
29
+ http.use_ssl = true
30
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
31
+ end
32
+
33
+ request = request_class.new(uri.request_uri, form_data)
34
+
35
+ if headers
36
+ headers.each do |key, value|
37
+ request[key] = value
38
+ end
39
+ end
40
+
41
+ if @credentials[:basic_auth]
42
+ request.basic_auth *@credentials[:basic_auth]
43
+ end
44
+
45
+ response = http.request(request)
46
+ raise Error.new(request_class, url, response) if response.code.to_i >= 300
47
+
48
+ to_canonical_response(response)
49
+ end
50
+
51
+ def to_canonical_response(response)
52
+ headers = {}
53
+ response.each_header {|key, value| headers[key] = value }
54
+ [response.code.to_i, response.body, headers]
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,3 @@
1
+ module Goapi
2
+ VERSION = "0.1.0"
3
+ end
metadata ADDED
@@ -0,0 +1,98 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: goapi
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Xiao Li
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2015-08-23 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: json
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.8'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.8'
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.10'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.10'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '10.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '10.0'
55
+ description:
56
+ email:
57
+ - swing1979@gmail.com
58
+ executables: []
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - ".gitignore"
63
+ - Gemfile
64
+ - README.md
65
+ - Rakefile
66
+ - bin/console
67
+ - bin/setup
68
+ - examples/agent_stats.rb
69
+ - examples/job_stats.rb
70
+ - examples/junit_reports.rb
71
+ - goapi.gemspec
72
+ - lib/goapi.rb
73
+ - lib/goapi/http.rb
74
+ - lib/goapi/version.rb
75
+ homepage: https://github.com/ThoughtWorksStudios/goapi
76
+ licenses: []
77
+ metadata: {}
78
+ post_install_message:
79
+ rdoc_options: []
80
+ require_paths:
81
+ - lib
82
+ required_ruby_version: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ">="
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
87
+ required_rubygems_version: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ requirements: []
93
+ rubyforge_project:
94
+ rubygems_version: 2.4.5
95
+ signing_key:
96
+ specification_version: 4
97
+ summary: Go API ruby client.
98
+ test_files: []