telly 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: daa063e14ee499acd934b81ead0304419094190b
+  data.tar.gz: a8f8240fe98212a75bca1f81c722981b735930cf
+SHA512:
+  metadata.gz: 9060bdcd028c788d6c2224b591d342d905510e2f9f75fab049ef4d8f92a241139d7d27888ae07d64141bfdc6958e54c81ddea93e6e907b205e6eb09a3a1ec4ec
+  data.tar.gz: 0f8d56d22cbccfe8d6e88650f2ab480db43bf88e561d003d40213f0aa224ad9efda3291e52d90f8cfbd8ce29b1bcd063acc2e6fd4b9adf33cb45f18cb4e070f4

data/bin/telly

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+# == telly.rb
+# This script adds test results in testrail from a finished beaker run
+# It takes in a beaker junit file and a TestRail Testrun ID
+# 
+# It matches the beaker tests with TestRail testcases by looking for the 
+#   test case ID in the beaker script. The combination of a test run and a test case
+#   allows the script to add a result for a particular instance of a test case.
+#   In TestRail parlance, this is confusingly called a test.
+# From the TestRail API docs:
+#   "In TestRail, tests are part of a test run and the test cases are part of the
+#   related test suite. So, when you create a new test run, TestRail creates a test
+#   for each test case found in the test suite of the run. You can therefore think
+#   of a test as an “instance” of a test case which can have test results, comments 
+#   and a test status.""
+
+require 'telly'
+require 'optparse'
+
+# Parses the command line options
+# @return [Void]
+def parse_opts
+  options_hash = {}
+
+  optparse = OptionParser.new do|parser|
+    options_hash = {
+      testrun_id: nil,
+      junit_file: nil,
+    }
+
+    parser.on( '-t', '--testrun-id TESTRUN_ID', 'The testrun id' ) do |testrun_id|
+      options_hash[:testrun_id] = testrun_id
+    end
+
+    parser.on( '-j', '--junit-folder JUNIT_FILE', 'Beaker junit file' ) do |junit_file|
+      options_hash[:junit_file] = junit_file
+    end
+
+    parser.on( '-h', '--help', 'Display this screen' ) do
+      puts parser
+      exit
+    end
+
+    parser.parse!
+
+    if not options_hash[:testrun_id] or not options_hash[:junit_file]
+      puts "Error: Missing option(s)"
+      puts parser
+      exit
+    end
+  end
+
+  options_hash
+end
+
+Telly::main(parse_opts)

data/lib/telly.rb

@@ -0,0 +1,289 @@
+#!/usr/bin/env ruby
+
+require 'nokogiri'
+require 'yaml'
+require 'pp'
+require 'testrail'
+
+# == telly.rb
+# This module provides functions to add test results in testrail from a 
+# finished beaker run. 
+# 
+# It takes in a beaker junit file and a TestRail Testrun ID
+# 
+# It matches the beaker tests with TestRail testcases by looking for the 
+#   test case ID in the beaker script. The combination of a test run and a test case
+#   allows the script to add a result for a particular instance of a test case.
+#   In TestRail parlance, this is confusingly called a test.
+# From the TestRail API docs:
+#   "In TestRail, tests are part of a test run and the test cases are part of the
+#   related test suite. So, when you create a new test run, TestRail creates a test
+#   for each test case found in the test suite of the run. You can therefore think
+#   of a test as an “instance” of a test case which can have test results, comments 
+#   and a test status.""
+module Telly
+
+  TESTRAIL_URL = 'https://testrail.ops.puppetlabs.net/'
+  CREDENTIALS_FILE = '~/.testrail_credentials.yaml'
+
+  # Used for extracted the test case ID from beaker scripts
+  TESTCASE_ID_REGEX = /.*(?<jira_ticket>\w+-\d+).*[cC](?<testrun_id>\d+)/
+
+  # Testrail Status IDs
+  PASSED = 1
+  BLOCKED = 2
+  FAILED = 5
+
+  def do_stub_test(credentials)
+    api = get_testrail_api(credentials)
+
+    api.send_post
+  end
+
+
+  ##################################
+  # Main
+  ##################################
+
+  # Run the importer
+  # 
+  # @param [Hash] An optparse object
+  #
+  # @return [Void]
+  #
+  # @example password = Telly::main(parse_opts)
+  def Telly.main(options)
+    # Get pass/fail/skips from junit file
+    results = load_junit_results(options[:junit_file])
+
+    puts "Run results:"
+    puts "#{results[:passes].length} Passing"
+    puts "#{results[:failures].length} Failing or Erroring"
+    puts "#{results[:skips].length} Skipped"
+
+    # Set results in testrail
+    bad_results = set_testrail_results(results, options[:junit_file], options[:testrun_id])
+
+    # Print error messages
+    if not bad_results.empty?
+      puts "Error: There were problems processing these test scripts:"
+      bad_results.each do |test_script, error|
+        puts "#{test_script}:\n\t#{error}"
+      end
+    end
+  end
+
+
+  ##################################
+  # TestRail API
+  ##################################
+
+  # Load testrail credentials from file
+  #
+  # @return [Hash] Contains testrail_username and testrail_password
+  #
+  # @example password = load_credentials()["testrail_password"]
+  def Telly.load_credentials(credentials_file)
+    begin
+      YAML.load_file(File.expand_path(credentials_file))  
+    rescue
+      puts "Error: Could not find #{credentials_file}"
+      puts "Create #{credentials_file} with the following:"
+      puts "testrail_username: your.username\ntestrail_password: yourpassword"
+
+      exit
+      
+    end
+  end
+
+
+  # Returns a testrail API object that talks to testrail
+  # 
+  # @param [Hash] credentials A hash containing at least two keys, testrail_username and testrail_password
+  #
+  # @return [TestRail::APIClient] The API object for talking to TestRail
+  #
+  # @example api = get_testrail_api(load_credentials)
+  def Telly.get_testrail_api(credentials)
+    client = TestRail::APIClient.new(TESTRAIL_URL)
+    client.user = credentials["testrail_username"]
+    client.password = credentials["testrail_password"]
+
+    return client
+  end
+
+  # Sets the results in testrail. 
+  # Tests that have testrail API exceptions are kept track of in bad_results
+  #
+  # @param [Hash] results A hash of lists of xml objects from the junit output file.
+  # @param [String] junit_file The path to the junit xml file
+  #                 Needed for determining the path of the test file in add_failure, etc
+  # @param [String] testrun_id The TestRail test run ID
+  # 
+  # @return [Void] 
+  #
+  def Telly.set_testrail_results(results, junit_file, testrun_id)
+    credentials = load_credentials(CREDENTIALS_FILE)
+    api = get_testrail_api(credentials)
+
+    # Results that couldn't be set in testrail for some reason
+    bad_results = {}
+
+    # passes
+    results[:passes].each do |junit_result|
+      begin
+        submit_result(api, PASSED, junit_result, junit_file, testrun_id)    
+      rescue TestRail::APIError => e
+        bad_results[junit_result[:name]] = e.message
+      end
+    end
+
+    # Failures
+    results[:failures].each do |junit_result|
+      begin
+        submit_result(api, FAILED, junit_result, junit_file, testrun_id)    
+      rescue TestRail::APIError => e
+        bad_results[junit_result[:name]] = e.message
+      end
+    end
+
+    # Skips
+    results[:skips].each do |junit_result|
+      begin
+        submit_result(api, BLOCKED, junit_result, junit_file, testrun_id)    
+      rescue TestRail::APIError => e
+        bad_results[junit_result[:name]] = e.message
+      end
+    end
+
+    return bad_results
+  end
+
+  # Submits a test result to TestRail
+  #
+  # @param [TestRail::APIClient] api TestRail API object
+  # @param [int] status The testrail status to set
+  # @param [Nokogiri::XML::Element] junit_result The nokogiri node that holds the junit result
+  # @param [String] junit_file Path to the junit file the test result originated from
+  # @param [String] testrun_id The testrun ID
+  #
+  # @return [Void]
+  # 
+  # @raise [TestRail::APIError] When there is a problem with the API request, testrail raises
+  #                             this exception. Should be caught for error reporting
+  #
+  # @example submit_result(api, BLOCKED, junit_result, junit_file, testrun_id)
+  def Telly.submit_result(api, status, junit_result, junit_file, testrun_id)
+    test_file_path = beaker_test_path(junit_file, junit_result)
+
+    puts junit_result.class
+    testcase_id = testcase_id_from_beaker_script(test_file_path)
+
+    time_elapsed = make_testrail_time(junit_result[:time])
+
+    # Make appropriate comment for testrail
+    case status
+    when FAILED
+      error_message = junit_result.xpath('./failure').first[:message]
+      testrail_comment = "Failed with message:\n#{error_message}"
+    when BLOCKED
+      skip_message = junit_result.xpath('system-out').first.text
+      testrail_comment = "Skipped with message:\n#{skip_message}"
+    else
+      testrail_comment = "Passed"
+    end
+
+    puts "\nSetting result for test case: #{testcase_id}"
+    puts "Adding comment:\n#{testrail_comment}"
+
+    api.send_post("add_result_for_case/#{testrun_id}/#{testcase_id}", 
+      {
+        status_id: status,
+        comment: testrail_comment,
+        elapsed: time_elapsed,
+      }
+    )
+  end
+
+
+  # Returns a string that testrail accepts as an elapsed time
+  # Input from beaker is a float in seconds, so it rounds it to the 
+  # nearest second, and adds an 's' at the end
+  # 
+  # Testrail throws an exception if it gets "0s", so it returns a 
+  # minimum of "1s"
+  #
+  # @param [String] seconds_string A string that contains only a number, the elapsed time of a test
+  #
+  # @return [String] The elapsed time of the test run, rounded and with an 's' appended
+  #
+  # @example puts make_testrail_time("2.34") # "2s"
+  def Telly.make_testrail_time(seconds_string)
+    # If time is 0, make it 1
+    rounded_time = [seconds_string.to_f.round, 1].max
+    # Test duration
+    time_elapsed = "#{rounded_time}s"
+
+    return time_elapsed
+  end
+
+
+  ##################################
+  # Junit and Beaker file functions
+  ##################################
+
+  # Loads the results of a beaker run.
+  # Returns hash of failures, passes, and skips that each hold a list of 
+  # junit xml objects
+  #
+  # @param [String] junit_file Path to a junit xml file
+  # 
+  # @return [Hash] A hash containing xml objects for the failures, skips, and passes
+  #
+  # @example load_junit_results("~/junit/latest/beaker_junit.xml")
+  def Telly.load_junit_results(junit_file)
+    junit_doc = Nokogiri::XML(File.read(junit_file))
+
+    failures = junit_doc.xpath('//testcase[failure]')
+    skips = junit_doc.xpath('//testcase[skip]')
+    passes = junit_doc.xpath('//testcase[not(failure) and not(skip)]')
+
+    return {failures: failures, skips: skips, passes: passes}
+  end
+
+
+  # Extracts the test case id from the test script
+  #
+  # @param [String] beaker_file Path to a beaker test script
+  # 
+  # @return [String] The test case ID
+  #
+  # @example testcase_id_from_beaker_script("~/tests/test_the_things.rb") # 1234
+  def Telly.testcase_id_from_beaker_script(beaker_file)
+    # Find first matching line
+    match = File.readlines(beaker_file).map { |line| line.match(TESTCASE_ID_REGEX) }.compact.first
+
+    match[:testrun_id]
+  end
+
+
+  # Calculates the path to a beaker test file by combining the junit file path
+  # with the test name from the junit results.
+  # Makes the assumption that junit folder that beaker creates will always be 
+  # 2 directories up from the beaker script base directory.
+  # TODO somewhat hacky, maybe a config/command line option
+  #
+  # @param [String] junit_file_path Path to a junit xml file
+  # @param [String] junit_result Path to a junit xml file
+  # 
+  # @return [String] The path to the beaker script from the junit test result
+  #
+  # @example load_junit_results("~/junit/latest/beaker_junit.xml")
+  def Telly.beaker_test_path(junit_file_path, junit_result)
+    beaker_folder_path = junit_result[:classname]
+    test_filename = junit_result[:name]
+
+    File.join(File.dirname(junit_file_path), "../../", beaker_folder_path, test_filename)
+  end
+
+end

data/lib/testrail.rb

@@ -0,0 +1,106 @@
+#
+# TestRail API binding for Ruby (API v2, available since TestRail 3.0)
+#
+# Learn more:
+#
+# http://docs.gurock.com/testrail-api2/start
+# http://docs.gurock.com/testrail-api2/accessing
+#
+# Copyright Gurock Software GmbH. See license.md for details.
+#
+
+require 'net/http'
+require 'net/https'
+require 'uri'
+require 'json'
+
+module TestRail
+	class APIClient
+		@url = ''
+		@user = ''
+		@password = ''
+
+		attr_accessor :user
+		attr_accessor :password
+
+		def initialize(base_url)
+			if !base_url.match(/\/$/)
+				base_url += '/'
+			end
+			@url = base_url + 'index.php?/api/v2/'
+		end
+
+		#
+		# Send Get
+		#
+		# Issues a GET request (read) against the API and returns the result
+		# (as Ruby hash).
+		#
+		# Arguments:
+		#
+		# uri                 The API method to call including parameters
+		#                     (e.g. get_case/1)
+		#
+		def send_get(uri) 
+			_send_request('GET', uri, nil)
+		end
+
+		#
+		# Send POST
+		#
+		# Issues a POST request (write) against the API and returns the result
+		# (as Ruby hash).
+		#
+		# Arguments:
+		#
+		# uri                 The API method to call including parameters
+		#                     (e.g. add_case/1)
+		# data                The data to submit as part of the request (as
+		#                     Ruby hash, strings must be UTF-8 encoded)
+		#
+		def send_post(uri, data)
+			_send_request('POST', uri, data)
+		end
+
+		private
+		def _send_request(method, uri, data)
+			url = URI.parse(@url + uri)
+			if method == 'POST'
+				request = Net::HTTP::Post.new(url.path + '?' + url.query)
+				request.body = JSON.dump(data)
+			else
+				request = Net::HTTP::Get.new(url.path + '?' + url.query)
+			end
+			request.basic_auth(@user, @password)
+			request.add_field('Content-Type', 'application/json')
+
+			conn = Net::HTTP.new(url.host, url.port)
+			if url.scheme == 'https'
+				conn.use_ssl = true
+				conn.verify_mode = OpenSSL::SSL::VERIFY_NONE
+			end
+			response = conn.request(request)
+
+			if response.body && !response.body.empty?
+				result = JSON.parse(response.body)
+			else
+				result = {}
+			end
+
+			if response.code != '200'
+				if result && result.key?('error')
+					error = '"' + result['error'] + '"'
+				else
+					error = 'No additional error message received'
+				end
+				raise APIError.new('TestRail API returned HTTP %s (%s)' %
+					[response.code, error])
+			end
+			
+			result
+		end
+	end
+
+	class APIError < StandardError
+	end
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: telly
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Joe Pinsonault
+- Puppetlabs
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Imports beaker results into TestRail test run results
+email: joe.pinsonault@gmail.com
+executables:
+- telly
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/telly
+- lib/telly.rb
+- lib/testrail.rb
+homepage: http://rubygems.org/gems/telly
+licenses:
+- Apache
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Imports beaker results into TestRail test run results
+test_files: []