autoproj-jenkins 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f5d96120cc78350275e92584c61af35c413bcafb
+  data.tar.gz: d1bd6d3060ea76972df27145c9fceed592354ba9
+SHA512:
+  metadata.gz: 6d3f538eac2dd5471fb6bcf43f5f788e7babbe67ce8ced5edc8cc73a10eefeeece7e57dc9c55eacf37c2ebbac9b38ee27b2eec1e8112a78ef14fd28ad883e80d
+  data.tar.gz: 31525d9641c6df34e2b1b41af1700d3436381ca40bdacb3a4ec1b8b145598babbee284c4def436bb58d686ef721c3e89cf2f7d11da01be6e5682642846bd5e3e

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.sw?

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in autoproj-jenkins.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Sylvain Joyeux
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# Autoproj::Jenkins
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/autoproj/jenkins`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'autoproj-jenkins'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install autoproj-jenkins
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/autoproj-jenkins.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+  t.warning = false
+end
+
+task :default => :test

data/autoproj-jenkins.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'autoproj/jenkins/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "autoproj-jenkins"
+  spec.version       = Autoproj::Jenkins::VERSION
+  spec.authors       = ["Sylvain Joyeux"]
+  spec.email         = ["sylvain.joyeux@m4x.org"]
+
+  spec.summary       = %q{Exporting an autoproj build to a jenkins server}
+  spec.homepage      = "https://github.com/doudou/autoproj-jenkins"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'jenkins_api_client'
+  spec.add_dependency 'autoproj', '>= 2.0.0.rc20'
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 11.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "flexmock"
+end

data/lib/autoproj/cli/jenkins.rb

@@ -0,0 +1,100 @@
+require 'autoproj/cli/inspection_tool'
+require 'autoproj/jenkins'
+
+module Autoproj
+    module CLI
+        class Jenkins < Autoproj::CLI::InspectionTool
+            attr_reader :server
+            attr_reader :updater
+
+            def initialize(ws, job_prefix: '', **options)
+                super(ws)
+                @server = Autoproj::Jenkins::Server.new(**options)
+                @updater = Autoproj::Jenkins::Updater.new(ws, server, job_prefix: job_prefix)
+            end
+
+            def parse_vcs_credentials(credentials)
+                results = Autoproj::Jenkins::Credentials.new
+                credentials.each do |argument|
+                    credential = Autoproj::Jenkins::Credentials.parse(argument)
+                    results.add(credential)
+                end
+                results
+            end
+
+            def create_or_update_buildconf_job(*package_names, force: false, dev: false, credentials_id: nil, vcs_credentials: [])
+                initialize_and_load
+
+                if dev
+                    gemfile = 'buildconf-vagrant-Gemfile'
+                    autoproj_install_path = '/opt/autoproj/bin/autoproj_install'
+                else
+                    gemfile = 'buildconf-Gemfile'
+                    autoproj_install_path = nil
+                end
+
+                finalize_setup(package_names, recursive: false, non_imported_packages: :return)
+                updater.create_or_update_buildconf_job(*package_names, gemfile: gemfile,
+                                                       autoproj_install_path: autoproj_install_path, dev: dev,
+                                                       credentials_id: credentials_id,
+                                                       vcs_credentials: parse_vcs_credentials(vcs_credentials))
+            end
+
+            def add_or_update_packages(*package_names, dev: false, vcs_credentials: [])
+                initialize_and_load
+                source_packages, _ = finalize_setup(package_names, non_imported_packages: nil)
+                source_packages = source_packages.map do |package_name|
+                    ws.manifest.package_definition_by_name(package_name)
+                end
+
+                if dev
+                    gemfile = 'buildconf-vagrant-Gemfile'
+                    autoproj_install_path = '/opt/autoproj/bin/autoproj_install'
+                else
+                    gemfile = 'buildconf-Gemfile'
+                    autoproj_install_path = nil
+                end
+
+                updater.update(
+                    *source_packages,
+                    gemfile: gemfile,
+                    autoproj_install_path: autoproj_install_path,
+                    vcs_credentials: parse_vcs_credentials(vcs_credentials))
+            end
+
+            # Returns the "roots" in the trigger graph
+            #
+            # The trigger graph is the inverse of the dependency graph
+            # (a package's dependencies are built before the package itself)
+            #
+            # @param [Array<String>] package_names the packages whose trigger
+            #   roots we want to find
+            # @return [Array<String>] the trigger root packages
+            def trigger_root_packages(*package_names)
+                package_names.find_all do |pkg_name|
+                    pkg = ws.manifest.find_autobuild_package(pkg_name)
+                    if !pkg
+                        raise ArgumentError, "#{pkg_name} is not a known package"
+                    end
+                    pkg.dependencies.all? do |dep_name|
+                        !package_names.include?(dep_name)
+                    end
+                end
+            end
+
+            # Trigger the build of the given packages
+            #
+            # It actually only triggers the jobs that are roots in the trigger
+            # graph
+            #
+            # @param [Array<String>] package_names the names of the packages to
+            #   build
+            def trigger_packages(*package_names)
+                trigger_root_packages(*package_names).each do |pkg_name|
+                    server.trigger_job(updater.job_name_from_package_name(pkg_name))
+                end
+            end
+        end
+    end
+end
+

data/lib/autoproj/cli/main_jenkins.rb

@@ -0,0 +1,123 @@
+module Autoproj
+    module CLI
+        # The 'jenkins' subcommand for autoproj
+        class MainJenkins < Thor
+            class_option :username, desc: 'username to the Jenkins CLI (a password will be requested if --password is not given)'
+            class_option :password, desc: 'password to the Jenkins CLI (needs --username)'
+            class_option :job_prefix, desc: 'string that should be used as prefix to all generated job names',
+                type: :string, default: ''
+
+            namespace 'jenkins'
+
+            no_commands do
+                def request_password
+                    STDOUT.print "Password: "
+                    STDOUT.flush
+                    STDIN.noecho do |io|
+                        io.readline.chomp
+                    end
+                end
+
+                def create_ops(url, target_os: nil)
+                    if username = options[:username]
+                        password = options[:password] || request_password
+                        auth = Hash[username: username,
+                                    password: password]
+                    elsif options[:password]
+                        raise ArgumentError, "--password given without --username"
+                    else
+                        auth = Hash.new
+                    end
+
+                    workspace_options = Hash.new
+                    if target_os
+                        names, versions =  target_os.split(':')
+                        names = names.split(',')
+                        names << 'default'
+                        versions = versions.split(',')
+                        versions << 'default'
+                        workspace_options[:os_package_resolver] = OSPackageResolver.new(operating_system: [names, versions])
+                    end
+                    ws = Autoproj::Workspace.default(**workspace_options)
+
+                    STDERR.puts "connecting to jenkins '#{url}' with prefix '#{options[:prefix]}'"
+                    Jenkins.new(ws,
+                                job_prefix: options[:job_prefix],
+                                server_url: url,
+                                **auth)
+                end
+            end
+
+            desc 'init URL [PACKAGE NAMES]', 'initialize the jenkins server by creating the base build job, optionally restricting the build to certain packages'
+            option :force, desc: 'if set, delete any existing job'
+            option :trigger, desc: 'trigger the job once created',
+                type: :boolean, default: false
+            option :dev, desc: 'assume that the jenkins instance is a development instance under vagrant and that autoproj-jenkins is made available as /opt/autoproj-jenkins',
+                type: :boolean, default: false
+            option :target_os, desc: "the autoproj definition for the target OS as name0,name1:version0,version1",
+                default: nil
+            option :credentials_id, desc: "the credentials ID of the username/password credentials that autoproj-jenkins should use to access the jenkins CLI",
+                default: 'autoproj-jenkins-cli'
+            option :vcs_credentials, desc: 'list of vcs_type:URLs for which credentials should be provided (see documentation)',
+                type: :array, default: []
+            def init(url, *package_names)
+                require 'autoproj/cli/jenkins'
+                ops = create_ops(url, target_os: options[:target_os])
+
+                ops.create_or_update_buildconf_job(
+                    *package_names,
+                    credentials_id: options[:credentials_id],
+                    force: options[:force],
+                    vcs_credentials: options[:vcs_credentials],
+                    dev: options[:dev])
+                if options[:trigger]
+                    ops.trigger_buildconf_job
+                end
+            end
+
+
+            desc 'update [PACKAGE_NAMES]', 'add the following package and its dependencies to the jenkins build'
+            option :force, desc: 'ignore the current state, generate jobs as if nothing was ever done'
+            option :dev, desc: 'assume that the jenkins instance is a development instance under vagrant and that autoproj-jenkins is made available as /opt/autoproj-jenkins',
+                type: :boolean, default: false
+            option :vcs_credentials, desc: 'list of vcs_type:URLs for which credentials should be provided (see documentation)',
+                type: :array, default: []
+            def update(url, *package_names)
+                require 'autoproj/cli/jenkins'
+                ops = create_ops(url)
+                Autoproj.report(silent: !options[:debug], debug: options[:debug]) do
+                    updated_jobs = ops.add_or_update_packages(*package_names, dev: options[:dev], vcs_credentials: options[:vcs_credentials])
+                    updated_jobs.sort.each do |job_name|
+                        puts job_name
+                    end
+                end
+            end
+            
+            desc 'postprocess-tests OUTPUT_DIR [PACKAGE_NAME]', 'postprocesses test result formatted in various formats to convert them into the JUnit XML format understood by Jenkins',
+                hide: true
+            option :after, desc: "if provided, any report file that is older than this file will be ignored",
+                default: nil
+            def postprocess_tests(output_dir, *package_names)
+                require 'autoproj/cli/test_postprocessing'
+                ops = TestPostprocessing.new(Workspace.default)
+                if options[:after]
+                    reference_time = File.stat(options[:after]).mtime
+                end
+                ops.process(output_dir, *package_names, after: reference_time)
+            end
+
+            desc 'relativize ROOT_DIR INPUT_TEXT OUTPUT_TEXT', 'replaces INPUT_TEXT by OUTPUT_TEXT in all files that can contain absolute paths',
+                hide: true
+            def relativize(root_dir, input_text, output_text)
+                require 'autoproj/jenkins'
+                relativize = Autoproj::Jenkins::Relativize.new(Pathname.new(root_dir), input_text, output_text)
+                processed_paths = relativize.process
+                puts "modified #{processed_paths.size} file"
+                processed_paths.each do |p|
+                    puts "  #{p}"
+                end
+            end
+        end
+    end
+end
+

data/lib/autoproj/cli/test_postprocessing.rb

@@ -0,0 +1,85 @@
+require 'autoproj/cli/inspection_tool'
+
+module Autoproj
+    module CLI
+        class TestPostprocessing < Autoproj::CLI::InspectionTool
+            attr_reader :convertions
+
+            def self.test_format_converter_dir
+                File.expand_path(
+                    File.join('..', 'jenkins', 'test_format_converters'),
+                    __dir__)
+            end
+
+            TEST_RESULT_CONVERTERS = Hash[
+                '*.boost.xml' => File.join(test_format_converter_dir, 'boost-test.xsl'),
+                '*.junit.xml' => nil
+            ]
+
+            def initialize(ws, convertions: TEST_RESULT_CONVERTERS)
+                super(ws)
+                @convertions = convertions
+            end
+
+            class ConvertionFailed < RuntimeError; end
+
+            def process(output_dir, *package_names, after: nil)
+                initialize_and_load
+                source_packages, _ = finalize_setup(package_names, recursive: false)
+                source_packages = source_packages.map do |package_name|
+                    ws.manifest.package_definition_by_name(package_name)
+                end
+
+                has_failures = false
+                source_packages.each do |pkg|
+                    utility = pkg.autobuild.test_utility
+                    found_something = false
+                    convertions.each do |glob, xsl|
+                        Dir.glob(File.join(utility.target_dir, glob)) do |input_file|
+                            input_mtime = File.stat(input_file).mtime
+                            if after && input_mtime < after
+                                Autoproj.message "  ignoring #{input_file}, its modification time is #{input_mtime} which is after #{after}"
+                                next
+                            end
+
+                            found_something = true
+                            FileUtils.mkdir_p output_dir
+                            output_file = File.join(output_dir, File.basename(input_file))
+                            begin
+                                if xsl
+                                    xsl_process(input_file, xsl, output_file)
+                                else
+                                    FileUtils.copy_file input_file, output_file
+                                end
+                                Autoproj.message "  generated #{output_file} from #{input_file} for #{pkg.name}"
+                            rescue Exception => e
+                                Autoproj.error e.message
+                                has_failures = true
+                            end
+                        end
+                    end
+
+                    if !found_something
+                        Autoproj.message "found no test results for #{pkg.name}"
+                    end
+                end
+            ensure
+                if has_failures
+                    raise ConvertionFailed, "some files failed to convert, see output for more details"
+                end
+            end
+
+            def xsl_process(input_file, stylesheet, output_file)
+                if File.read(input_file).strip.empty?
+                    return
+                end
+
+                if !system('saxonb-xslt', "-o:#{output_file}", "-xsl:#{stylesheet}", "-s:#{input_file}")
+                    raise ArgumentError, "failed to convert #{input_file} using #{stylesheet}"
+                end
+            end
+        end
+    end
+end
+
+

data/lib/autoproj/jenkins/credentials.rb

@@ -0,0 +1,67 @@
+module Autoproj::Jenkins
+    # Management of the mapping from a VCS description to the corresponding
+    # Jenkins credential
+    class Credentials
+        Credential = Struct.new :vcs, :protocol, :host do
+            # Whether this credential object matches the given VCS
+            #
+            # @param [Autoproj::VCSDefinition] vcs
+            def matches?(vcs)
+                return if vcs.type.to_sym != self.vcs
+                vcs_url = URI.parse(vcs.url)
+                vcs_url.scheme == protocol &&
+                    vcs_url.host == host
+            end
+
+            # The ID of the jenkins credential
+            def jenkins_id
+                "autoproj-#{vcs}-#{protocol}-#{host}"
+            end
+        end
+
+        # Parse a string that represents a single credential and return it
+        #
+        # @param [String] string a string of the form "vcs_type:URI", e.g.
+        #   "git:https://github.com"
+        # @return [Credential]
+        def self.parse(string)
+            vcs, *uri = string.split(':')
+            if !vcs || vcs.empty? || uri.empty?
+                raise ArgumentError, "expected VCS:URL but got #{string}"
+            elsif !Autoproj::Jenkins.vcs_supported?(vcs)
+                raise UnhandledVCS, "#{vcs} is not a supported VCS"
+            end
+            uri = URI.parse(uri.join(':'))
+            Credential.new(vcs.to_sym, uri.scheme, uri.host)
+        end
+
+        attr_reader :credentials
+
+        def initialize
+            @credentials = Hash.new
+        end
+
+        def empty?
+            credentials.empty?
+        end
+
+        def [](vcs)
+            credentials[vcs.to_sym] || Array.new
+        end
+
+        def add(credential)
+            (credentials[credential.vcs] ||= Array.new) << credential
+        end
+
+        # Return the credential description for the given VCS
+        #
+        # @param [Autoproj::VCSDefinition] vcs
+        # @return [nil,Credential]
+        def for(vcs)
+            self[vcs.type].find do |c|
+                c.matches?(vcs)
+            end
+        end
+    end
+end
+

data/lib/autoproj/jenkins/exceptions.rb

@@ -0,0 +1,7 @@
+module Autoproj::Jenkins
+    # Exception raised when trying to handle a package whose VCS we don't
+    # integrate
+    class UnhandledVCS < RuntimeError
+    end
+end
+

data/lib/autoproj/jenkins/relativize.rb

@@ -0,0 +1,74 @@
+module Autoproj
+    module Jenkins
+        # Process files that might contain full paths and transform them by
+        # replacing the original path with a new one
+        class Relativize
+            FILE_PATTERN = Regexp.union(
+                /\/pkgconfig\/.*.pc$/,
+                /\.cmake$/,
+                /\.txt$/
+            )
+
+            # The path of the root to be filtered
+            #
+            # @return [Pathname]
+            attr_reader :root_path
+
+            # The text to be replaced
+            #
+            # @return [String]
+            attr_reader :original_text
+
+            # The text replacing {#original_text}
+            #
+            # @return [String]
+            attr_reader :replacement_text
+
+            # A pattern used to filter which files should be filtered
+            #
+            # @return [#===]
+            attr_reader :file_pattern
+
+            def initialize(root_path, original_text, replacement_text, file_pattern: FILE_PATTERN)
+                @root_path        = root_path
+                @original_text    = original_text
+                @replacement_text = replacement_text
+                @file_pattern = file_pattern
+            end
+
+            # Process all files matching {#file_pattern} within {#root_path}
+            #
+            # @return [Array<Pathname>] the files that have been processed
+            def process
+                processed_files = Array.new
+                root_path.find do |candidate|
+                    if candidate.file? && (file_pattern === candidate.to_s)
+                        if process_file(candidate)
+                            processed_files << candidate
+                        end
+                    end
+                end
+                processed_files
+            end
+
+            # @api private
+            #
+            # Replaces text in a given file
+            #
+            # @return [Boolean] true if the pattern was found
+            def process_file(path)
+                replaced = false
+                filtered = path.each_line.map do |line|
+                    line.gsub(original_text) { replaced = true; replacement_text }
+                end
+                if replaced
+                    path.open('w') do |io|
+                        io.puts filtered.join
+                    end
+                    true
+                end
+            end
+        end
+    end
+end
+