cigale 0.1.0

data/lib/cigale/cli.rb

@@ -0,0 +1,198 @@
+
+require "cigale/macro_context"
+require "cigale/exts"
+require "cigale/generator"
+
+require "logger"
+require "slop"
+require "jenkins_api_client"
+
+require "yaml"
+require "fileutils"
+require "builder"
+
+module Cigale
+  # Command-line interface, need to be exploded further
+  class CLI < Exts
+    include Cigale::Generator
+
+    def logger
+      @logger ||= Logger.new($stdout).tap do |log|
+        log.progname = "cigale"
+      end
+    end
+
+    def initialize (args)
+      @numjobs = 0
+
+      opts = Slop.parse args do |o|
+        o.banner = "Usage: cigale [options] [command] [spec_file.yml]"
+
+        o.string "-o", "output", "Output directory", :default => "."
+        o.bool "--fixture", "fixture", "Enable fixture mode", :default => false
+        o.bool "--masquerade", "masquerade", "Try to pass off as jenkins-job-builder", :default => false
+        o.integer "--blowup", "blowup", "Max macro expansion rounds", :default => 1023
+        o.string "--test-category", "test_category", "Test category"
+        o.string "-l", "loglevel", "Logger level", :default => "INFO"
+      end
+      @opts = opts
+
+      logger.level = Logger.const_get opts[:loglevel]
+
+      if opts.arguments.empty?
+        logger.error "No command given"
+        print_usage!
+        return
+      end
+
+      cmd, input = opts.arguments
+      case cmd
+      when "test", "dump", "update"
+        # all good
+      else
+        logger.error "Unknown command: #{cmd}"
+        exit 1
+      end
+
+      unless input
+        logger.error "No input given"
+        print_usage!
+        exit 1
+      end
+
+      inputs = if File.directory? input
+        Dir.glob(File.join(input, "**", "*.y{a,}ml"))
+      else
+        [input]
+      end
+
+      entries = []
+
+      for input in inputs
+        logger.info "Parsing #{input}"
+        parsed = YAML.load_file(input)
+
+        unless Array === parsed
+          raise "Top-level entity in YAML must be an array" unless opts[:fixture]
+          parsed = [{"job" => parsed}]
+        end
+
+        entries += parsed
+      end
+
+      # sort out macro definitions from actual entries
+      @library = {}
+      @definitions = []
+
+      for entry in entries
+        etype, edef = first_pair(entry)
+
+        case etype
+        when Symbol
+          # ruby's yaml parses `:key: val` as `:key => val` rather
+          # than `":key" => val`, so macro definitions in cigale have symbol keys
+          @library[etype.to_s] = edef
+        else
+          @definitions.push entry
+        end
+      end
+
+      logger.info "Parsed #{@library.size} macros, #{@definitions.size} job definitions."
+
+      output = opts[:output]
+      client = nil
+
+      case cmd
+      when "test", "dump"
+        logger.info "Creating directory #{output}"
+        FileUtils.mkdir_p output
+      when "update"
+        client = get_client!
+        all_jobs = client.job.list_all
+        logger.info "Before update, job list: "
+        all_jobs.each { |x| logger.info "  - #{x}" }
+      end
+
+      for entry in @definitions
+        entry = fully_expand(entry)
+
+        etype, edef = first_pair(entry)
+        if edef["name"].nil?
+          raise "Jobs must have names" unless opts[:fixture]
+          edef["name"] = "fixture"
+          edef["project-type"] ||= "project"
+        end
+
+        job_name = edef["name"]
+        job_path = File.join(output, job_name)
+
+        case etype
+        when "job"
+          case cmd
+          when "dump"
+            File.open(job_path + ".yml", "w") do |f|
+              f.write entry.to_yaml
+            end
+          else
+            xml = ::Builder::XmlMarkup.new(:indent => 2)
+            xml.instruct! :xml, :version => "1.0", :encoding => "utf-8"
+            translate_job xml, edef
+
+            case cmd
+            when "test"
+              File.open(job_path + ".xml", "w") do |f|
+                f.write(xml.target!)
+              end
+            when "update"
+              client.job.create_or_update(job_name, xml.target!)
+            end
+          end
+        else
+          raise "Unknown top-level type: #{etype}"
+        end
+      end
+
+      logger.info "Generated #{@numjobs} jobs."
+    end
+
+    def fully_expand (entry)
+      original = entry
+      iterations = 0
+
+      # TODO detect circular dependencies instead
+      while iterations < @opts[:blowup]
+        ctx = MacroContext.new :library => @library
+        entry = ctx.expand(entry)
+        return entry unless ctx.had_expansions?
+        iterations += 1
+      end
+
+      raise "Blew up while trying to expand #{original.inspect}"
+    end
+
+    def underize (name)
+      name.gsub(/-/, '_')
+    end
+
+    def print_usage!
+      puts "cigale v#{VERSION}"
+      puts "Usage: cigale [test] -o output/directory [spec file or directory]"
+    end
+
+    def get_client!
+      config_path = "secret/cigale.yml"
+      config = {}
+
+      begin
+        config = YAML.load_file(config_path)
+      rescue Errno::ENOENT => e
+        logger.error "Config file missing: #{config_path}"
+        logger.error "Please create one. The `server` section can have any of the fields listed at:"
+        logger.error " http://github.arangamani.net/jenkins_api_client/JenkinsApi/Client.html#initialize-instance_method"
+        exit 1
+      end
+
+      JenkinsApi::Client.new(config["server"])
+    end
+  end
+end

data/lib/cigale/exts.rb

@@ -0,0 +1,86 @@
+
+require "pp" # pretty-print
+
+module Cigale
+  class Exts
+    # Lets you do `dig(hash, "a.b.c")` instead of `hash["a"]["b"]["c"]`
+    def dig(h, dotted_path)
+      parts = dotted_path.split '.', 2
+      match = h[parts[0]]
+      if !parts[1] or match.nil?
+        return match
+      else
+        return dig(match, parts[1])
+      end
+    end
+
+    # Given {:a => b, :c => d, :e => f}, returns [:a, b]
+    def first_pair(h)
+      h.each_pair.next
+    end
+
+    # Given `{"a" => "b"}`, returns ["a", "b"].
+    # Given `"a"`, returns ["a", {}]
+    def asplode(node)
+      type = nil
+      spec = {}
+
+      case node
+      when Hash
+        type, spec = first_pair(node)
+      when String
+        type = node
+      else
+        raise "Invalid markup: #{node.inspect}"
+      end
+
+      return type, spec
+    end
+
+    # Make sure something is an array
+    def toa (input)
+      case input
+      when Array
+        return input
+      when nil
+        return []
+      else
+        return [input]
+      end
+    end
+
+    # Make sure something is a hash
+    def toh (input)
+      case input
+      when Hash
+        return input
+      when nil
+        return {}
+      else
+        raise "Can't toh {input.inspect}"
+      end
+    end
+
+    # Give a default to a boolean parameter
+    # (val || default) doesn't work because val
+    # may be false instead of nil
+    def boolp (val, default)
+      if val.nil?
+        default
+      else
+        val
+      end
+    end
+  end
+end
+
+require "builder/xmlbase"
+
+module Builder
+  class XmlBase
+    # expose private method '_indent' for our 'raw' implementation
+    def indent!
+      _indent
+    end
+  end
+end

data/lib/cigale/generator.rb

@@ -0,0 +1,302 @@
+
+require "cigale/property"
+require "cigale/parameter"
+require "cigale/wrapper"
+require "cigale/scm"
+require "cigale/builder"
+require "cigale/publisher"
+require "cigale/trigger"
+
+include Cigale::Property
+include Cigale::Parameter
+include Cigale::Wrapper
+include Cigale::SCM
+include Cigale::Builder
+include Cigale::Publisher
+include Cigale::Trigger
+
+module Cigale
+  module Generator
+    def translate_job (xml, jdef)
+      @numjobs += 1
+
+      type = jdef["project-type"]
+      testcat = @opts[:test_category]
+
+      project = case type
+      when "matrix"
+        "matrix-project"
+      when "maven"
+        "maven2-moduleset"
+      when "flow"
+        "com.cloudbees.plugins.flow.BuildFlow"
+      when "multijob"
+        "com.tikal.jenkins.plugins.multijob.MultiJobProject"
+      when "externaljob"
+        "hudson.model.ExternalJob"
+      else
+        "project"
+      end
+
+      xml.tag! project do
+        case type
+        when "matrix"
+          translate_matrix_project xml, jdef
+        when "maven"
+          translate_maven_moduleset xml, jdef
+        when "flow"
+          translate_flow_project xml, jdef
+        end # case project
+
+        if testcat.nil? || testcat == "general"
+          xml.actions
+          if testcat.nil?
+            if @opts[:masquerade]
+              # to let tests pass
+              xml.description "<!-- Managed by Jenkins Job Builder -->"
+            else
+              xml.description "<!-- Managed by cigale -->"
+            end
+          end
+          xml.keepDependencies false
+          xml.blockBuildWhenDownstreamBuilding false
+          xml.blockBuildWhenUpstreamBuilding false
+          xml.concurrentBuild false
+          if val = jdef["workspace"]
+            xml.customWorkspace val
+          end
+          if val = jdef["child-workspace"] and type == "matrix"
+            xml.childCustomWorkspace val
+          end
+
+          if node = jdef["node"]
+            xml.assignedNode node
+            xml.canRoam false
+          else
+            xml.canRoam true
+          end
+
+          if retcount = jdef["retry-count"]
+            xml.scmCheckoutRetryCount retcount
+          end
+        end
+
+        if testcat.nil? || testcat == "properties" || testcat == "parameters"
+          translate_properties xml, jdef
+        end
+
+        if testcat.nil? || testcat == "scm"
+          translate_scms xml, jdef["scm"]
+        end
+
+        if testcat.nil? || testcat == "triggers"
+          tok = jdef["token"] and xml.authToken tok
+          translate_triggers xml, jdef["triggers"]
+        end
+
+        if testcat.nil? || testcat == "builders"
+          if post = jdef["postbuilders"]
+            translate_builders xml, "postbuilders", jdef["postbuilders"]
+          end
+          translate_builders xml, "builders", jdef["builders"]
+        end
+
+        if testcat.nil? || testcat == "publishers"
+          translate_publishers xml, "publishers", jdef["publishers"]
+        end
+
+        if testcat.nil? || testcat == "wrappers"
+          translate_wrappers xml, jdef["wrappers"]
+        end
+      end
+    end # translate
+
+    # kind = 'property', 'parameter', 'builder'
+    # type = 'scmpoll', 'github', etc.
+    def translate (kind, xml, type, spec)
+      if type == "raw"
+        return insert_raw xml, spec
+      end
+
+      classes = self.send "#{kind}_classes"
+      clazz = classes[type]
+      raise "Unknown #{kind} type: #{type}" unless clazz
+
+      method = method_for_translate(kind, type)
+
+      case clazz
+      when String
+        xml.tag! clazz do
+          self.send method, xml, spec
+        end
+      when Array
+        xml.tag! *clazz do
+          self.send method, xml, spec
+        end
+      else
+        self.send method, xml, spec
+      end
+    end
+
+    def method_for_translate (kind, type)
+      "translate_#{underize(type)}_#{kind}"
+    end
+
+    def insert_raw (xml, spec)
+      for l in spec["xml"].split("\n")
+        xml.indent!
+        xml << l + "\n"
+      end
+    end
+
+    def translate_matrix_project (xml, jdef)
+      exstrat = jdef["execution-strategy"] || {}
+      xml.executionStrategy :class => "hudson.matrix.DefaultMatrixExecutionStrategyImpl" do
+        xml.runSequentially boolp(exstrat["sequential"], false)
+
+        if touchstone = exstrat["touchstone"]
+          xml.touchStoneCombinationFilter touchstone["expr"]
+          xml.touchStoneResultCondition do
+            translate_build_status xml, touchstone["result"].upcase, false
+          end
+        end
+      end
+
+
+      if combfil = exstrat["combination-filter"]
+        xml.combinationFilter combfil.strip
+      else
+        xml.combinationFilter
+      end
+
+      if axes = jdef["axes"]
+        xml.axes do
+          for a in axes
+            axis = a.values.first
+
+            name = axis["name"]
+
+            clazz = case axis["type"]
+            when "label-expression"
+              "hudson.matrix.LabelExpAxis"
+            when "slave"
+              "hudson.matrix.LabelAxis"
+            when "user-defined"
+              "hudson.matrix.TextAxis"
+            when "jdk"
+              name ||= "jdk"
+              "hudson.matrix.JDKAxis"
+            when "python"
+              name ||= "PYTHON"
+              "jenkins.plugins.shiningpanda.matrix.PythonAxis"
+            when "tox"
+              name ||= "TOXENV"
+              "jenkins.plugins.shiningpanda.matrix.ToxAxis"
+            when "dynamic"
+              "ca.silvermaplesolutions.jenkins.plugins.daxis.DynamicAxis"
+            else
+              raise "Unknown axis type: #{axis["type"]}"
+            end
+
+            xml.tag! clazz do
+              xml.name name
+              xml.values do
+                for v in axis["values"]
+                  xml.string v
+                end
+              end
+
+              case axis["type"]
+              when "dynamic"
+                xml.varName axis["values"].first
+                xml.axisValues do
+                  xml.string "default"
+                end
+              end
+            end
+          end # for a in axes
+        end
+      else
+        xml.axes
+      end
+    end
+
+    def translate_maven_moduleset (xml, jdef)
+      mav = jdef["maven"]
+      return unless mav
+
+      if rootmod = mav["root-module"]
+        xml.rootModule do
+          xml.groupId rootmod["group-id"]
+          xml.artifactId rootmod["artifact-id"]
+        end
+      end
+
+      xml.goals mav["goals"]
+      if mav["private-repository"]
+        xml.localRepository :class => "hudson.maven.local_repo.PerJobLocalRepositoryLocator"
+      end
+      xml.ignoreUpstremChanges true # sic.
+      xml.rootPOM mav["root-pom"]
+
+      if mav["incremental-build"]
+        xml.aggregatorStyleBuild false
+        xml.incrementalBuild true
+      else
+        xml.aggregatorStyleBuild true
+        xml.incrementalBuild false
+      end
+
+      xml.siteArchivingDisabled !boolp(mav["automatic-site-archiving"], true)
+      xml.fingerprintingDisabled false
+      xml.perModuleEmail true
+      xml.archivingDisabled !boolp(mav["automatic-archiving"], true)
+      xml.resolveDependencies !!mav["resolve-dependencies"]
+      xml.processPlugins !!mav["process-plugins"]
+      xml.mavenValidationLevel -1
+      xml.runHeadless !!mav["run-headless"]
+      if mavcw = mav["custom-workspace"]
+        xml.customWorkspace mavcw
+      end
+
+      if mset = mav["settings"]
+        if mset.start_with? "org.jenkinsci"
+          xml.settings :class => "org.jenkinsci.plugins.configfiles.maven.job.MvnSettingsProvider" do
+            xml.settingsConfigId mset
+          end
+        else
+          xml.settings :class => "jenkins.mvn.FilePathSettingsProvider" do
+            xml.path mset
+          end
+        end
+      else
+        xml.settings :class => "jenkins.mvn.DefaultSettingsProvider"
+      end
+
+      if mgset = mav["global-settings"]
+        if mgset.start_with? "org.jenkinsci"
+          xml.globalSettings :class => "org.jenkinsci.plugins.configfiles.maven.job.MvnGlobalSettingsProvider" do
+            xml.settingsConfigId mgset
+          end
+        else
+          xml.globalSettings :class => "jenkins.mvn.FilePathGlobalSettingsProvider" do
+            xml.path mgset
+          end
+        end
+      else
+        xml.globalSettings :class => "jenkins.mvn.DefaultGlobalSettingsProvider"
+      end
+
+      xml.runPostStepsIfResult do
+        status = mav["post-step-run-condition"] || "FAILURE"
+        translate_build_status xml, status, false
+      end
+    end
+
+    def translate_flow_project (xml, jdef)
+      xml.dsl
+      xml.buildNeedsWorkspace false
+    end
+
+  end # Generator
+end # Cigale

data/lib/cigale/macro_context.rb

@@ -0,0 +1,103 @@
+
+require "cigale/exts"
+
+module Cigale
+  class MacroContext < Exts
+    def initialize (options)
+      @library = options[:library]
+      @params = options[:params]
+      @had_expansions = false
+    end
+
+    # get a macro
+    def lookup (macro_name)
+      mdef = @library[macro_name]
+      raise "Undefined macro: #{macro_name} — got #{@library.keys.inspect}" unless mdef
+      mdef
+    end
+
+    # walk tree, expand macros we find
+    def expand (entity)
+      case entity
+      when Array
+        # list of things, need to expand each individually
+        entity.map do |x|
+          expand(x)
+        end
+      when Hash
+        entity.map do |k, v|
+          case k
+          when /^\.(.*)$/
+            # macro call
+            if expanding?
+              # keep child macro as-is for later expansion
+              [k, expand(v)]
+            else
+              mdef = lookup($1)
+
+              res = self.with_params(v).expand(mdef)
+              case res
+              when Hash
+                first_pair(res)
+              when String, Array
+                return res
+              else
+                raise "Invalid macro expansion result: #{res.inspect}"
+              end
+            end
+          else
+            [k, expand(v)]
+          end
+        end.to_h
+      else
+        # not a list, not a hash
+        interpolate(entity)
+      end
+    end
+
+    # given "{name}"
+    def interpolate (entity)
+      return entity unless expanding?
+
+      case entity
+      when /^{([a-zA-Z0-9*\. -]*)}$/
+        # just paste param verbatim — could be a string,
+        # could be a hash, a list, whatever, just jam it in there.
+        get_param($1)
+      when String
+        entity.gsub /{([a-zA-Z0-9*\. -]*)}/ do |m|
+          get_param($1)
+        end
+      else
+        entity
+      end
+    end
+
+    def get_param (spec)
+      raise "Found param {#{spec}} outside expansion" unless expanding?
+
+      name, type = spec.split(" ")
+      if name == '*'
+        return @params
+      end
+
+      res = dig(@params, name)
+      raise "Unspecified param {#{name}}" if res.nil?
+      res
+    end
+
+    def with_params (params)
+      raise "Can't expand while expanding" if expanding?
+      @had_expansions = true
+      MacroContext.new(:library => @library, :params => params)
+    end
+
+    def had_expansions?
+      @had_expansions
+    end
+
+    def expanding?
+      not @params.nil?
+    end
+  end
+end

data/lib/cigale/parameter/bool.rb

@@ -0,0 +1,12 @@
+module Cigale::Parameter
+  def translate_bool_parameter (xml, pdef)
+    xml.name pdef["name"]
+    xml.description pdef["description"]
+    default = pdef["default"]
+    if default.nil? || default == ''
+      xml.defaultValue
+    else
+      xml.defaultValue default
+    end
+  end
+end