vanagon 0.9.3 → 0.10.0

data/lib/vanagon/project/dsl.rb

@@ -24,6 +24,7 @@ class Vanagon
       # @param block [Proc] DSL definition of the project to call
       def project(name, &block)
         yield(self)
+        environment 'VANAGON_PROJECT', @name
       end
 
       # Accessor for the project.
@@ -81,6 +82,9 @@ class Vanagon
       # @param the_name [String] name of the project
       def name(the_name)
         @project.name = the_name
+        # Overwrite the environment variable name using the reset name
+        # of the project.
+        environment 'VANAGON_PROJECT_NAME', @project.name
       end
 
       # Sets the homepage for the project. Mainly for use in packaging.
@@ -95,6 +99,7 @@ class Vanagon
       # @param page [Integer] timeout in seconds
       def timeout(to)
         @project.timeout = to
+        environment 'VANAGON_PROJECT_TIMEOUT', @project.timeout
       end
 
       # Sets the run time requirements for the project. Mainly for use in packaging.
@@ -135,6 +140,7 @@ class Vanagon
       # @param ver [String] version of the project
       def version(ver)
         @project.version = ver.tr('-', '.')
+        environment 'VANAGON_PROJECT_VERSION', @project.version
       end
 
       # Sets the release for the project. Mainly for use in packaging.
@@ -142,6 +148,7 @@ class Vanagon
       # @param rel [String] release of the project
       def release(rel)
         @project.release = rel
+        environment 'VANAGON_PROJECT_RELEASE', @project.release
       end
 
       # Sets the version for the project based on a git describe of the
@@ -149,8 +156,8 @@ class Vanagon
       # and reachable from the current commit in that repository.
       #
       def version_from_git
-        version = Git.open(File.expand_path("..", Vanagon::Driver.configdir)).describe('HEAD', tags: true)
-        @project.version = version.split('-').reject(&:empty?).join('.')
+        git_version = Git.open(File.expand_path("..", Vanagon::Driver.configdir)).describe('HEAD', tags: true)
+        version(git_version.split('-').reject(&:empty?).join('.'))
       rescue Git::GitExecuteError
         warn "Directory '#{dirname}' cannot be versioned by git. Maybe it hasn't been tagged yet?"
       end
@@ -160,6 +167,7 @@ class Vanagon
       # @param vend [String] vendor or author of the project
       def vendor(vend)
         @project.vendor = vend
+        environment 'VANAGON_PROJECT_VENDOR', @project.vendor
       end
 
       # Adds a directory to the list of directories provided by the project, to be included in any packages of the project
@@ -172,6 +180,12 @@ class Vanagon
         @project.directories << Vanagon::Common::Pathname.new(dir, mode: mode, owner: owner, group: group)
       end
 
+      # Adds an arbitrary environment variable to the project, which will be passed
+      # on to the platform and inherited by any components built on that platform
+      def environment(name, value)
+        @project.environment[name] = value
+      end
+
       # Add a user to the project
       #
       # @param name [String] name of the user to create
@@ -248,6 +262,7 @@ class Vanagon
       # Counter for the number of times a project should retry a task
       def retry_count(retry_count)
         @project.retry_count = retry_count
+        environment 'VANAGON_PROJECT_RETRY_COUNT', @project.retry_count
       end
     end
   end

data/lib/vanagon/utilities.rb

@@ -264,6 +264,8 @@ class Vanagon
       template = File.read(erbfile)
       message  = ERB.new(template, nil, "-")
       message.result(b)
+        .gsub(/[\n]+{3,}/, "\n\n")
+        .squeeze("\s")
     end
 
     # Helper method that takes a template and writes the evaluated contents to a file on disk

data/resources/Makefile.erb

@@ -1,11 +1,15 @@
-<% dirnames = get_directories.map {|d| d.path } %>
-export SHELL := $(shell which bash)
+SHELL = $(PWD)/profiling_shell.sh
+
+<%- merged_environment.to_a(" := ").each do |var| -%>
+export <%= var %>
+<%- end -%>
 
 tempdir := $(shell mktemp -d -p /var/tmp 2>/dev/null || mktemp -d -t 'tmp')
 workdir := $(PWD)
 
 all: file-list-before-build <%= package_name %>
 
+<%= package_name %>: export VANAGON_TARGET := create-package
 <%= package_name %>: <%= @name %>-<%= @version %>.tar.gz
 	<%= generate_package.join("\n\t") %>
 
@@ -13,14 +17,14 @@ file-list-before-build:
 <%- if dirnames.empty? -%>
 	touch file-list-before-build
 <%- else -%>
-	(<%= @platform.find %> -H "<%= dirnames.join('" "') %>" 2>/dev/null || <%= @platform.find %> "<%= dirnames.join('" "') %>" 2>/dev/null) | <%= "xargs -I[] cygpath --mixed --long-name --absolute [] |" if @platform.is_windows? %> <%= @platform.sort %> | uniq > file-list-before-build
+	(<%= @platform.find %> -L "<%= dirnames.join('" "') %>" 2>/dev/null || <%= @platform.find %> "<%= dirnames.join('" "') %>" -follow 2>/dev/null) | <%= "xargs -I[] cygpath --mixed --long-name --absolute [] |" if @platform.is_windows? %> <%= @platform.sort %> | uniq > file-list-before-build
 <%- end -%>
 
 file-list-after-build: <%= @components.map {|comp| comp.name }.join(" ") %>
 <%- if dirnames.empty? -%>
 	touch file-list-after-build
 <%- else -%>
-	(<%= @platform.find %> -H "<%= dirnames.join('" "') %>" 2>/dev/null || <%= @platform.find %> "<%= dirnames.join('" "') %>" 2>/dev/null) | <%= "xargs -I[] cygpath --mixed --long-name --absolute [] |" if @platform.is_windows? %> <%= @platform.sort %> | uniq > file-list-after-build
+	(<%= @platform.find %> -L "<%= dirnames.join('" "') %>" 2>/dev/null || <%= @platform.find %> "<%= dirnames.join('" "') %>" -follow 2>/dev/null) | <%= "xargs -I[] cygpath --mixed --long-name --absolute [] |" if @platform.is_windows? %> <%= @platform.sort %> | uniq > file-list-after-build
 <%- end -%>
 
 <%= @name %>-<%= @version %>.tar.gz: file-list <%= @cleanup ? 'cleanup-components' : '' %>

data/resources/metrics/profiling_shell.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# this variant of the Profiling shell wrapper
+# uses only the Bash keyword `time`, and the generally
+# standard command `tee` to get run time. This was
+# needed because the BSD/sysv variants of `date` only
+# offer second-resolution. No milliseconds, no nano-
+# seconds.
+
+# Create copies of stdout & stderr's file descriptors
+# so that the output of the subshell that `time` runs
+# can be captured and used.
+exec 3>&1
+exec 4>&2
+
+# Write some log files, that can be post-processed
+# relatively easily.
+__stdout="${VANAGON_TARGET}.stdout.log"
+__stderr="${VANAGON_TARGET}.stderr.log"
+
+# Define a default time format (seconds and milliseconds),
+# and then use some insane output redirection hacks to capture
+# the output of the `time` keyword. We're using `tee` to
+# output to stdout & stderr but we should figure out how to
+# supress that for quieter Vanagon builds.
+TIMEFORMAT='%3R'
+__seconds="$( {
+  time bash -o pipefail "${@}" > >(tee -ai "${__stdout}" >&3) 2> >(tee -ai "${__stderr}" >&4 )
+} 2>&1)"
+
+# If any part of the pipeline failed, then
+# this status should correspond to whatever
+# failing status the pipeline returned. We
+# want to check & preserve that.
+if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
+  exit "${PIPESTATUS[0]}"
+fi
+
+# Finally, if everything completed successfully,
+# finesse the numbers and fire them off to our
+# statsd interface.
+if [[ "${VANAGON_TARGET}" ]] && [[ "${VANAGON_STATSD_HOST}" ]] && [[ "${VANAGON_STATSD_PORT}" ]]; then
+  __elapsed="$(echo "scale=0;${__seconds}*1000/1" | bc -l)"
+  echo -n "vanagon.${VANAGON_PROJECT}.${VANAGON_PLATFORM}.${VANAGON_TARGET}:${__elapsed}|ms" > "/dev/udp/${VANAGON_STATSD_HOST}/${VANAGON_STATSD_PORT}"
+fi

data/resources/rpm/project.spec.erb

@@ -142,7 +142,7 @@ install -d %{buildroot}
 
 # Here we turn all dirs in the file-list into %dir entries to avoid duplicate files
 for entry in `cat %{SOURCE1}`; do
-  if [ -n "$entry" -a -d "$entry" -a ! -L "$entry" ]; then
+  if [ -n "$entry" -a -d "$entry" ]; then
     PATH=/opt/freeware/bin:$PATH sed -i "s|^\($entry\)\$|%dir \1|g" %{SOURCE1}
   fi
 done

data/spec/lib/makefile_spec.rb

@@ -5,7 +5,7 @@ describe Makefile::Rule do
     subject { described_class.new("empty") }
 
     it "creates an empty rule" do
-      expect(subject.format).to eq "empty:\n"
+      expect(subject.format).to eq "empty: export VANAGON_TARGET := empty\nempty:\n"
     end
   end
 
@@ -13,7 +13,7 @@ describe Makefile::Rule do
     subject { described_class.new("simple", recipe: ["touch simple"]) }
 
     it "creates the rule with the recipe" do
-      expect(subject.format).to eq "simple:\n\ttouch simple\n"
+      expect(subject.format).to eq "simple: export VANAGON_TARGET := simple\nsimple:\n\ttouch simple\n"
     end
   end
 
@@ -21,7 +21,7 @@ describe Makefile::Rule do
     subject { described_class.new("depends", dependencies: ["mydeps"]) }
 
     it "creates the rule with the recipe" do
-      expect(subject.format).to eq "depends: mydeps\n"
+      expect(subject.format).to eq "depends: export VANAGON_TARGET := depends\ndepends: mydeps\n"
     end
   end
 
@@ -29,7 +29,7 @@ describe Makefile::Rule do
     subject { described_class.new("deluxe", recipe: ["touch deluxe"], dependencies: ["mydeps"]) }
 
     it "creates the rule with the recipe" do
-      expect(subject.format).to eq "deluxe: mydeps\n\ttouch deluxe\n"
+      expect(subject.format).to eq "deluxe: export VANAGON_TARGET := deluxe\ndeluxe: mydeps\n\ttouch deluxe\n"
     end
   end
 
@@ -44,7 +44,7 @@ describe Makefile::Rule do
     end
 
     it "inserts tabs after each newline in the recipe" do
-      expect(subject.format).to eq "multiline:\n\t[ -d build ] || mkdir -p build\n\tcd build &&\n\tcmake .. &&\n\tmake &&\n\tmake install\n"
+      expect(subject.format).to eq "multiline: export VANAGON_TARGET := multiline\nmultiline:\n\t[ -d build ] || mkdir -p build\n\tcd build &&\n\tcmake .. &&\n\tmake &&\n\tmake install\n"
     end
   end
 end

data/spec/lib/vanagon/component/dsl_spec.rb

@@ -587,14 +587,7 @@ end" }
         it 'adds the file to the configfiles list' do
           comp = Vanagon::Component::DSL.new('install-config-file-test', {}, platform)
           comp.install_configfile('thing1', 'place/to/put/thing1')
-          expect(comp._component.configfiles).to include(Vanagon::Common::Pathname.configfile('place/to/put/thing1', mode: '0644'))
-          expect(comp._component.files).not_to include(Vanagon::Common::Pathname.file('place/to/put/thing1'))
-        end
-
-        it 'sets owner, group, and mode for the configfiles' do
-          comp = Vanagon::Component::DSL.new('install-config-file-test', {}, platform)
-          comp.install_configfile('thing1', 'place/to/put/thing1', owner: 'bob', group: 'timmy', mode: '0022')
-          expect(comp._component.configfiles).to include(Vanagon::Common::Pathname.configfile('place/to/put/thing1', mode: '0022', group: 'timmy', owner: 'bob'))
+          expect(comp._component.configfiles).to include(Vanagon::Common::Pathname.configfile('place/to/put/thing1'))
           expect(comp._component.files).not_to include(Vanagon::Common::Pathname.file('place/to/put/thing1'))
         end
       end
@@ -624,7 +617,7 @@ end" }
         it 'adds the file to the configfiles list' do
           comp = Vanagon::Component::DSL.new('install-config-file-test', {}, platform)
           comp.install_configfile('thing1', 'place/to/put/thing1')
-          expect(comp._component.configfiles).to include(Vanagon::Common::Pathname.configfile('place/to/put/thing1.pristine', mode: '0644'))
+          expect(comp._component.configfiles).to include(Vanagon::Common::Pathname.configfile('place/to/put/thing1.pristine'))
           expect(comp._component.configfiles).not_to include(Vanagon::Common::Pathname.file('place/to/put/thing1'))
         end
       end
@@ -656,18 +649,40 @@ end" }
   end
 
   describe '#environment' do
+    before :each do
+      @comp = Vanagon::Component::DSL.new('env-test', {}, {})
+    end
+
+    before :example do
+      @path = {'PATH' => '/usr/local/bin'}
+      @alternate_path = {'PATH' => '/usr/bin'}
+      @cflags = {'CFLAGS' => '-I /usr/local/bin'}
+      @merged_env = @cflags.merge(@alternate_path)
+    end
+
     it 'adds an override to the environment for a component' do
-      comp = Vanagon::Component::DSL.new('env-test', {}, {})
-      comp.environment({'PATH' => '/usr/local/bin'})
-      expect(comp._component.environment).to eq({'PATH' => '/usr/local/bin'})
+      @comp.environment(@path)
+      @path.each_pair do |key, value|
+        expect(@comp._component.environment[key]).to eq(value)
+      end
     end
 
     it 'merges against the existing environment' do
-      comp = Vanagon::Component::DSL.new('env-test', {}, {})
-      comp.environment({'PATH' => '/usr/local/bin'})
-      comp.environment({'PATH' => '/usr/bin'})
-      comp.environment({'CFLAGS' => '-I /usr/local/bin'})
-      expect(comp._component.environment).to eq({'PATH' => '/usr/bin', 'CFLAGS' => '-I /usr/local/bin'})
+      # Set a value that should *NOT* be present
+      @comp.environment(@path)
+      # And then set two values that should
+      @comp.environment(@alternate_path)
+      @comp.environment(@cflags)
+
+      # Test that our canary doesn't exist
+      @path.each_pair do |key, value|
+        expect(@comp._component.environment[key]).to_not eq(value)
+      end
+
+      # And then validate our expected values
+      @merged_env.each_pair do |key, value|
+        expect(@comp._component.environment[key]).to eq(value)
+      end
     end
   end
 

data/spec/lib/vanagon/component/rules_spec.rb

@@ -43,17 +43,14 @@ describe Vanagon::Component::Rules do
 
     it "extracts the source" do
       component.extract_with = "/usr/bin/tar"
-      expect(rule.recipe.first).to eq ": && /usr/bin/tar"
+      expect(rule.recipe.first).to eq "/usr/bin/tar"
     end
 
     it "sets environment variables before running the unpack steps" do
       component.extract_with = "/usr/bin/tar"
       component.environment = {"PATH" => "/opt/pl-build-tools/bin:$$PATH"}
       expect(rule.recipe.first).to eq(
-        [
-          "export PATH=\"/opt/pl-build-tools/bin:$$PATH\"",
-          "/usr/bin/tar"
-        ].join(" && ")
+        [ "/usr/bin/tar" ].join(" && ")
       )
     end
 
@@ -108,7 +105,6 @@ describe Vanagon::Component::Rules do
       expect(rule.recipe[1]).to eq(
         [
           "cd /foo/bar",
-          ":",
           "./configure",
           "cmake .."
         ].join(" && \\\n")
@@ -121,7 +117,6 @@ describe Vanagon::Component::Rules do
       expect(rule.recipe[1]).to eq(
         [
           "cd /foo/bar",
-          "export PATH=\"/opt/pl-build-tools/bin:$$PATH\"",
           "./configure",
           "cmake .."
         ].join(" && \\\n")
@@ -145,7 +140,6 @@ describe Vanagon::Component::Rules do
       expect(rule.recipe.first).to eq(
         [
           "cd /foo/bar",
-          ":",
           "make",
           "make test",
         ].join(" && \\\n")
@@ -158,7 +152,6 @@ describe Vanagon::Component::Rules do
       expect(rule.recipe.first).to eq(
         [
           "cd /foo/bar",
-          "export PATH=\"/opt/pl-build-tools/bin:$$PATH\"",
           "make",
           "make test"
         ].join(" && \\\n")
@@ -188,7 +181,6 @@ describe Vanagon::Component::Rules do
       expect(rule.recipe.first).to eq(
         [
           "cd /foo/bar",
-          ":",
           "make cpplint",
           "make test",
         ].join(" && \\\n")
@@ -201,7 +193,6 @@ describe Vanagon::Component::Rules do
       expect(rule.recipe.first).to eq(
         [
           "cd /foo/bar",
-          "export PATH=\"/opt/pl-build-tools/bin:$$PATH\"",
           "make cpplint",
           "make test"
         ].join(" && \\\n")
@@ -225,7 +216,6 @@ describe Vanagon::Component::Rules do
       expect(rule.recipe.first).to eq(
         [
           "cd /foo/bar",
-          ":",
           "make install",
           "make reallyinstall",
         ].join(" && \\\n")
@@ -238,7 +228,6 @@ describe Vanagon::Component::Rules do
       expect(rule.recipe.first).to eq(
         [
           "cd /foo/bar",
-          "export PATH=\"/opt/pl-build-tools/bin:$$PATH\"",
           "make install",
           "make reallyinstall"
         ].join(" && \\\n")

data/spec/lib/vanagon/component_spec.rb

@@ -2,38 +2,63 @@ require 'vanagon/component'
 
 describe "Vanagon::Component" do
   describe "#get_environment" do
+    subject { Vanagon::Component.new('env-test', {}, {}) }
+
+    it "prints a deprecation warning to STDERR" do    
+      expect { subject.get_environment }.to output(/deprecated/).to_stderr
+    end
+
     it "returns a makefile compatible environment" do
-      comp = Vanagon::Component.new('env-test', {}, {})
-      comp.environment = {'PATH' => '/usr/local/bin'}
-      expect(comp.get_environment).to eq('export PATH="/usr/local/bin"')
+      subject.environment = {'PATH' => '/usr/local/bin'}
+      expect(subject.get_environment).to eq %(export PATH="/usr/local/bin")
     end
 
     it 'merges against the existing environment' do
-      comp = Vanagon::Component.new('env-test', {}, {})
-      comp.environment = {'PATH' => '/usr/bin', 'CFLAGS' => '-I /usr/local/bin'}
-      expect(comp.get_environment).to eq('export PATH="/usr/bin" CFLAGS="-I /usr/local/bin"')
+      subject.environment = {'PATH' => '/usr/bin', 'CFLAGS' => '-I /usr/local/bin'}
+      expect(subject.get_environment).to eq %(export PATH="/usr/bin" CFLAGS="-I /usr/local/bin")
     end
 
     it 'returns : for an empty environment' do
-      comp = Vanagon::Component.new('env-test', {}, {})
-      expect(comp.get_environment).to eq(':')
+      expect(subject.get_environment).to eq %(: no environment variables defined)
     end
   end
 
   describe "#get_build_dir" do
-    subject(:comp) do
+    subject do
       Vanagon::Component.new('build-dir-test', {}, {}).tap do |comp|
         comp.dirname = "build-dir-test"
       end
     end
 
     it "uses the dirname when no build_dir was set" do
-      expect(comp.get_build_dir).to eq "build-dir-test"
+      expect(subject.get_build_dir).to eq "build-dir-test"
     end
 
     it "joins the dirname and the build dir when a build_dir was set" do
-      comp.build_dir = "cmake-build"
-      expect(comp.get_build_dir).to eq File.join("build-dir-test", "cmake-build")
+      subject.build_dir = "cmake-build"
+      expect(subject.get_build_dir).to eq File.join("build-dir-test", "cmake-build")
+    end
+  end
+
+  describe "#get_sources" do
+    before :each do
+      @workdir = Dir.mktmpdir
+      @file_name = 'fake_file.txt'
+      @fake_file = "file://spec/fixtures/files/#{@file_name}"
+    end
+
+    subject do
+      # Initialize a new instance of Vanagon::Component and define a
+      # new secondary source. We can now reason about this instance and
+      # test behavior for retrieving secondary sources.
+      Vanagon::Component.new('build-dir-test', {}, {}).tap do |comp|
+        comp.sources << OpenStruct.new(url: @fake_file)
+      end
+    end
+
+    it "copies secondary sources into the workdir" do
+      subject.get_sources(@workdir)
+      expect(File.exist?(File.join(@workdir, @file_name))).to be true
     end
   end
 end