omnibus 5.3.0 → 5.4.0

data/lib/omnibus/project.rb

@@ -16,7 +16,7 @@
 #
 
 require 'time'
-require 'json'
+require 'ffi_yajl'
 require 'omnibus/manifest'
 require 'omnibus/manifest_entry'
 require 'omnibus/reports'
@@ -1087,7 +1087,7 @@ module Omnibus
 
     def write_json_manifest
       File.open(json_manifest_path, 'w') do |f|
-        f.write(JSON.pretty_generate(built_manifest.to_hash))
+        f.write(FFI_Yajl::Encoder.encode(built_manifest.to_hash, pretty: true))
       end
     end
 
@@ -1206,7 +1206,7 @@ module Omnibus
 
         update_with_string(digest, name)
         update_with_string(digest, install_dir)
-        update_with_string(digest, JSON.fast_generate(overrides))
+        update_with_string(digest, FFI_Yajl::Encoder.encode(overrides))
 
         if filepath && File.exist?(filepath)
           update_with_file_contents(digest, filepath)

data/lib/omnibus/publisher.rb

@@ -16,6 +16,8 @@
 
 module Omnibus
   class Publisher
+    include Digestable
+
     class << self
       #
       # Shortcut class method for creating a new instance of this class and

data/lib/omnibus/publishers/artifactory_publisher.rb

@@ -28,17 +28,28 @@ module Omnibus
         log.debug(log_key) { "Validating '#{package.name}'" }
         package.validate!
 
-        # Upload the actual package
-        log.info(log_key) { "Uploading '#{package.name}'" }
-
         retries = Config.publish_retries
 
         begin
           upload_time = Benchmark.realtime do
+            remote_path = remote_path_for(package)
+            properties  = default_properties.merge(metadata_properties_for(package))
+
+            # Upload the package
+            log.info(log_key) { "Uploading '#{package.name}'" }
             artifact_for(package).upload(
               repository,
-              remote_path_for(package),
-              default_properties.merge(metadata_properties_for(package)),
+              remote_path,
+              properties,
+            )
+            # Upload the package's assoacited `*.metadata.json` file
+            log.info(log_key) { "Uploading '#{package.name}.metadata.json'" }
+            artifact_for(package.metadata).upload(
+              repository,
+              "#{remote_path}.metadata.json",
+              # *.metadata.json files should not include
+              # the checksum properties
+              properties.dup.delete_if { |k,v| k =~ /^omnibus\.(md5|sha)/ },
             )
           end
         rescue Artifactory::Error::HTTPError => e
@@ -71,18 +82,21 @@ module Omnibus
     #
     # The artifact object that corresponds to this package.
     #
-    # @param [Package] package
-    #   the package to create the artifact from
+    # @param [Package,Metadata] artifact
+    #   the package or metadata file to create the artifact from
     #
     # @return [Artifactory::Resource::Artifact]
     #
-    def artifact_for(package)
+    def artifact_for(artifact)
+      md5  = artifact.respond_to?(:metadata) ? artifact.metadata[:md5] : digest(artifact.path, :md5)
+      sha1 = artifact.respond_to?(:metadata) ? artifact.metadata[:sha1] : digest(artifact.path, :sha1)
+
       Artifactory::Resource::Artifact.new(
-        local_path: package.path,
+        local_path: artifact.path,
         client:     client,
         checksums: {
-          'md5'  => package.metadata[:md5],
-          'sha1' => package.metadata[:sha1],
+          'md5'  => md5,
+          'sha1' => sha1,
         }
       )
     end
@@ -96,13 +110,20 @@ module Omnibus
     # @return [Artifactory::Resource::Build]
     #
     def build_for(packages)
-      name = packages.first.metadata[:name]
-      # Attempt to load the version-manifest.json file which represents
-      # the build.
-      manifest = if File.exist?(version_manifest)
-                   Manifest.from_file(version_manifest)
+      metadata = packages.first.metadata
+      name     = metadata[:name]
+
+      # Attempt to load the version manifest data from the packages metadata
+      manifest = if version_manifest = metadata[:version_manifest]
+                   Manifest.from_hash(version_manifest)
                  else
-                   Manifest.new(packages.first.metadata[:version])
+                   Manifest.new(
+                     metadata[:version],
+                     # we already know the `version_manifest` entry is
+                     # missing so we can't pull in the `build_git_revision`
+                     nil,
+                     metadata[:license],
+                   )
                  end
 
       # Upload the actual package
@@ -118,9 +139,10 @@ module Omnibus
           version: Omnibus::VERSION,
         },
         properties: default_properties.merge(
-          'omnibus.project' => name,
-          'omnibus.version' => manifest.build_version,
-          'omnibus.version_manifest' => manifest.to_json,
+          'omnibus.project'            => name,
+          'omnibus.version'            => manifest.build_version,
+          'omnibus.build_git_revision' => manifest.build_git_revision,
+          'omnibus.license'            => manifest.license,
         ),
         modules: [
           {
@@ -131,13 +153,21 @@ module Omnibus
               manifest.build_version,
             ].join(':'),
             artifacts: packages.map do |package|
-              {
-                type: File.extname(package.path).split('.').last,
-                sha1: package.metadata[:sha1],
-                md5:  package.metadata[:md5],
-                name: package.metadata[:basename],
-              }
-            end
+              [
+                {
+                  type: File.extname(package.path).split('.').last,
+                  sha1: package.metadata[:sha1],
+                  md5:  package.metadata[:md5],
+                  name: package.metadata[:basename],
+                },
+                {
+                  type: File.extname(package.metadata.path).split('.').last,
+                  sha1: digest(package.metadata.path, :sha1),
+                  md5:  digest(package.metadata.path, :md5),
+                  name: File.basename(package.metadata.path),
+                }
+              ]
+            end.flatten
           }
         ]
       )
@@ -193,15 +223,18 @@ module Omnibus
         'omnibus.architecture'     => package.metadata[:arch],
         'omnibus.version'          => package.metadata[:version],
         'omnibus.iteration'        => package.metadata[:iteration],
+        'omnibus.license'          => package.metadata[:license],
         'omnibus.md5'              => package.metadata[:md5],
         'omnibus.sha1'             => package.metadata[:sha1],
         'omnibus.sha256'           => package.metadata[:sha256],
-        'omnibus.sha512'           => package.metadata[:sha512],
-      }
-      metadata.merge!(
-        'build.name'   => package.metadata[:name],
-        'build.number' => package.metadata[:version],
-      ) if build_record?
+        'omnibus.sha512'           => package.metadata[:sha512]
+      }.tap do |h|
+        if build_record?
+          h['build.name'] = package.metadata[:name]
+          h['build.number'] = package.metadata[:version]
+        end
+      end
+
       metadata
     end
 
@@ -223,15 +256,6 @@ module Omnibus
       @options[:repository]
     end
 
-    #
-    # The path to the builds version-manfest.json file (as supplied as an option).
-    #
-    # @return [String]
-    #
-    def version_manifest
-      @options[:version_manifest] || ''
-    end
-
     #
     # The path where the package will live inside of the Artifactory repository.
     # This is dynamically computed from the values in the project definition

data/lib/omnibus/publishers/s3_publisher.rb

@@ -30,7 +30,7 @@ module Omnibus
 
         # Upload the metadata first
         log.debug(log_key) { "Uploading '#{package.metadata.name}'" }
-        store_object(key_for(package, package.metadata.name), package.metadata.to_json,
+        store_object(key_for(package, package.metadata.name), FFI_Yajl::Encoder.encode(package.metadata.to_hash, pretty: true),
                      nil, access_policy)
 
         # Upload the actual package

data/lib/omnibus/software.rb

@@ -1059,7 +1059,7 @@ module Omnibus
         update_with_string(digest, builder.shasum)
         update_with_string(digest, name)
         update_with_string(digest, version_for_cache)
-        update_with_string(digest, JSON.fast_generate(overrides))
+        update_with_string(digest, FFI_Yajl::Encoder.encode(overrides))
 
         if filepath && File.exist?(filepath)
           update_with_file_contents(digest, filepath)

data/lib/omnibus/version.rb

@@ -15,5 +15,5 @@
 #
 
 module Omnibus
-  VERSION = '5.3.0'
+  VERSION = '5.4.0'
 end

data/omnibus.gemspec

@@ -32,12 +32,14 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'ruby-progressbar', '~> 1.7'
   gem.add_dependency 'aws-sdk',          '~> 2'
   gem.add_dependency 'thor',             '~> 0.18'
+  gem.add_dependency "ffi-yajl",         '~> 2.2'
 
   gem.add_development_dependency 'bundler'
   gem.add_development_dependency 'artifactory', '~> 2.0'
   gem.add_development_dependency 'aruba',       '~> 0.5'
   gem.add_development_dependency 'fauxhai',     '~> 3.2'
   gem.add_development_dependency 'rspec',       '~> 3.0'
+  gem.add_development_dependency 'rspec-json_expectations'
   gem.add_development_dependency 'rspec-its'
   gem.add_development_dependency 'webmock'
   gem.add_development_dependency 'rake'

data/resources/ips/doc-transform.erb

@@ -0,0 +1,2 @@
+<transform dir path=<%= pathdir %>$ -> edit group bin sys>
+<transform file depend -> edit pkg.debug.depend.file ruby env>

data/resources/ips/gen.manifestfile.erb

@@ -0,0 +1,7 @@
+set name=pkg.fmri value=developer/versioning/<%= fmri_package_name %>
+set name=pkg.description value="<%= description %>"
+set name=pkg.summary value="<%= summary %>"
+set name=variant.arch value=<%= arch %>
+set name=info.classification \
+    value="org.opensolaris.category.2008:Meta Packages/Group Packages" \
+    value="org.opensolaris.category.2008:Applications/System Utilities"

data/spec/spec_helper.rb

@@ -1,5 +1,6 @@
 require 'rspec'
 require 'rspec/its'
+require "rspec/json_expectations"
 require 'webmock/rspec'
 
 require 'cleanroom/rspec'

data/spec/unit/manifest_spec.rb

@@ -21,7 +21,7 @@ module Omnibus
       it "returns a ManifestEntry for the requested software" do
         me = ManifestEntry.new("foobar", {})
         subject.add("foobar", me)
-        expect(subject.entry_for("foobar")).to eq(me)
+        expect(subject.entry_for(:foobar)).to eq(me)
       end
 
       it "raises an error if no such manifest entry exists" do
@@ -45,55 +45,54 @@ module Omnibus
         second = ManifestEntry.new("wombat", {})
         subject.add("foobar", first)
         subject.add("wombat", second)
-        expect(subject.entry_names).to eq(["foobar", "wombat"])
+        expect(subject.entry_names).to eq([:foobar, :wombat])
       end
     end
 
     describe "#to_hash" do
       it "returns a Hash containg the current manifest format" do
-        expect(subject.to_hash['manifest_format']).to eq(Manifest::LATEST_MANIFEST_FORMAT)
+        expect(subject.to_hash[:manifest_format]).to eq(Manifest::LATEST_MANIFEST_FORMAT)
       end
 
       it "includes entries for software in the manifest" do
         subject.add("foobar", ManifestEntry.new("foobar", {}))
-        expect(subject.to_hash['software']).to have_key("foobar")
+        expect(subject.to_hash[:software]).to have_key(:foobar)
       end
 
       it "converts the manifest entries to hashes" do
         subject.add("foobar", ManifestEntry.new("foobar", {}))
-        expect(subject.to_hash['software']['foobar']).to be_a(Hash)
+        expect(subject.to_hash[:software][:foobar]).to be_a(Hash)
       end
 
       it "returns a build_version if one was passed in" do
-        expect(Omnibus::Manifest.new("1.2.3").to_hash["build_version"]).to eq("1.2.3")
+        expect(Omnibus::Manifest.new("1.2.3").to_hash[:build_version]).to eq("1.2.3")
       end
 
       it "returns a build_git_revision if one was passed in" do
-        expect(Omnibus::Manifest.new("1.2.3", "e8e8e8").to_hash["build_git_revision"]).to eq("e8e8e8")
+        expect(Omnibus::Manifest.new("1.2.3", "e8e8e8").to_hash[:build_git_revision]).to eq("e8e8e8")
       end
 
       it "returns a license if one was passed in" do
-        expect(Omnibus::Manifest.new("1.2.3", "e8e8e8", "Apache").to_hash["license"]).to eq("Apache")
+        expect(Omnibus::Manifest.new("1.2.3", "e8e8e8", "Apache").to_hash[:license]).to eq("Apache")
       end
     end
 
     describe "#from_hash" do
       let(:manifest) {
-        { "manifest_format" => 1,
-          "build_version" => "12.4.0+20150629082811",
-          "build_git_revision" => "2e763ac957b308ba95cef256c2491a5a55a163cc",
-          "software" => {
-            "zlib" => {
-              "locked_source" => {
-                "url" => "an_url"
+        { manifest_format: 1,
+          build_version: "12.4.0+20150629082811",
+          build_git_revision: "2e763ac957b308ba95cef256c2491a5a55a163cc",
+          software: {
+            zlib: {
+              locked_source: {
+                url: "an_url",
               },
-              "locked_version" => "new.newer",
-              "source_type" => "url",
-              "described_version" => "new.newer"}}}
-      }
-
-      let(:v2_manifest) {
-        {"manifest_format" => 2}
+              locked_version: "new.newer",
+              source_type: "url",
+              described_version: "new.newer",
+            }
+          }
+        }
       }
 
       it "has a build_version" do
@@ -109,11 +108,44 @@ module Omnibus
       end
 
       it "normalizes the source to use symbols" do
-        expect(Manifest.from_hash(manifest).entry_for("zlib").locked_source).to eq({:url => "an_url"})
-      end
-
-      it "raises an error if it doesn't recognize the manifest version" do
-        expect{Manifest.from_hash(v2_manifest)}.to raise_error Manifest::InvalidManifestFormat
+        expect(Manifest.from_hash(manifest).entry_for(:zlib).locked_source).to eq({url: "an_url"})
+      end
+
+      context "v2 manifest" do
+        let(:manifest) {
+          { manifest_format: 2,
+            build_version: "12.4.0+20150629082811",
+            build_git_revision: "2e763ac957b308ba95cef256c2491a5a55a163cc",
+            license: "Unspecified",
+            software: {
+              zlib: {
+                locked_source: {
+                  url: "an_url"
+                },
+                locked_version: "new.newer",
+                source_type: "url",
+                described_version: "new.newer",
+                license: "Zlib",
+              }
+            }
+          }
+        }
+
+        it "has a license" do
+          expect(Manifest.from_hash(manifest).license).to eq('Unspecified')
+        end
+      end
+
+      context "unsupported manifest" do
+        let(:manifest) {
+          {
+            manifest_format: 99,
+          }
+        }
+
+        it "raises an error if it doesn't recognize the manifest version" do
+          expect{Manifest.from_hash(manifest)}.to raise_error Manifest::InvalidManifestFormat
+        end
       end
     end
   end

data/spec/unit/metadata_spec.rb

@@ -6,10 +6,13 @@ module Omnibus
       double(described_class, path: '/path/to/package.deb.metadata.json')
     end
 
+    let(:package_path) { '/path/to/package.deb' }
+    let(:license_path) { '/opt/project/LICENSE' }
+
     let(:package) do
       double(Package,
         name:   'package',
-        path:   '/path/to/package.deb',
+        path:   package_path,
         md5:    'abc123',
         sha1:   'abc123',
         sha256: 'abcd1234',
@@ -17,10 +20,112 @@ module Omnibus
       )
     end
 
+    let(:project) do
+      double(Project,
+        name:             'some-project',
+        friendly_name:    'Some Project',
+        homepage:         'https://some.project.io',
+        build_version:    '1.2.3',
+        build_iteration:  '1',
+        license:          'Apache-2.0',
+        built_manifest:    double(Manifest,
+                                  to_hash: {
+                                    manifest_format: 2,
+                                    build_version: '1.2.3',
+                                    build_git_revision: 'SHA',
+                                    license: 'Apache-2.0'
+                                  }
+                                ),
+        license_file_path: license_path,
+      )
+    end
+
     let(:data) { { foo: 'bar' } }
+    let(:license_file_content) do
+      <<-EOH
+some_project 1.2.3 license: "Apache-2.0"
+
+                              Apache License
+                        Version 2.0, January 2004
+                     http://www.apache.org/licenses/
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+   "License" shall mean the terms and conditions for use, reproduction,
+   and distribution as defined by Sections 1 through 9 of this document.
+
+...
+      EOH
+    end
 
     subject { described_class.new(package, data) }
 
+    describe '.generate' do
+      let(:metadata_json_content) { StringIO.new }
+      let(:package_path) { '/path/to/package.deb' }
+
+      before do
+        allow(File).to receive(:exist?).with(package_path).and_return(true)
+        allow(Package).to receive(:new).with(package_path).and_return(package)
+        allow(File).to receive(:exist?).with(license_path).and_return(true)
+        allow(File).to receive(:read).with(license_path).and_return(license_file_content)
+        # mock out `Metadata#save`
+        allow(File).to receive(:open).with("#{package_path}.metadata.json", 'w+').and_yield(metadata_json_content)
+      end
+
+      it 'creates a *.metadata.json file with the correct content' do
+
+        described_class.generate(package_path, project)
+
+        expect(metadata_json_content.string).to include_json(
+          basename: 'package',
+          md5:      'abc123',
+          sha1:     'abc123',
+          sha256:   'abcd1234',
+          sha512:   'abcdef123456',
+          platform: 'ubuntu',
+          platform_version: '12.04',
+          arch: 'x86_64',
+          name: 'some-project',
+          friendly_name: 'Some Project',
+          homepage: 'https://some.project.io',
+          version: '1.2.3',
+          iteration: '1',
+          version_manifest: {
+            manifest_format: 2,
+            build_version: '1.2.3',
+            build_git_revision: 'SHA',
+            license: 'Apache-2.0',
+          },
+          license_content: license_file_content,
+        )
+      end
+
+      context 'the package file does not exist' do
+        before do
+          allow(File).to receive(:exist?).with(package_path).and_return(false)
+        end
+
+        it 'raises an exception' do
+          expect { described_class.generate(package_path, project) }.to raise_error(Omnibus::NoPackageFile)
+        end
+      end
+
+      context 'the license file does not exist' do
+        before do
+          allow(File).to receive(:exist?).with(license_path).and_return(false)
+        end
+
+        it 'does not include the license content' do
+          described_class.generate(package_path, project)
+          expect(metadata_json_content.string).to include_json(
+            license_content: '',
+          )
+        end
+      end
+    end
+
     describe '.arch' do
       let(:architecture) { 'x86_64' }
 
@@ -250,7 +355,7 @@ module Omnibus
 
     describe '#to_json' do
       it 'generates pretty JSON' do
-        expect(subject.to_json).to eq <<-EOH.gsub(/^ {10}/, '').strip
+        expect(subject.to_json.strip).to eq <<-EOH.gsub(/^ {10}/, '').strip
           {
             "foo": "bar"
           }