enhancerepo 0.4.1

data/lib/enhance_repo/rpm_md/products.rb

@@ -0,0 +1,142 @@
+#--
+# 
+# enhancerepo is a rpm-md repository metadata tool.
+# Copyright (C) 2008, 2009 Novell Inc.
+# Author: Duncan Mac-Vicar P. <dmacvicar@suse.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.  A copy of the GNU General Public License is
+# also available at http://www.gnu.org/copyleft/gpl.html.
+#
+#++
+#
+require 'rpm'
+require 'tempdir'
+require 'pathname'
+require 'nokogiri'
+require 'ftools'
+
+module EnhanceRepo
+  module RpmMd
+
+    # products.xml metadata generator
+    # reads the release files from a repository
+    class Products < Data
+      
+      # Holder for products we read from the
+      # release files
+      class ProductData
+        attr_accessor :name
+        attr_accessor :vendor
+        attr_accessor :version
+        attr_accessor :release
+        attr_accessor :arch
+        attr_accessor :productline
+        attr_accessor :summary
+        attr_accessor :description
+      end
+      
+      def initialize(dir)
+        @dir = dir
+        @products = []
+      end
+
+      def rpm_extract_file(rpmpath, path)
+        Tempdir.open do |tmppath|
+          File.makedirs(tmppath)
+          Dir.chdir(tmppath) do
+            `rpm2cpio '#{rpmpath}' | cpio -iv --make-directories #{File.join(".", path)} 2>/dev/null`
+          end
+          File.open(File.join(tmppath, path)) do |f|
+            yield f if block_given?
+            return f.read
+          end
+        end
+      end
+
+      def products_in_file_in_rpm(rpmpath, path)
+        products = []
+        rpm_extract_file(rpmpath, path) do |f|
+          doc = Nokogiri::XML(f)
+          #print doc.to_s
+          product = ProductData.new
+          # set attributes of the product based on the xml data
+          [:name, :version, :release, :arch, :vendor, :summary, :description].each do |attr|
+            product.send("#{attr}=".to_sym, doc.root.xpath("./#{attr}").text)
+          end
+          products << product
+          yield product if block_given?
+        end
+        products
+      end
+
+      # scan the products from the rpm files in the repository
+      def read_packages
+#        log.info "Looking for product release packages"
+        Dir["#{@dir}/**/*-release-*.rpm", "#{@dir}/**/*-migration-*.rpm"].each do |rpmfile|
+          pkg = RPM::Package.new(rpmfile)
+          # we dont care for packages not providing a product
+          next if pkg.provides.select{|x| x.name == "product()"}.empty?
+          log.info "Found product release package #{rpmfile}"
+          # this package contains a product
+          # go over each product file
+          pkg.files.map {|x| x.to_s }.each do |path|
+            next if not ( File.extname(path) == ".prod" && File.dirname(path) == "/etc/products.d" )
+            # we have a product file. Extract it
+            log.info "`-> product file : #{path}"
+            products_in_file_in_rpm(File.expand_path(rpmfile), File.expand_path(path)) do |product|
+              log.info "`-> found product : #{product.name}"
+              @products << product
+            end
+          end
+        end
+      end
+
+      def empty?
+        @products.empty?
+      end
+
+      def size
+        @products.size
+      end
+      
+      def write(io)
+        builder = Nokogiri::XML::Builder.new do |xml|
+          xml.products do
+            @products.each do |product|
+              xml.product do
+                xml.name product.name
+                version = RPM::Version.new("#{product.version}-#{product.release}")
+                epoch = version.e
+                epoch ||= "0"
+                xml.version :epoch => epoch, :ver => version.v, :rel => version.r
+                xml.arch product.arch
+                xml.vendor product.vendor
+                xml.summary product.summary
+                xml.description product.description                
+              end
+            end
+          end
+        end
+        # write the result
+        #io.write(builder.to_xml)
+        io.write(builder.doc.to_xml(:save_with => Nokogiri::XML::Node::SaveOptions::AS_XML))
+      end
+
+      
+    end
+    
+  end
+end

data/lib/enhance_repo/rpm_md/repo.rb

@@ -0,0 +1,202 @@
+#--
+# 
+# enhancerepo is a rpm-md repository metadata tool.
+# Copyright (C) 2008, 2009 Novell Inc.
+# Author: Duncan Mac-Vicar P. <dmacvicar@suse.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.  A copy of the GNU General Public License is
+# also available at http://www.gnu.org/copyleft/gpl.html.
+#
+#++
+#
+require 'rubygems'
+require 'builder'
+require 'rexml/document'
+require 'digest/sha1'
+require 'zlib'
+require 'yaml'
+
+require 'enhance_repo/package_id'
+require 'enhance_repo/rpm_md/data'
+require 'enhance_repo/rpm_md/primary'
+require 'enhance_repo/rpm_md/file_lists'
+require 'enhance_repo/rpm_md/other'
+require 'enhance_repo/rpm_md/update_info'
+require 'enhance_repo/rpm_md/suse_info'
+require 'enhance_repo/rpm_md/suse_data'
+require 'enhance_repo/rpm_md/delta_info'
+require 'enhance_repo/rpm_md/products'
+require 'enhance_repo/rpm_md/index'
+
+module EnhanceRepo
+  module RpmMd
+    
+    include REXML
+    
+    class Repo
+
+      include Logger
+      
+      attr_accessor :index
+
+      # extensions
+      attr_reader :primary, :other, :filelists, :susedata, :suseinfo, :deltainfo, :updateinfo, :products
+      def initialize(config)
+        @dir = config.dir
+        @outputdir = config.outputdir
+
+        @index = Index.new
+        repomdfile = File.join(@dir, @index.metadata_filename)
+        # populate the index
+        if File.exist?(repomdfile)
+          @index.read_file(File.new(repomdfile))
+        end
+        
+        @primary = Primary.new(config.dir)
+        @primary.indent = config.indent
+        
+        @filelists = FileLists.new(config.dir)
+        @other = Other.new(config.dir)
+        @susedata = SuseData.new(config.dir)
+        @updateinfo = UpdateInfo.new(config)
+        @suseinfo = SuseInfo.new(config.dir)
+        @deltainfo = DeltaInfo.new(config.dir)
+        @products = Products.new(config.dir)
+      end
+
+      def sign(keyid)
+        # check if the index is written to disk
+        repomdfile = File.join(@dir, @index.metadata_filename)
+        if not File.exist?(repomdfile)
+          raise "#{repomdfile} does not exist."
+        end
+        # call gpg to sign the repository
+        `gpg -sab -u #{keyid} -o '#{repomdfile}.asc' '#{repomdfile}'`
+        if not File.exists?("#{repomdfile}.asc")
+          log.info "Could't not generate signature #{repomdfile}.asc"
+          exit(1)
+        else
+          log.info "#{repomdfile}.asc signature generated"
+        end
+
+        # now export the public key
+        `gpg --export -a -o '#{repomdfile}.key' #{keyid}`
+
+        if not File.exists?("#{repomdfile}.key")
+          log.info "Could't not generate public key #{repomdfile}.key"
+          exit(1)
+        else
+          log.info "#{repomdfile}.key public key generated"
+        end
+      end
+
+      def write
+
+        datas = [@primary, @filelists, @other, @updateinfo,
+                 @susedata, @suseinfo, @deltainfo, @products ]
+
+        # select the datas that are not empty
+        # those need to be saved
+        non_empty_data = datas.reject { |x| x.empty? }
+        # files present in the index, which were changed
+        changed_files = []
+        # files present on disk, but not in the index
+        missing_files = []
+        # files present in the index, but not on disk
+        superflous_files = []
+        
+        # now look for files that changed or dissapeared
+        Dir.chdir(@dir) do
+          # look all files except the index itself
+          metadata_files = Dir["repodata/*.xml*"].reject do |x|            
+            x  =~ /#{@index.metadata_filename}/ ||
+            x =~ /\.key$/ ||
+            x =~ /\.asc$/
+          end
+          # remove datas in the index not present in the disk
+          @index.resources.reject! do |resource|
+            reject = ! metadata_files.include?(resource.location)
+            log.info "Removing not existing #{resource.location} from index" if reject
+            reject
+          end
+          
+          non_empty_files = non_empty_data.map { |x| x.metadata_filename }
+          # ignore it if it is already in the non_empty_list
+          # as it will be added to the index anyway
+          metadata_files.reject!{ |x| non_empty_files.include?(x) }
+          metadata_files.each do |metadata_file|
+            # find the indexed resource for this file
+            indexed_resource = @index.resources.select { |x| x.location == metadata_file }.first
+            # add it to the list of changed resources if the timestamp
+            # are differents
+            if indexed_resource.nil?
+              missing_files << metadata_file
+              next
+            elsif File.mtime(File.join(@dir, metadata_file)).to_i != indexed_resource.timestamp.to_i
+              changed_files << metadata_file
+              next
+            end
+          end
+        end
+        
+        # write down changed datas
+        non_empty_data.each do |data|
+          write_gz_extension_file(data)
+        end
+
+        # update the index
+        non_empty_data.each do |d|
+          log.info "Adding #{d.metadata_filename} to #{@index.metadata_filename} index"
+          @index.add_file_resource(File.join(@outputdir, d.metadata_filename), d.metadata_filename)
+        end
+
+        missing_files.each do |f|
+          log.info "Adding missing #{f} to #{@index.metadata_filename} index"
+          @index.add_file_resource(File.join(@outputdir, f), f)
+        end
+                
+        changed_files.each do |f|
+          log.info "Replacing changed #{f} on #{@index.metadata_filename} index"
+          @index.add_file_resource(File.join(@outputdir, f), f)
+        end
+        
+        # now write the index
+        File.open((@outputdir + @index.metadata_filename), 'w') do |f|
+          log.info "Saving #{@index.metadata_filename} .."
+          @index.write(f)
+        end
+      end
+
+      # writes an extension to an xml filename if
+      # the extension is not empty
+      def write_gz_extension_file(data)
+        filename = Pathname.new(File.join(@outputdir, data.metadata_filename))
+        FileUtils.mkdir_p filename.dirname
+        log.info "Saving #{filename} .."
+        if not filename.dirname.exist?
+          log.info "Creating non existing #{filename.dirname} .."
+          filename.dirname.mkpath
+        end
+        # compress the output
+        Zlib::GzipWriter.open(filename) do |gz|
+          data.write(gz)
+        end
+      end
+      
+    end
+
+  end
+end

data/lib/enhance_repo/rpm_md/resource.rb

@@ -0,0 +1,44 @@
+#--
+# 
+# enhancerepo is a rpm-md repository metadata tool.
+# Copyright (C) 2008, 2009 Novell Inc.
+# Author: Duncan Mac-Vicar P. <dmacvicar@suse.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.  A copy of the GNU General Public License is
+# also available at http://www.gnu.org/copyleft/gpl.html.
+#
+#++
+#
+
+module EnhanceRepo
+  module RpmMd
+
+    # represents a resource in repomd.xml
+    class Resource
+      attr_accessor :type
+      attr_accessor :location, :checksum, :timestamp, :openchecksum
+
+      # define equality based on the location
+      # as it has no sense to have two resources for the
+      #same location
+      def ==(other)
+        return (location == other.location) if other.is_a?(Resource)
+        false
+      end      
+    end
+    
+  end
+end

data/lib/enhance_repo/rpm_md/suse_data.rb

@@ -0,0 +1,191 @@
+#--
+# 
+# enhancerepo is a rpm-md repository metadata tool.
+# Copyright (C) 2008, 2009 Novell Inc.
+# Author: Duncan Mac-Vicar P. <dmacvicar@suse.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.  A copy of the GNU General Public License is
+# also available at http://www.gnu.org/copyleft/gpl.html.
+#
+#++
+#
+
+module EnhanceRepo
+  module RpmMd
+
+    class Property
+      attr_accessor :name
+      def initialize(name)
+        @name = name
+      end
+
+      def hash
+        @name.hash
+      end
+
+      def eql?(other)
+        @name.eql?(other.name)
+      end
+    end
+
+    class ValueProperty < Property
+      def initialize(name, value)
+        super(name)
+        @value = value
+      end
+
+      def write(builder, pkgid)
+        builder.tag!(@name, @value)
+      end
+    end
+
+    class DiskUsageProperty < Property
+
+      def initialize(pkgid, rpmfile)
+        super('diskusage')
+        @pkgid = pkgid
+        @rpmfile = rpmfile
+      end
+      
+      def write(builder, pkgid)
+        dirsizes = Hash.new
+        dircount = Hash.new
+        `rpm -q --queryformat \"[%{FILENAMES} %{FILESIZES}\n]\" -p '#{@rpmfile}'`.each_line do |line|
+          file, size = line.split
+          dirsizes[File.dirname(file)] = 0 if not dirsizes.has_key?(File.dirname(file))
+          dircount[File.dirname(file)] = 0 if not dircount.has_key?(File.dirname(file))
+          
+          dirsizes[File.dirname(file)] += size.to_i
+          dircount[File.dirname(file)] += 1
+        end
+
+        builder.diskusage do |b|
+          b.dirs do |b|
+            dirsizes.each do |k, v|
+              b.dir('name' => k, 'size' => v, 'count' => dircount[k] )
+            end
+          end
+        end      
+      end
+      
+    end
+
+    # represents SUSE extensions to
+    # primary data
+    #
+    # See:
+    # http://en.opensuse.org/Standards/Rpm_Metadata#SUSE_primary_data_.28susedata.xml.29
+    #
+    class SuseData < Data
+
+      def initialize(dir)
+        @dir = dir
+        @diskusage_enabled = false
+
+        # the following hash automatically creates a sub
+        # hash for non found values
+        # @properties = Hash.new { |h,v| h[v]= Hash.new }
+        @properties = Hash.new
+
+      end
+
+      # add an attribute named name for a
+      # package identified with pkgid
+      def add_attribute(pkgid, prop)
+        if not @properties.has_key?(pkgid)
+          @properties.store(pkgid, Hash.new)
+        end
+        @properties[pkgid][prop.name] = prop
+      end
+      
+      def add_eulas
+        # add eulas
+        Dir["#{@dir}/**/*.eula"].each do |eulafile|
+          base = File.basename(eulafile, '.eula')
+          # =>  look for all rpms with that name in that dir
+          Dir["#{File.dirname(eulafile)}/#{base}*.rpm"].each do | rpmfile |
+            pkgid = PackageId.new(rpmfile)
+            if pkgid.matches(base)
+              eulacontent = File.new(eulafile).read
+              add_attribute(pkgid, ValueProperty.new('eula', eulacontent))
+              log.info "Adding eula: #{eulafile.to_s} to #{pkgid.to_s}"
+            end
+          end
+        end
+        # end of directory iteration
+      end
+
+      def add_keywords
+        # add keywords
+        log.info "Adding repository keywords"
+        Dir["#{@dir}/**/*.keywords"].each do |keywordfile|
+          base = File.basename(keywordfile, '.keywords')
+          # =>  look for all rpms with that name in that dir
+          Dir["#{File.dirname(keywordfile)}/#{base}*.rpm"].each do | rpmfile |
+            pkgid = PackageId.new(rpmfile)
+            if pkgid.matches(base)
+              f = File.new(keywordfile)
+              f.each_line do |line|
+                keyword = line.chop
+                add_attribute(pkgid, ValueProperty.new('keyword', keyword)) if not keyword.empty?
+              end
+              log.info "`-> adding keyword: #{keywordfile.to_s} to #{pkgid.to_s}"
+            end
+          end
+        end
+        # end of directory iteration
+      end
+
+      def empty?
+        @properties.empty?
+      end
+
+      def size
+        @properties.size
+      end
+
+      # write an extension file like other.xml
+      def write(file)
+        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+        builder.instruct!
+        xml = builder.tag!(name) do |b|
+          @properties.each do |pkgid, props|
+            #log.info "Dumping package #{pkgid.to_s}"
+            b.package('pkgid' => pkgid.checksum, 'name' => pkgid.name) do |b|
+              b.version('ver' => pkgid.version.v, 'rel' => pkgid.version.r, 'arch' => pkgid.arch, 'epoch' => 0.to_s )
+              props.each do |propname, prop|
+                #log.info "   -> property #{prop.name}"
+                prop.write(builder, pkgid)
+              end
+            end # end package tag
+          end # iterate over properties
+        end #done builder
+      end
+      
+      def add_disk_usage
+        @diskusage_enabled = true
+        log.info "Calculating disk usage..."
+        # build the pkgid hash
+        Dir["#{@dir}/**/*.rpm"].each do |rpmfile|
+          pkgid = PackageId.new(rpmfile)
+          add_attribute(pkgid, DiskUsageProperty.new(pkgid, rpmfile))
+        end
+      end
+      
+    end
+
+  end
+end