enhancerepo 0.4.1

data/CHANGELOG.rdoc

@@ -0,0 +1,18 @@
+
+== 0.4 (not released)
+
+* improved logger support
+* --split-updates, patch by Jordi Massaguer Pla
+* primary data generation (EXPERIMENTAL)
+* allows to generate metadata to a separate directory
+* product metadata generation extracted from *-release rpm packages
+* benchmark feature
+* speed improvements
+
+== 0.3.2
+
+* patch generation support
+
+== 0.3.1
+
+* deltarpm support

data/Manifest.txt

@@ -0,0 +1,85 @@
+lib/enhance_repo.rb
+lib/tempdir/tempfile.rb
+lib/enhance_repo/xml_comparer.rb
+lib/enhance_repo/rpm_md.rb
+lib/enhance_repo/package_id.rb
+lib/enhance_repo/rpm_md/update.rb
+lib/enhance_repo/rpm_md/suse_data.rb
+lib/enhance_repo/rpm_md/primary.rb
+lib/enhance_repo/rpm_md/resource.rb
+lib/enhance_repo/rpm_md/data.rb
+lib/enhance_repo/rpm_md/delta_info.rb
+lib/enhance_repo/rpm_md/suse_info.rb
+lib/enhance_repo/rpm_md/other.rb
+lib/enhance_repo/rpm_md/index.rb
+lib/enhance_repo/rpm_md/update_smart_fields.rb
+lib/enhance_repo/rpm_md/update_info.rb
+lib/enhance_repo/rpm_md/file_lists.rb
+lib/enhance_repo/rpm_md/repo.rb
+lib/enhance_repo/rpm_md/products.rb
+lib/enhance_repo/rpm_md/extra_primary_data.rb
+lib/enhance_repo/other.rb
+lib/enhance_repo/cli.rb
+lib/enhance_repo/product_id.rb
+lib/enhance_repo/logger.rb
+lib/enhance_repo/config_opts.rb
+lib/tempdir.rb
+test/repo_test.rb
+test/susedata_test.rb
+test/updateinfo_test.rb
+test/tempdir_test.rb
+test/products_test.rb
+test/primary_test.rb
+test/rpmmd_test.rb
+test/test_helper.rb
+test/repomdindex_test.rb
+test/all_tests.rb
+test/index_test.rb
+test/deltainfo_test.rb
+test/extensions_test.rb
+test/update_test.rb
+test/data/rpms/repo-1/repodata/susedata.xml.gz
+test/data/rpms/repo-1/repodata/filelists.xml.gz
+test/data/rpms/repo-1/repodata/deltainfo.xml.gz.gz
+test/data/rpms/repo-1/repodata/deltainfo.xml.gz
+test/data/rpms/repo-1/repodata/other.xml.gz
+test/data/rpms/repo-1/repodata/updateinfo.xml.gz
+test/data/rpms/repo-1/repodata/repomd.xml
+test/data/rpms/repo-1/repodata/primary.xml.gz
+test/data/rpms/repo-1/a-1.0-0.x86_64.rpm
+test/data/rpms/repo-1/a-2.0-0.x86_64.rpm
+test/data/rpms/repo-1/a-1.0_2.0-0_0.x86_64.delta.rpm
+test/data/rpms/repo-1/repoparts/update-a-1.xml
+test/data/rpms/repo-with-product/repodata/filelists.xml.gz
+test/data/rpms/repo-with-product/repodata/products.xml.gz
+test/data/rpms/repo-with-product/repodata/other.xml.gz
+test/data/rpms/repo-with-product/repodata/repomd.xml
+test/data/rpms/repo-with-product/repodata/primary.xml.gz
+test/data/rpms/repo-with-product/a-1.0-0.x86_64.rpm
+test/data/rpms/repo-with-product/openSUSE-release-11.2-1.6.i586.rpm
+test/data/rpms/repo-with-product/openSUSE-release-dvd-11.2-1.6.i586.rpm
+test/data/rpms/update-test-factory/update-test-reboot-needed-0-9999.1.2.noarch.rpm
+test/data/rpms/update-test-factory/update-test-security-0-9999.1.2.noarch.rpm
+test/data/rpms/update-test-factory/update-test-affects-package-manager-0-9999.1.2.noarch.rpm
+test/data/rpms/update-test-factory/update-test-interactive-0-9999.1.2.noarch.rpm
+test/data/rpms/update-test-factory/update-test-relogin-suggested-0-9999.1.2.noarch.rpm
+test/data/rpms/update-test-factory/update-test-optional-0-9999.1.2.noarch.rpm
+test/data/rpms/update-test-11.1/update-test-trival.rpm
+test/data/rpms/update-test-11.1/update-test-optional.rpm
+test/data/rpms/update-test-11.1/update-test-security.rpm
+test/data/rpms/update-test-11.1/update-test-reboot-needed.rpm
+test/data/rpms/update-test-11.1/update-test-affects-package-manager.rpm
+test/data/rpms/update-test-11.1/update-test-interactive.rpm
+test/data/repodata/update-test-11.1/filelists.xml.gz
+test/data/repodata/update-test-11.1/other.xml.gz
+test/data/repodata/update-test-11.1/repomd.xml
+test/data/repodata/update-test-11.1/primary.xml.gz
+test/data/packages/a-2.0.spec
+test/data/packages/a-1.0.spec
+test/data/repomd.xml
+Rakefile
+README
+README.rdoc
+CHANGELOG.rdoc
+bin/enhancerepo
+Manifest.txt

data/README

@@ -0,0 +1,10 @@
+
+enhancerepo
+
+Author: Duncan Mac-Vicar P. <dmacvicar@suse.de>
+
+Reference implementation for future rpmmd extensions
+
+For more visit: http://en.opensuse.org/Enhancerepo
+
+requires ruby-rpm.

data/README.rdoc

@@ -0,0 +1,15 @@
+
+= enhancerepo
+
+* http://en.opensuse.org/Enhancerepo
+
+== Introduction
+
+Reference implementation for future rpmmd extensions
+
+requires ruby-rpm.
+
+== Authors
+
+* Duncan Mac-Vicar P. <dmacvicar@suse.de>
+

data/Rakefile

@@ -0,0 +1,48 @@
+$: << File.join(File.dirname(__FILE__), "test")
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+
+task :default => [:docs, :test]
+
+Hoe.plugin :yard
+
+HOE = Hoe.spec 'enhancerepo' do
+  developer('Duncan Mac-Vicar P.', 'dmacvicar@suse.de')
+# s.homepage  =   "http://en.opensuse.org/Enhancerepo"
+  self.summary = "Adds additional information to repomd repositories"
+  self.description = "enhancerepo adds additional metadata to repommd repositories and
+servers as the testbed for the specification"
+  self.readme_file = ['README', ENV['HLANG'], 'rdoc'].compact.join('.')
+  self.history_file = ['CHANGELOG', ENV['HLANG'], 'rdoc'].compact.join('.')
+  self.extra_rdoc_files = FileList['*.rdoc']
+
+  self.extra_deps << ['trollop', '>= 1.0.5']
+  self.extra_deps << ['log4r', '>= 1.0.5']
+  self.extra_deps << ['nokogiri', '>= 1.4']
+  self.extra_deps << ['activesupport', '>= 2.3']
+
+  self.extra_dev_deps << ['shoulda', '>= 0']
+  self.extra_dev_deps << ['mocha', '>= 0']
+  self.extra_dev_deps << ['yard', '>= 0']
+end
+
+desc "Insert GPL into all source files"
+task :GPL do
+  gpl = File.readlines('GPL_header.txt')
+  FileList['**/*.rb'].each do |filename|
+    File.open(filename, 'r+') do |file|
+      lines = file.readlines      
+      # Skip shebang line
+      i = (lines[0].index('#!') == 0) ? 1 : 0
+      # Already have header?
+      next if lines[i].index('#--') == 0
+      
+      puts "Liberating #{filename}"
+      
+      file.pos = 0
+      file.print lines.insert(i, gpl).flatten
+      file.truncate(file.pos)
+    end
+  end
+end

data/TODO.rdoc

@@ -0,0 +1,4 @@
+
+* replace accessors and initialization with ActiveSupport's
+  attr_accessor_with_default
+

data/bin/enhancerepo

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+$: << '../lib' 
+$: << 'lib'
+require 'enhance_repo/cli.rb'

data/lib/enhance_repo.rb

@@ -0,0 +1,36 @@
+#--
+# 
+# enhancerepo is a rpm-md repository metadata tool.
+# Copyright (C) 2008, 2009 Novell Inc.
+# Author: Duncan Mac-Vicar P. <dmacvicar@suse.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.  A copy of the GNU General Public License is
+# also available at http://www.gnu.org/copyleft/gpl.html.
+#
+#++
+#
+module EnhanceRepo
+  VERSION = "0.4.1"
+end
+
+lib = File.dirname(__FILE__)
+
+require File.join(lib, 'enhance_repo', 'logger.rb')
+require File.join(lib, 'enhance_repo', 'config_opts.rb')
+require File.join(lib, 'enhance_repo', 'rpm_md.rb')
+
+
+  

data/lib/enhance_repo/cli.rb

@@ -0,0 +1,150 @@
+#--
+# 
+# enhancerepo is a rpm-md repository metadata tool.
+# Copyright (C) 2008, 2009 Novell Inc.
+# Author: Duncan Mac-Vicar P. <dmacvicar@suse.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.  A copy of the GNU General Public License is
+# also available at http://www.gnu.org/copyleft/gpl.html.
+#
+#++
+#
+require 'rubygems'
+require 'trollop'
+require 'enhance_repo'
+require 'pathname'
+require 'benchmark'
+require 'fileutils'
+
+EnhanceRepo::enable_logger
+
+opts = Trollop::options do
+  version "enhancerepo #{EnhanceRepo::VERSION}"
+  banner <<-EOS
+enhancerepo is a rpm-md metadata tool
+
+Usage:
+        enhancerepo [options] DIR
+
+        DIR: The repo base directory ( where repodata/ directory is located )
+EOS
+  opt :help, 'Show help'
+  opt :outputdir, 'Generate metadata to a different directory', :short => :o
+  opt :index, "Reindex the metadata and regenerates repomd.xml, even if nothing was changed using enhancerepo. Use this if you did manual changes to the metadata", :short => :x
+  opt :benchmark, 'Show benchmark statistics at the end'
+  
+  opt :primary, 'Add data from rpm files and generate primary.xml (EXPERIMENTAL)', :short => :p
+  opt :sign, 'Generates signature for the repository using key keyid', :short => :s, :type => :string
+  opt :updates, 'Add updates from *.updates files and generate updateinfo.xml', :short => :u
+  opt :generate_update, 'Generates an update from the given package list comparing package\'s last version changes', :type => :strings
+  opt :updates_base_dir, 'Looks for package also in <dir> Useful if you keep old packages in a different repos and updates in this one.', :type => :string
+  opt :split_updates, 'Splits current updateinfo.xml into update parts files in repoparts/'
+  opt :indent, 'Generate indented xml. Default: no', :short => :i
+  
+  opt :expire, 'Set repository expiration hint (Can be used to detect dead mirrors)', :type => :date, :short => :e
+  opt :repo_product, 'Adds product compatibility information', :type => :strings
+  opt :repo_keyword, 'Tags repository with keyword', :type => :strings
+
+  # === SUSE specific package data (susedata.xml)
+  
+  opt :eulas, 'Reads packagename.eula files and add the information to susedata.xml', :short => :l
+  opt :keywords, 'Reads packagename.keywords files and add keyword metadata to susedata.xml', :short => :k
+  opt :disk_usage, 'Reads rpm packages, generates disk usage information on susedata.xml', :short => :d
+
+  # Note: your .eula or .keywords file will be added to
+  # a package if it matches its name. If you want to add
+  # the attributes to a specific package, name the file
+  # name-version, name-version-release or
+  # name-version-release.arch
+
+  # === Package deltas support
+  opt :'create-deltas',
+      'Create [num] deltas for different versions of a package. If there is foo-1.rpm, foo-2.rpm, foo-3.rpm, foo-4.rpm num=1 will create a delta to go from version 3 to 4, while num=2 will create one from 2 to 4 too. This does not index the deltas. Use --deltas for that.', :default => 1
+  opt :deltas, 'Reads all *.delta.rpm files and add the information to deltainfo.xml. This indexes existing deltas, but won\'t create them. See --create-deltas for deltas creation.'
+
+  # === Product information support
+
+  opt :products, 'Reads release packages and generating product information in products.xml based on the information contained in the .prod files included in the packages.'
+
+  # other
+  opt :debug, 'Show debug information'
+end
+
+config = EnhanceRepo::ConfigOpts.new(opts)
+dir = ARGV.shift
+config.dir = Pathname.new(dir) if not dir.nil?
+
+# Check if dir is given
+if config.dir.nil?
+  EnhanceRepo.logger.fatal "Missing dir argument (try --help)"
+  exit 0
+end
+
+repomd = EnhanceRepo::RpmMd::Repo.new(config)
+
+# perform the operations in a rescue block
+
+time = Benchmark.measure do
+  begin
+    if config.primary
+      repomd.primary.read_packages
+      repomd.filelists.read
+      repomd.other.read
+    end
+
+    # merge keywords and products to suseinfo
+    repomd.suseinfo.products.merge(config.repoproducts)
+    repomd.suseinfo.keywords.merge(config.repokeywords)
+
+    repomd.susedata.add_eulas if config.eulas  
+    repomd.susedata.add_keywords if config.keywords
+    repomd.susedata.add_disk_usage if config.diskusage
+
+    if not config.generate_update.nil?
+      repomd.updateinfo.generate_update(config.generate_update, File.join(config.outputdir, 'repoparts') )
+    end
+
+    repomd.updateinfo.read_repoparts if config.updates  
+    repomd.updateinfo.split_updates(File.join(config.outputdir, 'repoparts')) if config.split_updates                                                                        
+
+    repomd.deltainfo.create_deltas(:outputdir => config.outputdir, :n => config.create_deltas) if config.create_deltas
+    repomd.deltainfo.add_deltas if config.deltas
+
+    repomd.products.read_packages if config.products
+
+    # add expiration date
+    repomd.suseinfo.expire = config.expire if not config.expire.nil?
+
+    # index if requested
+    repomd.index if config.index
+    
+    # write the repository out
+    repomd.write
+
+    # perform signature of the repository
+    repomd.sign(config.signkey) if not config.signkey.nil?  
+  rescue Exception => excp
+    EnhanceRepo.logger.fatal excp.message
+    if EnhanceRepo::enable_debug
+      EnhanceRepo.logger.fatal excp.class
+      EnhanceRepo.logger.fatal(excp.backtrace.join("\n"))
+    else
+      EnhanceRepo.logger.info "Pass --debug for more information..."
+    end
+  end
+
+end
+EnhanceRepo.logger.info(time) if config.benchmark

data/lib/enhance_repo/config_opts.rb

@@ -0,0 +1,98 @@
+#--
+# 
+# enhancerepo is a rpm-md repository metadata tool.
+# Copyright (C) 2008, 2009 Novell Inc.
+# Author: Duncan Mac-Vicar P. <dmacvicar@suse.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.  A copy of the GNU General Public License is
+# also available at http://www.gnu.org/copyleft/gpl.html.
+#
+#++
+#
+require 'set'
+
+module EnhanceRepo
+  
+  # Configuration class to hold the options
+  # passed from the command line to the
+  # components doing the work
+  #
+
+  class ConfigOpts
+    attr_accessor :index
+    attr_accessor :indent
+    attr_accessor :repoproducts
+    attr_accessor :repokeywords
+    attr_accessor :dir
+    # output dir, if specified, if not
+    # we use dir
+    attr_writer :outputdir
+    attr_accessor :signkey
+    attr_accessor :expire
+    attr_accessor :primary
+    attr_accessor :updates
+    attr_accessor :generate_update
+    attr_accessor :split_updates
+    attr_accessor :updatesbasedir
+    attr_accessor :eulas
+    attr_accessor :keywords
+    attr_accessor :diskusage
+    # wether to create delta rpm files
+    # and how many
+    attr_accessor :create_deltas
+    # whether to index delta rpms
+    attr_accessor :deltas
+
+    attr_accessor :products
+
+    attr_accessor :benchmark
+    
+    def outputdir
+      return @dir if @outputdir.nil?
+      return @outputdir
+
+    end
+    
+    def initialize(opts)
+      @repoproducts = Set.new
+      @repokeywords = Set.new
+      read_opts(opts)
+    end
+    
+    def read_opts(opts)
+      @index = opts[:index]
+      @expire = opts[:expire]
+      @primary = opts[:primary]
+      @repoproducts = @repoproducts.merge([*opts[:repo_products]])
+      @repokeywords = @repokeywords.merge([*opts[:repo_keywords]])
+      @signkey = opts[:sign]
+      @updates = opts[:updates]
+      @split_updates = opts[:split_updates]
+      @generate_update = opts[:generate_update]
+      @eulas = opts[:eulas]
+      @keywords = opts[:keywords]
+      @diskusage = opts[:disk_usage]
+      @deltas = opts[:deltas]
+      @create_deltas = opts[:create_deltas]
+      @products = opts[:products]
+      @benchmark = opts[:benchmark]      
+      @updatesbasedir = Pathname.new(opts[:updates_base_dir]) if opts[:updates_base_dir]
+      @outputdir = Pathname.new(opts[:outputdir]) if opts[:outputdir]
+      @dir = Pathname.new(opts[:dir]) if opts[:dir]
+    end
+    
+  end
+end