rpmrepository 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f0cdef908e95af3293de5d542c3eb9aa8c71a5c4
+  data.tar.gz: 6621f6ab0eebd01755f89fec3a352dca7e9832f3
+SHA512:
+  metadata.gz: 7c3ea38b17f3952ae1ec2f5c5195e3c11ddaa7d41fb66c1bc727ecda338e8c82e721f424fa61bae4fefba2c4afb54c5e2e1ed585531419db070a6656b201f31b
+  data.tar.gz: 45e71797540b581ae48bfd34e7a3ae064e054af458bd73ac44b5e54b4aa0b5f2995657a7a3b844cf67d0dcd4fc770fbfe681ea8f2bc025b1a23f897bad273b67

data/lib/package.rb

@@ -0,0 +1,199 @@
+#One package with possible multiple locations
+
+module RPM
+    
+    class Package < Monitor
+        
+        attr_reader :uris
+        attr_reader :name
+        attr_reader :version
+        attr_reader :release
+        attr_reader :architecture
+        attr_reader :size
+        attr_reader :file_size
+        attr_reader :digests
+
+        def initialize raw_uri
+            super()
+            @uris = []
+            synchronize! {
+              raise ArgumentError, "Only String or URI, not #{raw_uri.class}" unless raw_uri.kind_of? String or raw_uri.kind_of? URI::Generic
+              if raw_uri.kind_of? URI::Generic
+                  uri = raw_uri.dup
+              else
+                  uri = URI::parse raw_uri
+              end
+              if (uri.scheme == 'file' or uri.scheme == nil) and File.exist? uri.path
+                  'file'
+              elsif uri.class == URI::HTTP and Net::HTTP.new(uri.host,uri.port).get(uri).is_a? Net::HTTPSuccess
+                  'remote'
+              else
+                  raise ArgumentError, "Unreachable URI provided: #{uri.to_s}"
+              end
+              @uris.push uri
+              get_attributes
+            }
+        end
+        
+        def duplicate_to dst_dir, file_name = get_default_name
+            if dst_dir[/^\..*/]
+                dst_dir = File::expand_path dst_dir
+            end
+            raise ArgumentError, "Path must be absolute (#{dst_dir})" unless dst_dir[/^\//]
+            target_uri = URI::parse "file:#{dst_dir}/#{file_name}"
+            synchronize! {
+              return true if get_local_uris.include? target_uri
+              if uri = get_local_uris.first and not uri.nil?
+                  raise Errno::EEXIST, "File #{File::basename target_uri.path} already exist!" if File.exist? target_uri.path
+                  begin
+                      FileUtils::link uri.path, target_uri.path
+                  rescue Errno::EXDEV => e
+                      FileUtils::cp uri.path, target_uri.path
+                  end
+              else
+                  get_remote_uris.each { |remote_uri|
+                  begin
+                      raise Errno::EEXIST, "File #{File::basename target_uri.path} already exist!" if File.exist? target_uri.path
+                      response = Net::HTTP.new(remote_uri.host,remote_uri.port).get(remote_uri)
+                      if response.is_a? Net::HTTPSuccess
+                          File.open(target_uri.path, "w+") { |f|
+                              f.write response.body
+                          }
+                      else
+                          next
+                      end
+                      #check what we write?
+                      break
+                  rescue Errno::ENETUNREACH, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, URI::InvalidURIError => e
+                      next
+                  end
+                  }
+              end
+              @uris.push target_uri if File.exist? target_uri.path
+              return File.exist? target_uri.path
+            }
+        end
+        
+        #Removes all URL's undo provided dir
+        def deduplicate_undo dir
+            synchronize! {
+              (get_local_uris_undo dir).each { |uri|
+                  FileUtils::rm_f uri.path
+                  @uris.delete uri
+              }
+            }
+        end
+        
+        #Try to expand current package by other source uris
+        def join! other_package
+          synchronize! {
+            if same_as? other_package
+              @uris += other_package.uris
+              @uris.uniq!
+              return true
+            else
+              return false
+            end
+          }
+        end
+        
+        #Remove each reachable uri
+        def destroy!
+          synchronize! {
+            get_local_uris.each { |victim|
+                FileUtils::rm_f victim.path
+                @uris.delete victim
+            }
+          }
+        end
+        
+        #Return true if packages has same attributes
+        def same_as? other
+            other.is_a? RPM::Package and
+            @name == other.name and
+            @version == other.version and
+            @release == other.release and
+            @architecture == other.architecture and
+            @digests[:sha256] == other.digests[:sha256]
+        end
+        
+        #Return expected RPM Package file name
+        def get_default_name
+            "#{@name}-#{@version}-#{@release}.#{@architecture}.rpm"
+        end
+        
+        #Return local URI's undo spicified directory
+        def get_local_uris_undo dir
+            if dir[/^\..*/]
+                dir = File::expand_path dir
+            end
+            unless dir[/.*\/$/]
+                dir = dir + '/'
+            end
+            synchronize! {
+              get_local_uris.select { |uri| uri.path[/^#{dir}/] }
+            }
+        end
+        
+        #Return every local uris
+        def get_local_uris
+            synchronize! { @uris.select { |uri| uri.scheme == nil or uri.scheme == "file" } }
+        end
+        
+        #Return every remote uris
+        def get_remote_uris
+            synchronize! { @uris.select { |uri| uri.scheme != nil and uri.scheme != "file" } }
+        end
+        
+        #Try to repair package with broken local URIs
+        def repair!
+          synchronize {
+            @uris -= @uris.collect { |uri| (uri.scheme == 'file' or uri.scheme == nil) and not File.exist? uri.path }
+            if @uris.empty?
+              raise RuntimeError, 'Lack of URIs for curent package!'
+            else
+             return true
+            end
+          }
+        end
+        
+    private
+        #Get RPM package attributes on package creation
+        def get_attributes
+            tmp_file_dir = '/tmp/' + SecureRandom.uuid
+            tmp_file_name = tmp_file_dir + '/package.rpm'
+            FileUtils.mkdir tmp_file_dir
+            raise RuntimeError, "Can't get package to determine attributes" unless duplicate_to tmp_file_dir, 'package.rpm'
+            @name, @version, @release, @architecture, @size = `rpm -q --queryformat '%{NAME} %{VERSION} %{RELEASE} %{ARCH} %{SIZE}' -p #{tmp_file_name} 2> /dev/null`.split ' '
+            if @name.nil? or @version.nil? or @release.nil? or @architecture.nil?
+                raise RuntimeError, "Can't parse name from #{tmp_file_name} by rpm -q command"
+            end
+            raise RuntimeError, "Unexpected file #{tmp_file_name} disappearing!" unless File.file? tmp_file_name
+            @file_size = File.size tmp_file_name
+            @digests = { :sha1 => Digest::SHA1.hexdigest(File.read tmp_file_name), :sha256 => Digest::SHA256.hexdigest(File.read tmp_file_name) }
+        ensure
+            deduplicate_undo tmp_file_dir
+            FileUtils.rm_rf tmp_file_dir
+        end
+        
+        #Check that local URIs point at actually exist file
+        def check_uris
+          @uris.each { |uri|
+            if (uri.scheme == 'file' or uri.scheme == nil) and not File.exist? uri.path
+              #MESSAGE MATTERS!
+              raise RuntimeError, "File #{uri.path} not exist, but mentioned in package"
+            end
+          }
+        end
+        
+        #Synchronize with useful check - enshure that every synced action preceeded with local uris check
+        def synchronize!
+          synchronize {
+            check_uris
+            yield if block_given?
+          }
+        end
+        
+    end
+    
+end

data/lib/repofactory.rb

@@ -0,0 +1,69 @@
+#Attempt to sync concurrent managing of multiple repositories
+
+module RPM
+    
+    class RepoFactory
+        
+        def initialize
+            @locker = Mutex.new
+            @locker.synchronize {
+                #Logging - now not needed
+                #@logger = Logging.logger['repo ' + @name]
+                #@logger.debug 'initializing'
+                @repositories = []
+            }
+        end
+        
+        #Synced repository adding
+        def add_repository repository
+            @locker.synchronize {
+                if repository.kind_of? RPM::Repository
+                    if (get_repository_by repository.name).is_a? RPM::Repository
+                        raise ArgumentError, "Repository with such name already exist!"
+                    end
+                    @repositories.push repository
+                    return true
+                else
+                    return false
+                end
+            }
+        end
+        
+        #Synced repository removal and destroying
+        def destroy_repository_by! name
+            repo = get_repository_by name
+            return false if repo.nil?
+            repo.destroy!
+            @locker.synchronize {
+                @repositories.delete repo
+            }
+            return true
+        end
+        
+        #the way to non-block reading
+        def get_repositories
+            @repositories.clone
+        end
+        
+        #Return repository by name
+        def get_repository_by name
+            unless name.kind_of? String
+                return nil
+            end
+            possible_repos = get_repositories.select { |repo| repo.name == name }
+            case possible_repos.count
+                when 1
+                    return possible_repos.first
+                else
+                    return nil
+            end
+        end
+        
+        #wrap over Array's each
+        def each
+            get_repositories.each { |repository| yield repository if block_given? }
+        end
+        
+    end
+    
+end

data/lib/repository.rb

@@ -0,0 +1,63 @@
+#Local RPM repository with packages
+
+module RPM
+  class Repository < Monitor
+    
+    require_relative 'repository_api'
+    require_relative 'repository_caching'
+    require_relative 'repository_metadata'
+    
+    attr_reader :base_dir
+    attr_reader :status
+    attr_reader :tmp_dir
+    attr_accessor :name
+    
+    @@statuses = {
+      :initializing => "Repository initialization in progress",
+      :rebuilding => "Rebuild repository metadata",
+      :destroying => "Repository destruction in progress",
+      :destroyed => "Repository removed and not available",
+      :ok => "Repository in normal state",
+      :cleaning => "Cleaning up temporary data"
+    }
+    
+    #Init repository in specified directory
+    def initialize base_dir, name = SecureRandom.uuid, rebuild_args = ""
+      #Call Monitor's contructor to initialize it
+      super()
+      synchronize {
+        @status = :initializing
+        #Human-readable repo name (default here, but available from the outside)
+        @name = name
+        #Logging
+        @logger = Logging.logger['repo ' + @name]
+        @logger.debug 'initializing'
+        #Local basic directory 
+        raise ArgumentError, "Base directory must be non-relative" if base_dir.nil? or not base_dir[/^\/.+/]
+        FileUtils::mkdir_p base_dir, { :mode => 0700 } unless Dir.exist? base_dir
+        @base_dir = Dir.new base_dir
+        File.chmod 0700, @base_dir.path
+        #Dir for some tmp files
+        Dir.mkdir @base_dir.path + "/tmp" unless Dir.exist? @base_dir.path + "/tmp"
+        @tmp_dir = Dir.new @base_dir.path + "/tmp"
+        #Init repository
+        FileUtils::mkdir "#{@base_dir.path}/Packages", { :mode => 0700 } unless @base_dir.entries.include? "Packages"
+        #Parts-relates initialization
+        initialization_metadata rebuild_args
+        initialization_cache
+        @logger.info "initialized"
+        #Making MD
+        rebuild
+        #Cache warming
+        validate_packages_cache
+        @status = :ok
+      }
+    end
+    
+    #Include substructures
+    include RPM::Repository::Metadata
+    include RPM::Repository::Caching
+    include RPM::Repository::API
+    
+  end
+end

data/lib/repository_api.rb

@@ -0,0 +1,152 @@
+#Public user api of RPM::Repository
+#Should'n changed frequently
+
+module RPM::Repository::API
+  
+  #Add single package to repository
+  def add_package! package
+    add_packages! [package]
+  end
+  
+  #Add packages to repository
+  def add_packages! packages
+    @logger.info "adding packages"
+    duplicated_packages = []
+    rebuild_with {
+      packages.each { |package|
+        raise ArgumentError, "Package expected, but #{package.class}" unless package.is_a? RPM::Package
+        raise RuntimeError, "Package already exist!" if contains? package
+        package.duplicate_to "#{@base_dir.path}/Packages"
+        #hack that add package to cache. Escape package brain splitting (two packages with the same file)
+        if package.digests[get_checksum_type.to_sym]
+          @packages_cache[package.digests[get_checksum_type.to_sym]] = package
+        end
+        duplicated_packages.push package
+      }
+    }
+  rescue Exception => e
+    rebuild_with {
+      duplicated_packages.each { |package|
+        package.deduplicate_undo "#{@base_dir.path}/Packages"
+      }
+    }
+    raise e
+  end
+  
+  #Remove package
+  def remove_package! package
+    @logger.info "removing package"
+    rebuild_with {
+      raise ArgumentError, "No such package in repository: #{package.get_default_name}" unless contains? package
+      get_own(package).deduplicate_undo @base_dir.path
+    }
+  end
+  
+  #Remove packages
+  #Return two arrays: {:removed => [...], :skipped => [...]}
+  def remove_packages! packages
+    @logger.info "removing packages"
+    rezult = {:removed => [], :skipped => []}
+    rebuild_with {
+      packages.each { |package|
+        if contains? package
+          get_own(package).deduplicate_undo @base_dir.path
+          rezult[:removed].push package
+        else
+          rezult[:skipped].push package
+          @logger.warn "Package #{package} skipped: no such package in repository"
+        end
+      }
+    }
+    return rezult
+  end
+  
+  #Remove packages by regex
+  #Return list with removed URL's
+  def parse_out_packages! name_expression
+    return remove_packages! get_packages_list_by(name_expression)
+  end
+  
+  #Return packages array that match parameter
+  def get_packages_list_by name_expression
+    get_packages_list.select{ |package| package.get_default_name[name_expression] }
+  end
+  
+  #Return true if repository contains package with same signature
+  def contains? alien_package
+    not get_packages_list.select{ |package| package.same_as? alien_package }.empty?
+  end
+  alias include? contains?
+  
+  #Try to expand package URIs with own
+  #Return true if package expanded, false otherwise
+  def assimilate! alien_package
+    if contains? alien_package
+      alien_package.join! get_own(alien_package)
+      synchronize { @packages_cache[alien_package.digests[get_checksum_type.to_sym]] = alien_package }
+      return true
+    else
+      return false
+    end
+  end
+  
+  #Remove repository files
+  def destroy!
+    @logger.info "destroying repo"
+    return @status if @status == :destroyed
+    synchronize {
+      @status = :destroying
+      clean
+      FileUtils::rm_rf @base_dir.path
+      @status = :destroyed
+    }
+  end
+  
+  #Cleanup temporary files
+  def clean
+    @logger.info "cleaning repo"
+    synchronize {
+      @status = :cleaning
+      FileUtils::rm_rf @tmp_dir.path
+      FileUtils::mkdir @tmp_dir.path
+      @status = :ok
+    }
+  end
+  
+  #Return template for YUM configuration
+  def get_local_conf
+    "
+    [#{@name}]
+    name=#{@name}
+    baseurl=file:///#{@base_dir.path}
+    gpgcheck=0
+    enabled=0
+    "
+  end
+  
+  #Return array of packages
+  def get_packages_list
+    synchronize { validate_packages_cache && get_packages_from_cache }
+  end
+  
+  #Public and safe realization of rebuilding - just rebuild repository
+  def rebuild
+    rebuild_with
+  end
+private
+  
+  #Return Package from current repository
+  #Method not public because it brings packages split-brain (two same package into runtime)
+  def get_own package
+    same_packages = get_packages_list.select { |repo_package| repo_package.same_as? package }
+    case same_packages.count
+    when 1
+      return same_packages.first
+    when 0
+      return nil
+    else
+      raise RuntimeError, "More than one #{package.get_default_name} package in repository"
+    end
+  end
+  
+end

data/lib/repository_caching.rb

@@ -0,0 +1,81 @@
+#RPM Repository Caching module
+#Mix for RPM::Repository to realize MD in-memory caching
+
+module RPM::Repository::Caching
+private
+  
+  #Return values from cached packages list - UNSYNC!!!
+  def get_packages_from_cache
+    @packages_cache.values
+  end
+  
+  #Check that current package list cache valid. if not - validate
+  def validate_packages_cache
+    @logger.debug "validating packages cache"
+    synchronize {
+      if repomd_cache_hit?
+        @logger.debug "MD cache hit"
+        return true
+      else
+        refresh_md_files_cache
+        refresh_packages_cache
+        return true
+      end
+    }
+  end
+  
+  #All below used just once. In case of multithreading usage - sync it by synchronize { body }
+  
+  #true means 100% hit in cached repository state - UNSYNC!!! - used once
+  def repomd_cache_hit?
+    if @repomd_cache.is_a? REXML::Document
+      return read_repomd_doc.elements["/repomd/revision"].text == @repomd_cache.elements["/repomd/revision"].text
+    else
+      return false
+    end
+  end
+  
+  #refreshing each md file cach
+  def refresh_md_files_cache
+    @repomd_cache = read_repomd_doc
+    @md_content_cache.keys.each { |type|
+      @md_content_cache[type] = read_md_doc type.to_s
+    }
+  end
+  
+  #Actually check packages cache: create new one and fill it with old cache and newly created one
+  def refresh_packages_cache
+      #save current cache state
+      prev_cache = @packages_cache
+      #clean current cache - used by other functions to refresh cache 
+      @packages_cache = {}
+      @md_content_cache[:primary].each_element('/metadata/package') { |package_md|
+        chksum = package_md.elements['checksum'].text
+        if prev_cache[chksum]
+          #get package from previous cache
+          @packages_cache[chksum] = prev_cache[chksum]
+        else
+          #construct new one
+          url = ''
+          if package_md.elements['location'].attributes["xml:base"]
+            url = URI::parse (package_md.elements['location'].attributes["xml:base"]+'/'+package_md.elements['location'].attributes["href"])
+          else
+            url = URI::parse ('file://' + @base_dir.path+'/'+package_md.elements['location'].attributes["href"])
+          end
+          @packages_cache[chksum] = RPM::Package.new url
+        end
+      }
+      @logger.debug 'package cache refreshed'
+  end
+  
+  #Cache-related part of initialization
+  def initialization_cache
+    #cached and parsed main XML MD file - repomd.xml
+    @repomd_cache = nil
+    #cached and parsed other XML documents
+    @md_content_cache = { :primary => nil }
+    #Repository Packages cache. Format: checksum => RPM::Package
+    @packages_cache = {}
+  end
+  
+end

data/lib/repository_metadata.rb

@@ -0,0 +1,72 @@
+#RPM Repository Metadata module
+#Mix for RPM::Repository to work witn MD
+
+module RPM::Repository::Metadata
+private
+  
+  #Rebuild repository with current configuration with additional args and block - synced
+  def rebuild_with args = ""
+    synchronize {
+      yield if block_given?
+      @status = :rebuilding
+      #TODO: add group file to rebuilding
+      unless system "createrepo -v --profile --update #{@base_dir.path} -s #{get_checksum_type} #{@extended_rebuild_args} #{args} &> '#{@tmp_dir.path}/rebuild-#{Time.now.to_s}'"
+        raise RuntimeError, "Can't rebuild repository #{@name}"
+      end
+      @status = :ok
+    }
+    @logger.info "rebuilded"
+    return true
+  end
+  
+  #read and parse repomd.xml - UNSYNC! - low level operation - no reason to make locks
+  def read_repomd_doc
+    REXML::Document.new File.open(@base_dir.path + "/repodata/repomd.xml")
+  end
+  
+  #return md file document by type - UNSYNC!
+  #assume that @repomd_cache is valid
+  def read_md_doc type
+    @logger.debug "getting #{type} md file"
+    if @repomd_cache.elements["/repomd/data[@type=\"#{type}\"]/location"].attributes["href"]
+      path_to_md_file = @base_dir.path+'/'+@repomd_cache.elements["/repomd/data[@type=\"#{type}\"]/location"].attributes["href"]
+      #raw_md_file = File.read path_to_md_file
+      case path_to_md_file
+      when /\.gz$/
+        Zlib::GzipReader.open(path_to_md_file) { |gz|
+          return REXML::Document.new gz.read
+        }
+      when /\.xml$/
+        return REXML::Document.new File.read path_to_md_file
+      else
+        raise RuntimeError, "Can't determine type of #{path_to_md_file}"
+      end
+    else
+      raise ArgumentError, 'No #{type} md file record in repomd.xml'
+    end
+  end
+  
+  #Get checksum type from repomd.xml to be used for correct rebuilding - UNSYNC!!!
+  def get_checksum_type
+    #@logger.debug "getting checksum type"
+    if File::exist? (@base_dir.path + "/repodata/repomd.xml")
+      repomd_doc = read_repomd_doc
+      if repomd_doc.elements["/repomd/data/checksum"].attributes["type"]
+        return repomd_doc.elements["/repomd/data/checksum"].attributes["type"]
+      else
+        #if no such field (improbable)
+        return "sha256"
+      end
+    else
+      #if this is fresh repository
+      return "sha256"
+    end
+  end
+  
+  #MD-related initialization part
+  def initialization_metadata rebuild_args
+    #Default additional arguments for createrepo
+    @extended_rebuild_args = rebuild_args
+  end
+  
+end

data/lib/rpm.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+module RPM
+    
+    require 'fileutils'
+    require 'securerandom'
+    require 'uri'
+    require 'net/http'
+    require 'rexml/document'
+    require 'logging'
+    
+    require_relative 'package'
+    require_relative 'repository'
+    require_relative 'repofactory'
+    
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: rpmrepository
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- nothing
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-09-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Gem to create/manage RPM repository(ies)/package(s)
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/package.rb
+- lib/repofactory.rb
+- lib/repository.rb
+- lib/repository_api.rb
+- lib/repository_caching.rb
+- lib/repository_metadata.rb
+- lib/rpm.rb
+homepage: 
+licenses:
+- Beerware
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Deal with RPM Repository/Package
+test_files: []