repo-mgr 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 977383f34636b8ba22a0e7a936869ccf69060691787f9ebb2ed4bad5826af13b
-  data.tar.gz: fb53030359f59ecad95adcf2a304e304fdd5c6ebed32c0c3c4e74f2e214d877d
+  metadata.gz: a977e25ed86d9763543cd445087e3d4e94e6ecf830e707ffb8c9dfb9c5d19541
+  data.tar.gz: a7d2ea6036a1d3a4fc3fc2c6f575e7dda9eaa7bc2db1d7bc0b2fe7dcdb459891
 SHA512:
-  metadata.gz: e78a4a1dafca417fa59fe93d458ee6cf8b4fa22ad48ef0bd60ae95258e02adc9c9a3c6d91c4516224576fe0ba6c9f8eebed1bed8f82777b85836cde089b6133e
-  data.tar.gz: 2a3c93c1f6768d6296682c17e5eacc4cf416f46d58e2c232962ca1b86bc7fff4fddff2e5c6441bb9392f823d1e9fcbe1bcb0e4d8f64b108307adbfbce71ff07e
+  metadata.gz: 8044c481083ac23582a3d3133dbd225ab874188217766e71d0e82e3a76bf325776ef6241064347ee5fb39df5e63d89acfe7c086f592314062fe1c3a391645ae8
+  data.tar.gz: d58d378485e4a41741360d308652e56b4dc9a3abd4be396840d1131c6daf187fdce3bde7de00eb2242ac0207dfacd912c591783e6fe38eaf91e266f4e77892d0

data/README.md

@@ -16,6 +16,8 @@ To simplify things:
  * aptly (which, kind of obviously, manages deb repositories) uses "stable" as distribution and "main" as component.
  * The git publisher uses the `main` branch for `sync` only.
 
+The requred aptly version is 1.4.0+ which may be installed from [aptly's deb repository](https://www.aptly.info/download/). The version available in Ubuntu 20.04 (1.3.0) doesn't support gpg2.
+
 ## Install
 
 ```bash
@@ -26,21 +28,21 @@ gem install repo-mgr
 rake install
 ```
 
-As repo-mgr is a frontend for other tools, there's dependencies which must be installed separately.
+As repo-mgr is a frontend for other tools, there are dependencies which must be installed separately. It is not compulsory to install all dependencies, only those needed for a particular use case. The purpose for each tool is explained by `check-depends`.
 
-To check which dependencies are required and their status:
+To check which dependencies are required based on use case and their status:
 
 ```bash
 repo-mgr check-depends
-+------------+--------+
-| Binary     | Status |
-+------------+--------+
-| aptly      | ✔      |
-| dpkg-sig   | ✔      |
-| createrepo | ✔      |
-| rpm        | ✔      |
-| git        | ✔      |
-+------------+--------+
++------------+--------+-----------------------+
+| Binary     | Status | Purpose               |
++------------+--------+-----------------------+
+| aptly      | ✔      | Manage apt repository |
+| dpkg-sig   | ✔      | Sign deb packages     |
+| createrepo | ✔      | Manage rpm repository |
+| rpm        | ✔      | Sign rpm packages     |
+| git        | ✔      | Use git publisher     |
++------------+--------+-----------------------+
 ```
 
 For managing deb repositories:
@@ -67,6 +69,12 @@ sudo apt install git
 
 You can get our build of createrepo from our [deb repository](https://deb.staker.ltd/).
 
+Pro tip: if the pkg signing fails because gpg can't open the output device, add this to your shell config:
+
+```bash
+export GPG_TTY=$(tty)
+```
+
 ## How to use
 
 ```bash
@@ -87,6 +95,20 @@ repo-mgr add-pkg --repo foo --path path/to/bar_0.0.1_amd64.deb
 
 # publish the repository to a remote - for git publisher this means doing git push
 repo-mgr sync --repo foo
+
+# download repo from remote e.g when changing development machines
+# repo-mgr upsert-repo (...) # needed only once if the machine is brand new
+## --url must point to the root of the remote repository
+## i.e where the pool and dists directories can be found for deb repos
+## --keyring is required for deb repositories; it is a file from
+## /usr/share/keyrings - the base path is automatically prepended
+## WARNING: the packages are not signed when imported via this method!
+repo-mgr dl-repo --repo foo --type deb --url https://apt.example.com --keyring foo-keyring.gpg
+
+## --arch must be specified as each rpm repo arch is self-contained
+## the remote repository gpg key must be imported into the user's keyring and trusted
+## prior to starting the import process from remote rpm repo
+repo-mgr dl-repo --repo foo --type rpm --url https://rpm.example.com --arch x86_64
 ```
 
 ## Migrating from v0.1

data/lib/repo_mgr/backends/deb.rb

@@ -14,14 +14,7 @@ module RepoMgr
       end
 
       def add_repo(name)
-        return if aptly_repos.include? name
-
-        cmd = "aptly -config=#{@aptly_config_file} repo create #{name}"
-        out, status = Open3.capture2e cmd
-
-        unless status.exitstatus.zero?
-          Tools.error "aptly repo create failed with:\n#{out}"
-        end
+        aptly "repo create #{name}" unless aptly_repos.include? name
 
         repo_config = @config.cfg[:repos][name]
 
@@ -34,6 +27,24 @@ module RepoMgr
         save_aptly_config
       end
 
+      def dl_repo(options)
+        name = options[:repo]
+        url = options[:url]
+        keyring = options[:keyring]
+
+        if keyring.nil?
+          Tools.error 'you must specify a keyring file for deb repo'
+        end
+
+        keyring = "/usr/share/keyrings/#{keyring}"
+
+        aptly "-keyring=#{keyring} mirror create #{name} #{url} stable main"
+        aptly "-keyring=#{keyring} mirror update #{name}", :output
+        aptly "repo import #{name} #{name} Name"
+        aptly "mirror drop #{name}"
+        aptly("repo search #{name}", :return).split.map { |e| "#{e}.deb" }
+      end
+
       def add_pkg(repo, pkg)
         sign_pkg repo, pkg
         repo_add repo, pkg
@@ -51,14 +62,14 @@ module RepoMgr
         return out if status.exitstatus.zero? || allow_fail
 
         Tools.error "unable to check package signature for #{pkg} - "\
-          "dpkg-sig returned:\n#{out}"
+                    "dpkg-sig returned:\n#{out}"
       end
 
       def sign_pkg(repo, pkg)
         signature = check_sig pkg, allow_fail: true
 
         unless signature[-6, 5] == 'NOSIG'
-          return puts "-- dpkg-sig returned:\n#{signature.first}"
+          return puts "-- dpkg-sig returned:\n#{signature.split.first}"
         end
 
         if @config.cfg[:repos][repo].nil?
@@ -74,18 +85,9 @@ module RepoMgr
       end
 
       def rebuild_pkg_list(repo)
-        out, status = Open3.capture2e "aptly -config=#{@aptly_config_file} "\
-          "-with-packages repo show #{repo}"
-
-        unless status.exitstatus.zero?
-          Tools.error "aptly repo show failed with with:\n#{out}"
+        aptly("-with-packages repo search #{repo}", :return).split.map do |e|
+          "#{e}.deb"
         end
-
-        pkgs = out.split
-        mark = pkgs.find_index 'Packages:'
-        pkgs = pkgs.drop(mark + 1)
-
-        pkgs.map { |e| "#{e}.deb" }
       end
 
       def export(repo)
@@ -121,8 +123,6 @@ module RepoMgr
           dependencyVerboseResolve: false,
           gpgDisableSign: false,
           gpgDisableVerify: false,
-          # despite the binary being gpg, this must spell gpg2, otherwise aptly
-          # defaults to gpg1 with less than impressive results
           gpgProvider: 'gpg2',
           downloadSourcePackages: false,
           skipLegacyPool: true,
@@ -136,36 +136,30 @@ module RepoMgr
       end
       # rubocop:enable Metrics/MethodLength
 
-      def aptly_repos
-        cmd = "aptly -raw -config=#{@aptly_config_file} repo list"
+      def aptly(cmd, output = nil)
+        cmd = "aptly -config=#{@aptly_config_file} #{cmd}"
         out, status = Open3.capture2e cmd
 
-        unless status.exitstatus.zero?
-          Tools.error "aptly repo list failed with:\n#{out}"
+        Tools.error "#{cmd} failed with:\n#{out}" unless status.exitstatus.zero?
+
+        case output
+        when :output
+          puts out
+        when :return
+          out
         end
+      end
 
-        out.split("\n")
+      def aptly_repos
+        aptly('-raw repo list', :return).split
       end
 
       def aptly_published_repos
-        cmd = "aptly -raw -config=#{@aptly_config_file} publish list"
-        out, status = Open3.capture2e cmd
-
-        unless status.exitstatus.zero?
-          Tools.error "aptly publish list failed with:\n#{out}"
-        end
-
-        out.split("\n")
+        aptly('-raw publish list', :return).split("\n")
       end
 
       def aptly_publish_drop(repo)
-        cmd = "aptly -config=#{@aptly_config_file} publish drop stable "\
-          "filesystem:#{repo}:"
-        out, status = Open3.capture2e cmd
-
-        return if status.exitstatus.zero?
-
-        Tools.error "aptly publish drop failed with:\n#{out}"
+        aptly "publish drop stable filesystem:#{repo}:"
       end
 
       def save_aptly_config
@@ -173,23 +167,12 @@ module RepoMgr
       end
 
       def repo_add(repo, pkg)
-        cmd = "aptly -config=#{@aptly_config_file} repo add #{repo} #{pkg}"
-        out, status = Open3.capture2e cmd
-
-        return if status.exitstatus.zero?
-
-        Tools.error "aptly repo add failed with:\n#{out}"
+        aptly "repo add #{repo} #{pkg}"
       end
 
       def repo_rm(repo, pkg)
         package = File.basename pkg, File.extname(pkg)
-        cmd = "aptly -config=#{@aptly_config_file} repo remove "\
-          "#{repo} #{package}"
-        out, status = Open3.capture2e cmd
-
-        return if status.exitstatus.zero?
-
-        Tools.error "aptly repo remove failed with:\n#{out}"
+        aptly "repo remove #{repo} #{package}"
       end
 
       def repo_publish(repo)
@@ -198,14 +181,8 @@ module RepoMgr
         end
 
         keyid = @config.cfg[:repos][repo][:keyid]
-        cmd = "aptly -config=#{@aptly_config_file} -distribution=stable "\
-          "-gpg-key=#{keyid} publish repo #{repo} filesystem:#{repo}:"
-
-        out, status = Open3.capture2e cmd
-
-        return if status.exitstatus.zero?
-
-        Tools.error "aptly publish repo failed with:\n#{out}"
+        aptly "-distribution=stable -gpg-key=#{keyid} publish repo #{repo} "\
+              "filesystem:#{repo}:"
       end
     end
   end

data/lib/repo_mgr/backends/rpm.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: false
 
+require 'zlib'
 require 'open3'
 require 'gpgme'
+require 'digest'
+require 'faraday'
+require 'nokogiri'
+require 'stringio'
 require 'fileutils'
 
 module RepoMgr
@@ -27,6 +32,30 @@ module RepoMgr
         sync_repo repo
       end
 
+      def dl_repo(options)
+        name = options[:repo]
+        url = options[:url]
+        arch = options[:arch]
+
+        Tools.error 'you must specify an arch name for rpm repo' if arch.nil?
+
+        tmpdir = "/tmp/#{name}/#{arch}"
+        dest_dir = "#{@config.cfg_dir}/rpms/#{name}/#{arch}"
+        make_dirs tmpdir, dest_dir
+
+        repomd = dl_repomd url, arch, tmpdir
+        pkgs = dl_primary url, arch, tmpdir, repomd
+
+        pkgs.each do |hash, file|
+          dl_pkg hash, url, arch, file, tmpdir
+          copy_pkg "#{tmpdir}/#{file}", dest_dir
+        end
+
+        sync_repo name
+
+        pkgs.values
+      end
+
       def remove_pkg(repo, pkg)
         name = File.basename pkg
         arch = extract_arch pkg
@@ -43,7 +72,7 @@ module RepoMgr
         return out if status.exitstatus.zero?
 
         Tools.error "unable to check package signature for #{pkg} - "\
-          "rpm -K returned:\n#{out}"
+                    "rpm -K returned:\n#{out}"
       end
 
       def sign_pkg(repo, pkg)
@@ -114,7 +143,7 @@ module RepoMgr
         return if status.exitstatus.zero?
 
         Tools.error "unable to create repo for #{arch} - createrepo "\
-          "returned:\n#{out}"
+                    "returned:\n#{out}"
       end
 
       def sign_repo(repo)
@@ -131,6 +160,83 @@ module RepoMgr
 
         File.write 'repomd.xml.asc', signature
       end
+
+      def faraday_dl(url, tmpdir, file = File.basename(url))
+        puts "-- Download #{file}"
+        File.write "#{tmpdir}/#{file}", Faraday.get(url).body
+      end
+
+      def dl_repomd(url, arch, tmpdir)
+        faraday_dl "#{url}/#{arch}/repodata/repomd.xml", tmpdir
+        faraday_dl "#{url}/#{arch}/repodata/repomd.xml.asc", tmpdir
+
+        valid = false
+        crypto = GPGME::Crypto.new armor: true
+        sig = GPGME::Data.new File.read "#{tmpdir}/repomd.xml.asc"
+        data = File.read "#{tmpdir}/repomd.xml"
+        crypto.verify(sig, signed_text: data) do |sign|
+          valid = sign.valid?
+        end
+
+        unless valid == true
+          Tools.error "unable to check signature for #{tmpdir}/repomd.xml"
+        end
+
+        Nokogiri::XML data
+      end
+
+      def extract_pkgs(primary)
+        pkgs = {}
+
+        primary.css('package').each do |pkg|
+          pkg_hash = pkg.at('checksum[type=sha256]').text
+          pkgs[pkg_hash] = pkg.at('location')['href']
+        end
+
+        pkgs
+      end
+
+      def dl_primary(url, arch, tmpdir, repomd)
+        hash = repomd.at('data[type=primary]').at('checksum').text
+        primary = "#{url}/#{arch}/repodata/#{hash}-primary.xml.gz"
+
+        faraday_dl primary, tmpdir, 'primary.xml.gz'
+
+        primary_xml_gz = "#{tmpdir}/primary.xml.gz"
+        fl_hash = Digest::SHA256.hexdigest(File.read(primary_xml_gz))
+
+        unless fl_hash == hash
+          Tools.error "failed hash check for #{tmpdir}/primary.xml.gz"
+        end
+
+        primary_xml_gz = File.read("#{tmpdir}/primary.xml.gz")
+        primary = Zlib::GzipReader.new(StringIO.new(primary_xml_gz)).read
+        File.write("#{tmpdir}/primary.xml", primary)
+
+        extract_pkgs Nokogiri::XML(primary)
+      end
+
+      def dl_pkg(hash, url, arch, file, tmpdir)
+        unless File.exist? "#{tmpdir}/#{file}"
+          faraday_dl "#{url}/#{arch}/#{file}", tmpdir
+        end
+
+        fl_hash = Digest::SHA256.hexdigest(File.read("#{tmpdir}/#{file}"))
+
+        return if fl_hash == hash
+
+        Tools.error "failed hash check for #{tmpdir}/#{file}"
+      end
+
+      def copy_pkg(file, destdir)
+        puts "-- Copy to repo-mgr #{File.basename(file)}"
+        FileUtils.cp file, destdir
+      end
+
+      def make_dirs(tmpdir, dest_dir)
+        FileUtils.mkdir_p tmpdir
+        FileUtils.mkdir_p dest_dir
+      end
     end
   end
 end

data/lib/repo_mgr/cli.rb

@@ -29,16 +29,19 @@ module RepoMgr
 
     def check_depends
       rows = []
-
-      %w[aptly dpkg-sig createrepo rpm git].each do |bin_dep|
-        rows << if Tools.which bin_dep
-                  [bin_dep, '✔'.green]
-                else
-                  [bin_dep, '✘'.red]
-                end
+      deps = {
+        'aptly' => 'Manage apt repository',
+        'dpkg-sig' => 'Sign deb packages',
+        'createrepo' => 'Manage rpm repository',
+        'rpm' => 'Sign rpm packages',
+        'git' => 'Use git publisher'
+      }
+
+      deps.each do |bin_dep, purpose|
+        rows << which_depends(bin_dep, purpose)
       end
 
-      puts Terminal::Table.new headings: %w[Binary Status], rows: rows
+      puts Terminal::Table.new headings: %w[Binary Status Purpose], rows: rows
     end
 
     desc 'upsert-repo', 'Create/Update new repository'
@@ -82,6 +85,28 @@ module RepoMgr
       )
     end
 
+    desc 'dl-repo', 'Download repository from remote endpoint'
+    option :repo, type: :string, required: true, aliases: %w[-r],
+                  desc: 'The repository to download packages into'
+    option :type, type: :string, required: true, aliases: %w[-t],
+                  enum: types, desc: 'Repository type'
+    option :url, type: :string, required: true, aliases: %w[-u],
+                 desc: 'The URL for the remote repository'
+    option :keyring, type: :string, aliases: %w[-k],
+                     desc: 'Name of keyring file if required to auth repository'
+    option :arch, type: :string, aliases: %w[-a],
+                  desc: 'Pkg arch if multiple arch are supported by repo'
+
+    def dl_repo
+      backend, config = load_backend options[:type]
+
+      pkgs = backend.dl_repo options
+
+      pkgs.each do |pkg|
+        config.add_pkg options[:repo], pkg
+      end
+    end
+
     desc 'add-pkg', 'Add package to repository'
     option :repo, type: :string, required: true, aliases: %w[-r],
                   desc: 'The repository to add the package to'
@@ -100,7 +125,7 @@ module RepoMgr
       end
 
       puts "-- Added #{File.basename(options[:path])} to "\
-        "#{options[:repo]} repository"
+           "#{options[:repo]} repository"
     end
 
     desc 'list-pkgs', 'List repository packages'
@@ -132,7 +157,7 @@ module RepoMgr
       config.remove_pkg options[:repo], options[:path]
 
       puts "-- Removed #{File.basename(options[:path])} from "\
-        "#{options[:repo]} repository"
+           "#{options[:repo]} repository"
     end
 
     desc 'check-sig', 'Check package signature'
@@ -190,8 +215,15 @@ module RepoMgr
 
     private
 
+    def which_depends(bin_dep, purpose)
+      return [bin_dep, '✔'.green, purpose] if Tools.which bin_dep
+
+      [bin_dep, '✘'.red, purpose]
+    end
+
     def load_backend(path)
       type = File.extname(path).strip.downcase[1..-1]
+      type = path if type.nil?
 
       unless CLI.types.include? type
         Tools.error "unsupported package type #{type}"

data/lib/repo_mgr/config.rb

@@ -68,7 +68,7 @@ module RepoMgr
     def remove_pkg(repo, path)
       if @cfg[:repos][repo].nil?
         Tools.error "unable to remove packages from #{repo} "\
-          '- repo does not exist'
+                    '- repo does not exist'
       end
 
       @cfg[:packages] ||= {}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: repo-mgr
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Ștefan Rusu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-17 00:00:00.000000000 Z
+date: 2022-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colored
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.3
 - !ruby/object:Gem::Dependency
   name: git
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.3
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
@@ -81,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: jeweler
+  name: juwelier
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -176,7 +204,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.32
 signing_key: 
 specification_version: 4
 summary: deb and rpm repository manager