s3repo 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 037f096771d38b162f01efabbd4f21a3d38d2e54
-  data.tar.gz: 2639022f0ac61664ed66c9bdd49b28e8b3dc4d13
+  metadata.gz: 77fb43c11e189c5bfc7c347fa6b0d2efe566f9ff
+  data.tar.gz: 2ce120f3afd0e0b63a6d2f63c84822c3b974bc95
 SHA512:
-  metadata.gz: 40e4eba132b5c8374a4c5da86dc5043aca1700daf69c616ee91f018bd214e010cf64bda56b0c1187388252ffc21d95e328be2b68daf1f10d0fdf0d940c2f6978
-  data.tar.gz: d09e26179074b0c678e798bc814b1a2e4277bea5738d914c7af0c64c306896f4da46423dc03ec2b8af3f0917d432f707f8e8d38614598d50c7fe2dddc6cd7067
+  metadata.gz: c0c201ab4c0e8eefc8e7d2ba2490d92fdc9eb6e989c2b1a5cec9436580ca8c0ef5bf874052fac4056725343a9ccfbe5f96747cf1931297d81df5cee6ee251cf5
+  data.tar.gz: b2acfdd8d6386b69bddbd772d836b40d825a58b8adcbdf6a0ee64c484e397c94cbaf5d2e8cb99f303030bb031f24ed2e31e2bceab89fa801b587cea1fbfa0da7

data/.circle-ruby

@@ -0,0 +1,3 @@
+2.4.2
+2.3.5
+2.2.8

data/.prospectus

@@ -1,11 +1,25 @@
+Prospectus.extra_dep('file', 'prospectus_circleci')
+
+my_slug = 'amylum/s3repo'
+
 item do
-  expected do
-    static
-    set 'green'
-  end
+  noop
 
-  actual do
-    gemnasium
-    slug 'amylum/server'
+  deps do
+    item do
+      name 'gems'
+
+      expected do
+        static
+        set 'green'
+      end
+
+      actual do
+        gemnasium
+        slug my_slug
+      end
+    end
   end
+
+  extend ProspectusCircleci::Build.new(my_slug)
 end

data/.rubocop.yml

@@ -1,2 +1,5 @@
-Encoding:
-    Enabled: false
+inherit_gem:
+  goodcop: .rubocop.yml
+AllCops:
+  Include:
+  - 'bin/*'

data/Gemfile

@@ -1,4 +1,3 @@
 source 'https://rubygems.org'
 
 gemspec
-

data/README.md

@@ -3,7 +3,7 @@ s3repo
 
 [![Gem Version](https://img.shields.io/gem/v/s3repo.svg)](https://rubygems.org/gems/s3repo)
 [![Dependency Status](https://img.shields.io/gemnasium/amylum/s3repo.svg)](https://gemnasium.com/amylum/s3repo)
-[![Build Status](https://img.shields.io/circleci/project/amylum/s3repo.svg)](https://circleci.com/gh/amylum/s3repo)
+[![Build Status](https://img.shields.io/circleci/project/amylum/s3repo/master.svg)](https://circleci.com/gh/amylum/s3repo)
 [![Coverage Status](https://img.shields.io/codecov/c/github/amylum/s3repo.svg)](https://codecov.io/github/amylum/s3repo)
 [![Code Quality](https://img.shields.io/codacy/eef971ff937642219c1d4094001c33e7.svg)](https://www.codacy.com/app/akerl/s3repo)
 [![MIT Licensed](https://img.shields.io/badge/license-MIT-green.svg)](https://tldrlegal.com/license/mit-license)

data/Rakefile

@@ -7,8 +7,7 @@ RSpec::Core::RakeTask.new(:spec)
 
 desc 'Run Rubocop on the gem'
 RuboCop::RakeTask.new(:rubocop) do |task|
-  task.patterns = ['lib/**/*.rb', 'spec/**/*.rb', 'bin/*']
   task.fail_on_error = true
 end
 
-task default: [:spec, :rubocop, :build, :install]
+task default: %i[spec rubocop build install]

data/bin/s3repo

@@ -2,49 +2,72 @@
 
 require 's3repo'
 require 'mercenary'
+require 'yaml'
+require 'cymbal'
 
 def find(pattern, limits)
   Dir.glob(pattern).select { |x| !limits || limits.include?(File.dirname(x)) }
 end
 
+def common_opts(cmd)
+  cmd.option :config_file, '-c', '--config FILE', 'Path of config file'
+end
+
+def load_config(file)
+  file ||= 'config.yml'
+  file = File.expand_path(file)
+  raise("Config file not found: #{file}") unless File.exist? file
+  Cymbal.symbolize YAML.safe_load(File.read(file))
+end
+
+def repo(options)
+  config = load_config(options[:config_file])
+  S3Repo.new(config)
+end
+
+# rubocop:disable Metrics/BlockLength
 Mercenary.program(:s3repo) do |p|
   p.version S3Repo::VERSION
   p.description 'Package management tool for Archlinux repos'
   p.syntax 's3repo <subcommand> [args]'
 
   p.command(:build) do |c|
+    common_opts(c)
     c.syntax 'build [package...]'
     c.description 'Build package files from PKGBUILDs'
 
-    c.action do |args, _|
-      S3Repo.new.build_packages find('*/PKGBUILD', args), ENV['MAKEPKG_FLAGS']
+    c.action do |args, opts|
+      repo(opts).build_packages find('*/PKGBUILD', args)
     end
   end
 
   p.command(:upload) do |c|
+    common_opts(c)
     c.syntax 'upload [package...]'
     c.description 'Upload packages to repo'
 
-    c.action do |args, _|
-      S3Repo.new.add_packages find('*/*.pkg.tar.xz', args)
+    c.action do |args, opts|
+      repo(opts).add_packages find('*/*.pkg.tar.xz', args)
     end
   end
 
   p.command(:remove) do |c|
+    common_opts(c)
     c.syntax 'remove [package...]'
     c.description 'Remove packages from repo DB'
 
-    c.action do |args, _|
-      S3Repo.new.remove_packages args
+    c.action do |args, opts|
+      repo(opts).remove_packages args
     end
   end
 
   p.command(:prune) do |c|
+    common_opts(c)
     c.syntax 'prune'
     c.description 'Prune orphaned files from the repo'
 
-    c.action do
-      S3Repo.new.prune_files
+    c.action do |_, opts|
+      repo(opts).prune_files
     end
   end
 

data/circle.yml

@@ -1,8 +1,11 @@
+machine:
+  environment:
+    AWS_ACCESS_KEY_ID: '1'
+    AWS_SECRET_ACCESS_KEY: '1'
 dependencies:
   override:
-    - 'rvm-exec 2.1.6 bundle install'
-    - 'rvm-exec 2.2.2 bundle install'
+  - 'for i in $(cat .circle-ruby) ; do rvm-exec $i gem update --system --no-doc || exit 1 ; done'
+  - 'for i in $(cat .circle-ruby) ; do rvm-exec $i bundle install || exit 1 ; done'
 test:
   override:
-    - 'rvm-exec 2.1.6 bundle exec rake'
-    - 'rvm-exec 2.2.2 bundle exec rake'
+  - 'for i in $(cat .circle-ruby) ; do rvm-exec $i bundle exec rake || exit 1 ; done'

data/lib/s3repo.rb

@@ -13,7 +13,9 @@ end
 
 require 's3repo/version'
 require 's3repo/base'
+require 's3repo/signer'
 require 's3repo/metadata'
 require 's3repo/cache'
 require 's3repo/client'
+require 's3repo/templates'
 require 's3repo/repo'

data/lib/s3repo/base.rb

@@ -13,23 +13,19 @@ module S3Repo
     def run(cmd)
       results = `#{cmd} 2>&1`
       return results if $CHILD_STATUS.success?
-      fail "Failed running #{cmd}:\n#{results}"
+      raise "Failed running #{cmd}:\n#{results}"
     end
 
     def bucket
-      @bucket ||= @options[:bucket] || ENV['S3_BUCKET']
-      return @bucket if @bucket
-      fail('No bucket given')
+      @options[:bucket] || raise('No bucket given')
     end
 
     def client
-      @client ||= @options[:client] || Client.new(bucket: bucket)
+      @options[:client] ||= Client.new(@options)
     end
 
     def file_cache
-      @file_cache ||= @options[:file_cache] || Cache.new(
-        client: client, tmpdir: @options[:tmpdir]
-      )
+      @file_cache ||= @options[:file_cache] || Cache.new(@options)
     end
   end
 end

data/lib/s3repo/cache.rb

@@ -7,7 +7,6 @@ module S3Repo
   # Cache object, stores S3 objects on disk
   class Cache < Base
     TMPDIRS = [
-      ENV['S3REPO_TMPDIR'],
       ENV['TMPDIR'],
       Dir.tmpdir,
       '/tmp/s3repo'
@@ -18,20 +17,18 @@ module S3Repo
       [partialdir, cachedir].each { |x| FileUtils.mkdir_p x }
     end
 
-    def serve(key, refresh = true)
-      File.open(download(key, refresh), &:read)
-    rescue Aws::S3::Errors::NoSuchKey
-      nil
-    end
-
-    private
-
     def download(key, refresh = true)
       path = expand_path key
       get_object(key, path) if refresh || !cached?(path)
       path
     end
 
+    private
+
+    def file_cache
+      raise('Tried to call file_cache recursively')
+    end
+
     def expand_path(key)
       File.absolute_path(key, cachedir)
     end
@@ -71,7 +68,13 @@ module S3Repo
     end
 
     def tmpdir
-      @tmpdir ||= File.absolute_path(@options[:tmpdir] || TMPDIRS.compact.first)
+      @tmpdir ||= Dir.mktmpdir 's3repo', tmpdir_root
+    end
+
+    def tmpdir_root
+      @tmpdir_root ||= File.absolute_path(
+        @options[:tmpdir] || TMPDIRS.compact.first
+      )
     end
   end
 end

data/lib/s3repo/client.rb

@@ -5,21 +5,33 @@ module S3Repo
   # AWS API client
   class Client
     def initialize(params = {})
-      @api = Aws::S3::Client.new
-      @defaults = params
+      @options = params
+      @api = Aws::S3::Client.new(region: region)
     end
 
-    def respond_to?(method, include_all = false)
+    def respond_to_missing?(method, include_all = false)
       @api.respond_to?(method, include_all) || super
     end
 
-    def upload!(key, path)
+    def upload(key, body)
       puts "Uploading #{key}"
-      put_object key: key, body: File.open(path), &:read
+      put_object key: key, body: body
+    end
+
+    def upload_file(key, path)
+      upload(key, File.open(path).read)
     end
 
     private
 
+    def region
+      @options[:region] || raise('AWS region not set')
+    end
+
+    def bucket
+      @options[:bucket] || raise('Bucket not set')
+    end
+
     def method_missing(method, *args, &block)
       return super unless @api.respond_to?(method)
       define_singleton_method(method) do |*singleton_args|
@@ -30,10 +42,11 @@ module S3Repo
     end
 
     def build_params(args)
-      fail 'Too many arguments given' if args.size > 1
+      raise 'Too many arguments given' if args.size > 1
       params = args.first || {}
-      fail 'Argument must be a hash' unless params.is_a? Hash
-      @defaults.dup.merge!(params)
+      raise 'Argument must be a hash' unless params.is_a? Hash
+      params[:bucket] ||= bucket
+      params
     end
   end
 end

data/lib/s3repo/metadata.rb

@@ -5,11 +5,6 @@ module S3Repo
   ##
   # Metadata object, represents repo's DB file
   class Metadata < Base
-    def initialize(params = {})
-      super
-      FileUtils.mkdir_p db_dir
-    end
-
     def add_packages(paths)
       @db_path = nil
       paths.each do |path|
@@ -29,38 +24,31 @@ module S3Repo
     end
 
     def update!
-      sign_db if ENV['S3REPO_SIGN_DB']
-      client.upload!('repo.db', db_path)
+      sign_db if @options[:sign_db]
+      client.upload_file('repo.db', db_path)
+      client.upload_file('repo.db.tar.xz', db_path)
     end
 
     def packages
       return @packages if @packages
-      cmd = "tar tf #{db_path}"
+      cmd = "bsdtar tf #{db_path}"
       @packages = run(cmd).split.map { |x| x.split('/').first }.uniq
     end
 
     private
 
-    def sign_db
-      run "gpg --detach-sign --use-agent #{db_path}"
-      client.upload!('repo.db.sig', "#{db_path}.sig")
-    end
-
-    def db_path
-      @db_path ||= download_db
+    def signer
+      @options[:signer] ||= Signer.new(@options)
     end
 
-    def download_db
-      tmpfile = Tempfile.create(['repo', '.db.tar.xz'], db_dir)
-      tmpfile << file_cache.serve('repo.db')
-      tmpfile.close
-      tmpfile.path
+    def sign_db
+      sig_path = signer.sign(db_path)
+      client.upload_file('repo.db.sig', sig_path)
+      client.upload_file('repo.db.tar.xz.sig', sig_path)
     end
 
-    def db_dir
-      @db_dir ||= File.absolute_path(
-        @options[:tmpdir] || Cache::TMPDIRS.compact.first
-      )
+    def db_path
+      @db_path ||= file_cache.download('repo.db.tar.xz')
     end
   end
 end

data/lib/s3repo/repo.rb

@@ -5,25 +5,23 @@ module S3Repo
   ##
   # Repo object, represents an Arch repo inside an S3 bucket
   class Repo < Base
-    def initialize(params = {})
-      super
-    end
-
-    def build_packages(paths, makepkg_flags = '')
+    def build_packages(paths)
       paths.each do |path|
         dir = File.dirname(path)
         puts "Building #{File.basename(dir)}"
-        Dir.chdir(dir) { run "makepkg #{makepkg_flags}" }
+        Dir.chdir(dir) { run "makepkg #{@options[:makepkg_flags]}" }
       end
     end
 
     def add_packages(paths)
       paths.select! { |path| upload_package(path) }
       metadata.add_packages(paths) unless paths.empty?
+      templates.update! unless paths.empty?
     end
 
     def remove_packages(packages)
       metadata.remove_packages(packages)
+      templates.update!
     end
 
     def prune_files
@@ -47,19 +45,15 @@ module S3Repo
       !packages.find { |x| x.key == key }.nil?
     end
 
-    def serve(key)
-      refresh = !key.match(/\.pkg\.tar\.xz$/)
-      file_cache.serve(key, refresh)
-    end
-
     private
 
     def upload_package(path)
       key = File.basename(path)
-      sig_key, sig_path = [key, path].map { |x| x + '.sig' }
       return false if include? key
-      client.upload!(sig_key, sig_path) if ENV['S3REPO_SIGN_PACKAGES']
-      client.upload!(key, path)
+      sig_path = signer.sign(path)
+      sig_key = key + '.sig'
+      client.upload_file(sig_key, sig_path) if @options[:sign_packages]
+      client.upload_file(key, path)
       true
     end
 
@@ -70,7 +64,15 @@ module S3Repo
     end
 
     def metadata
-      @metadata ||= Metadata.new(client: client, file_cache: file_cache)
+      @options[:metadata] ||= Metadata.new(@options)
+    end
+
+    def signer
+      @options[:signer] ||= Signer.new(@options)
+    end
+
+    def templates
+      @templates ||= Templates.new(@options)
     end
 
     def package_cache