package_cloud 0.0.3 → 0.3.15

data/lib/package_cloud/cli/entry.rb

@@ -0,0 +1,397 @@
+require "cgi"
+require "uri"
+
+module PackageCloud
+  module CLI
+    JAVA_EXTS = ["jar", "aar", "war"]
+    PY_EXTS = ["gz", "bz2", "z", "tar", "egg-info", "whl", "egg"]
+    ANYFILE_EXTS = ["asc"]
+    NODE_EXTS = []
+    AMBIGUOUS_EXTS = ["tgz", "apk", "zip"]
+    SUPPORTED_EXTS = JAVA_EXTS + PY_EXTS + ANYFILE_EXTS + AMBIGUOUS_EXTS + ["gem", "deb", "rpm", "dsc"]
+
+    class Entry < Base
+      desc "repository SUBCMD ...ARGS", "manage repositories"
+      subcommand "repository", Repository
+
+      desc "distro SUBCMD ...ARGS", "manage repositories"
+      subcommand "distro", Distro
+
+      desc "master_token SUBCMD ...ARGS", "manage master tokens"
+      subcommand "master_token", MasterToken
+
+      desc "read_token SUBCMD ...ARGS", "manage read tokens"
+      subcommand "read_token", ReadToken
+
+      desc "gpg_key SUBCMD ...ARGS", "manage GPG keys"
+      subcommand "gpg_key", GpgKey
+
+      desc "promote user/repo[/distro/version] [@scope/]package_name user/destination_repo",
+           "promotes a package from user/repo [in dist/version] to user/destination_repo [also in dist/version]"
+      def promote(source_repo_desc, package_name, dest_repo_desc)
+        repo_name = source_repo_desc.split("/")[0..1].join("/")
+        dist = expand_dist_shortcut(source_repo_desc.split("/")[2..3].join("/"))
+
+        dest_repo_name = expand_dist_shortcut(dest_repo_desc.split("/")[0..1].join("/"))
+
+        if dist == "" && package_name =~ /\.gem$/
+          dist = "gems"
+        end
+
+        print "Looking for source repository at #{repo_name}... "
+        src_repo = client.repository(source_repo_desc)
+        print "success!\n"
+
+        print "Looking for destination repository at #{dest_repo_name}... "
+        repo = client.repository(dest_repo_desc)
+        print "success!\n"
+
+        if package_name.include?('@')
+          scope, unscoped_package_name = package_scope(package_name)
+
+          print "Attempting to promote scoped package #{scope}/#{unscoped_package_name} from #{repo_name}/#{dist} to #{dest_repo_name}..."
+          packages = src_repo.promote(dist, unscoped_package_name, dest_repo_name, scope)
+        else
+          print "Attempting to promote #{repo_name}/#{dist}/#{package_name} to #{dest_repo_name}..."
+          packages = src_repo.promote(dist, package_name, dest_repo_name)
+        end
+        puts "done!".color(:green)
+
+        if dist == "node/1"
+          puts "WARNING: This Node.js package will NOT be downloadable by clients until a dist tag is created. Read more: https://packagecloud.io/docs/#node_promote".color(:yellow)
+        end
+      end
+
+      desc "yank user/repo[/distro/version] [@scope/]package_name",
+           "yank package from user/repo [in dist/version]"
+      def yank(repo_desc, package_name)
+        ARGV.clear # otherwise gets explodes
+
+        # strip os/dist
+        repo_name = repo_desc.split("/")[0..1].join("/")
+        dist = expand_dist_shortcut(repo_desc.split("/")[2..3].join("/"))
+
+        if dist == "" && package_name =~ /\.gem$/
+          dist = "gems"
+        end
+
+        print "Looking for repository at #{repo_name}... "
+        repo = client.repository(repo_desc)
+        print "success!\n"
+
+        if package_name.include?('@')
+          scope, unscoped_package_name = package_scope(package_name)
+          print "Attempting to yank scoped package at #{repo_name}/#{dist} #{package_name}..."
+          repo.yank(dist, unscoped_package_name, scope)
+        else
+          print "Attempting to yank package at #{repo_name}/#{dist}/#{package_name}..."
+          repo.yank(dist, package_name)
+        end
+        puts "done!".color(:green)
+
+        if dist == "node/1"
+          puts "WARNING: Deleting Node.js packages can have unexpected side effects with dist tags. Read more: https://packagecloud.io/docs/#node_delete".color(:yellow)
+        end
+      end
+
+      desc "push user/repo[/distro/version] /path/to/packages",
+           "Push package(s) to repository (in distro/version, if required). Optional settings shown above."
+
+      option "skip-file-ext-validation", :type => :boolean,
+                                         :desc => "Skip checking validation of the file extension. Package upload will be attempted even if the extension is unrecognized."
+
+      option "yes", :type => :boolean,
+                    :desc => "Automatically answer 'yes' prompted during package push. Useful for automating uploads."
+
+      option "skip-errors", :type => :boolean,
+                            :desc => "Skip errors encountered during a package push and continue pushing the next package."
+
+      option "skip-duplicates", :type => :boolean,
+                                :desc => "Skip packages which already exist and continue pushing the next package."
+
+      option "coordinates", :type => :string,
+                            :desc => "Specify the exact maven coordinates to use for a JAR. Useful for JARs without coordinates, 'fat JARs', and WARs."
+
+      option "upload-before-push",
+        :type => :boolean,
+        :desc => "Upload package before push",
+        :default => false
+
+      def push(repo, package_file, *package_files)
+        total_time = Benchmark.measure do
+          ARGV.clear # otherwise gets explodes
+          package_files << package_file
+
+          exts = package_files.map { |f| f.split(".").last }.uniq
+
+          if package_files.length > 1 && exts.length > 1
+            abort("You can't push multiple packages of different types at the same time.\nFor example, use *.deb to push all your debs at once.".color(:red))
+          end
+
+          invalid_packages = package_files.select do |f|
+            !SUPPORTED_EXTS.include?(f.split(".").last.downcase)
+          end
+
+          if JAVA_EXTS.include?(exts.first.downcase) && options.has_key?("coordinates")
+            puts "Using coordinates #{options["coordinates"].color(:yellow)}"
+          end
+
+          if !options.has_key?("skip-file-ext-validation") && invalid_packages.any?
+            message = "I don't know how to push these packages:\n\n".color(:red)
+            invalid_packages.each do |p|
+              message << "  #{p}\n"
+            end
+            message << "\npackage_cloud only supports node.js, deb, gem, java, python, "\
+                       "rpm, alpine, helm, anyfile (.zip, .asc) packages".color(:red)
+            abort(message)
+          end
+
+          if !options.has_key?("yes") && exts.first == "gem" && package_files.length > 1
+            answer = get_valid("Are you sure you want to push #{package_files.length} packages? (y/n)") do |s|
+              s == "y" || s == "n"
+            end
+
+            if answer != "y"
+              abort("Aborting...".color(:red))
+            end
+          end
+
+          validator = Validator.new(client)
+          if PY_EXTS.include?(exts.first.downcase)
+            dist_id   = validator.distribution_id(repo, package_files, 'py')
+          elsif ANYFILE_EXTS.include?(exts.first.downcase)
+            dist_id   = validator.distribution_id(repo, package_files, 'anyfile')
+          elsif NODE_EXTS.include?(exts.first.downcase)
+            dist_id   = validator.distribution_id(repo, package_files, 'node')
+          elsif JAVA_EXTS.include?(exts.first.downcase)
+            abort_if_snapshot!(package_files)
+            dist_id   = validator.distribution_id(repo, package_files, 'jar')
+          elsif AMBIGUOUS_EXTS.include?(exts.first.downcase)
+            dist_id   = validator.distribution_id_from_repo_url(repo, package_files)
+          else
+            dist_id   = validator.distribution_id(repo, package_files, exts.first)
+          end
+
+          # strip os/dist
+          split_repo = repo.split("/")[0..1].join("/")
+
+          print "Looking for repository at #{split_repo}... "
+          client_repo = nil
+          measurement = Benchmark.measure do
+            client_repo = client.repository(split_repo)
+          end
+          print "success!\n"
+          $logger.debug("repository lookup request timing: #{measurement}")
+
+          package_files.each do |f|
+            url = nil
+            files = nil
+            urls = nil
+            ext = f.split(".").last
+
+            if ext == "dsc"
+              print "Checking source package #{f}... "
+              files = parse_and_verify_dsc(client_repo, f, dist_id)
+            end
+
+            if options["upload-before-push"]
+              print "Uploading #{f}... "
+              upload = client_repo.create_upload
+              begin
+                response = RestClient::Request.execute(
+                  :method => upload.fetch("form").fetch("method", "POST"),
+                  :url => upload.fetch("form").fetch("url"),
+                  :timeout => nil,
+                  :payload => upload.fetch("form").fetch("data", {}).merge(
+                    upload.fetch("form").fetch("file_input") => File.open(f, "r"),
+                  )
+                )
+                url = response.headers.fetch(:location)
+
+                # Amazon S3 uploads return urlencoded keys in the Location
+                # header, and we want to return the decoded object key for
+                # API operations.
+                uri = URI.parse(url)
+                if uri.host.end_with?(".amazonaws.com")
+                  uri.path = CGI.unescape(uri.path)
+                  url = uri.to_s
+                end
+
+                if files
+                  urls = files.map do |file|
+                    response = RestClient::Request.execute(
+                      :method => upload.fetch("form").fetch("method", "POST"),
+                      :url => upload.fetch("form").fetch("url"),
+                      :timeout => nil,
+                      :payload => upload.fetch("form").fetch("data", {}).merge(
+                        upload.fetch("form").fetch("file_input") => File.open(file, "r"),
+                      )
+                    )
+                    source_url = response.headers.fetch(:location)
+                    source_uri = URI.parse(source_url)
+                    if source_uri.host.end_with?(".amazonaws.com")
+                      source_uri.path = CGI.unescape(source_uri.path)
+                      source_url = source_uri.to_s
+                    end
+                    source_url
+                  end
+                end
+              rescue
+                print "error!\n".color(:red)
+                puts $!, *$!.backtrace, $!.response, $!.response.body
+                exit(1)
+              end
+            end
+
+            print "Pushing #{f}... "
+            measurement = Benchmark.measure do
+              if options.has_key?("skip-errors")
+                create_package_skip_errors(client_repo, f, url, dist_id, files, urls, ext, options["coordinates"])
+              elsif options.has_key?("skip-duplicates")
+                create_package_skip_duplicates(client_repo, f, url, dist_id, files, urls, ext, options["coordinates"])
+              else
+                create_package(client_repo, f, url, dist_id, files, urls, ext, options["coordinates"])
+              end
+            end
+            $logger.debug("create package request timing: #{measurement}")
+          end
+        end
+        $logger.debug("push command total timing: #{total_time}")
+      end
+
+      desc "version",
+           "print version information"
+      def version
+        puts "package_cloud CLI #{VERSION}\nSee https://packagecloud.io/docs#cli for more details."
+      end
+
+      private
+        def expand_dist_shortcut(dist)
+          case dist
+          when 'java'
+            'java/maven2'
+          when 'python'
+            'python/1'
+          when 'anyfile'
+            'anyfile/1'
+          when 'node'
+            'node/1'
+          else
+            dist
+          end
+        end
+
+        def package_scope(package_name)
+          name_parts = package_name.split('/')
+          if name_parts.size != 2 || !package_name.include?('@')
+            abort("Could not determine scope for '#{package_name}', it should look like: @my-scope/package-1.0.tgz".color(:red))
+          end
+          scope = name_parts[0]
+          unscoped_package_name = name_parts[1]
+          [scope, unscoped_package_name]
+        end
+
+        def abort_if_snapshot!(files)
+          if files.any? { |file| file.include?("-SNAPSHOT")  }
+            abort("SNAPSHOT uploads are not supported by the CLI, please use Maven instead: https://packagecloud.io/docs#wagon-instructions")
+          end
+        end
+
+        def create_package_skip_errors(client_repo, f, url, dist_id, files, urls, ext, coordinates=nil)
+          begin
+            client_repo.create_package(f, url, dist_id, files, urls, ext, coordinates=nil)
+          rescue RestClient::UnprocessableEntity => e
+            print "error (skipping):\n".color(:yellow)
+            json = JSON.parse(e.response)
+            json.each do |k,v|
+              puts "\n\t#{k}: #{v.join(", ")}\n"
+            end
+            puts ""
+          end
+        end
+
+        def create_package_skip_duplicates(client_repo, f, url, dist_id, files, urls, ext, coordinates=nil)
+          begin
+            client_repo.create_package(f, url, dist_id, files, urls, ext, coordinates=nil)
+          rescue RestClient::UnprocessableEntity => e
+            json = JSON.parse(e.response)
+            if json == { "filename" => ["has already been taken"] }
+              print "skipping, already exists.\n".color(:yellow)
+            else
+              print "error:\n".color(:red)
+              json.each do |k,v|
+                if v.is_a? String
+                  puts "\n\t#{k}: #{v}\n"
+                elsif v.is_a? Array
+                  puts "\n\t#{k}: #{v.join(", ")}\n"
+                end
+              end
+              puts ""
+              exit(1)
+            end
+          end
+        end
+
+        def create_package(client_repo, f, url, dist_id, files, urls, ext, coordinates=nil)
+          begin
+            client_repo.create_package(f, url, dist_id, files, urls, ext, coordinates)
+          rescue RestClient::UnprocessableEntity => e
+            print "error:\n".color(:red)
+            json = JSON.parse(e.response)
+            json.each do |k,v|
+              if v.is_a? String
+                puts "\n\t#{k}: #{v}\n"
+              elsif v.is_a? Array
+                puts "\n\t#{k}: #{v.join(", ")}\n"
+              end
+            end
+            puts ""
+            exit(1)
+          end
+        end
+
+        def parse_and_verify_dsc(repo, f, dist_id)
+          files = repo.parse_dsc(f, dist_id)
+          dirname = File.dirname(f)
+          find_and_verify(dirname, files)
+        end
+
+        def find_and_verify(dir, files)
+          file_paths = []
+          files.each do |f|
+            filepath = File.join(dir, f["filename"])
+            if !File.exist?(filepath)
+              print "Unable to find file name: #{f["filename"]} for source package: #{filepath}\n".color(:red)
+              abort("Aborting...".color(:red))
+            end
+
+            disk_size = File.stat(filepath).size
+            if disk_size != f["size"]
+              print "File #{f["filename"]} has size: #{disk_size}, expected: #{f["size"]}\n".color(:red)
+              abort("Aborting...".color(:red))
+            end
+
+            print "Found DSC package file #{f["filename"]} for upload\n"
+
+            file_paths << filepath
+          end
+          file_paths
+        end
+
+        def get_valid(prompt)
+          selection = ""
+          times = 0
+          until yield(selection)
+            if times > 0
+              puts "#{selection} is not a valid selection."
+            end
+            print "#{prompt}: "
+            selection = ::Kernel.gets.chomp
+            times += 1
+          end
+
+          selection
+        end
+    end
+  end
+end

data/lib/package_cloud/cli/gpg_key.rb

@@ -0,0 +1,76 @@
+module PackageCloud
+  module CLI
+    class GpgKey < Base
+      desc "list user/repository",
+           "list all GPG keys for specified repository"
+      def list(repo_name)
+        print "Looking for repository at #{repo_name}... "
+        repo = client.repository(repo_name)
+        print "success!\n"
+
+        keys = repo.gpg_keys
+        puts "GPG Keys for #{repo_name}:\n"
+
+        keys.each_with_index do |key, i|
+          if key.keytype == "package"
+            keytype = "Package signing key"
+          else
+            keytype = "Repository signing key"
+          end
+
+          puts "Key name: #{key.name}"
+          puts "Key type: #{keytype}"
+          puts "Key fingerprint: #{key.fingerprint}"
+          puts "GPG key url: #{key.download_url}"
+          puts
+        end
+      end
+
+      desc "create user/repository /path/to/gpg-key",
+           "create a package signing GPG key using the specified file"
+      def create(repo_name, file_path)
+        print "Looking for repository at #{repo_name}... "
+        repo = client.repository(repo_name)
+        print "success!\n"
+        repo.create_gpg_key(file_path)
+      end
+
+      desc "destroy user/repository keyname",
+           "destroy specified package signing GPG key"
+      def destroy(repo_name, keyname)
+        ARGV.clear
+        print "Looking for repository at #{repo_name}... "
+        repo = client.repository(repo_name)
+        print "success!\n"
+
+        key = repo.gpg_keys.detect do |key|
+          key.name == keyname
+        end
+
+        if key
+          msg = "\nAre you sure you want to delete the GPG key #{keyname}? (y/n)"
+          answer = get_valid(msg) do |s|
+            s == "y" || s == "n"
+          end
+
+          if answer == "y"
+            print "Attempting to destroy GPG key named #{keyname}... "
+            begin
+              key.destroy
+            rescue RestClient::ResourceNotFound =>e
+              print "\nError, could not find key: #{keyname}. No GPG keys deleted.\n".color(:red)
+              print "Please note that you cannot delete repository signing keys.\n"
+              exit(1)
+            else
+              print "success!\n".color(:green)
+            end
+          else
+            puts "Aborting...".color(:red)
+          end
+        else
+          puts "Wasn't able to find a GPG key name: #{keyname}".color(:red)
+        end
+      end
+    end
+  end
+end

data/lib/package_cloud/cli/master_token.rb

@@ -0,0 +1,75 @@
+module PackageCloud
+  module CLI 
+    class MasterToken < Base
+      desc "list user/repository",
+           "list master tokens for specified repository"
+      def list(repo_name)
+        print "Looking for repository at #{repo_name}... "
+        repo = client.repository(repo_name)
+        print "success!\n"
+
+        tokens = repo.master_tokens
+        puts "Tokens for #{repo_name}:"
+        puts ""
+        tokens.each_with_index do |token, i|
+          puts "  #{token.name} (#{token.value})"
+          puts "  read tokens:"
+          token.read_tokens.each do |read_token|
+          puts "    { id: #{read_token.id}, name: #{read_token.name}, value: #{read_token.value} }"
+          puts
+          end
+          puts "" unless i == tokens.length - 1
+        end
+      end
+
+      desc "create user/repository token_name",
+           "create a master token for the specified repository"
+      def create(repo_name, token_name)
+        print "Looking for repository at #{repo_name}... "
+        repo = client.repository(repo_name)
+        print "success!\n"
+
+        print "Attempting to create token named #{token_name}... "
+        resp = repo.create_master_token(token_name)
+        print "success!\n".color(:green)
+        print "Master token #{resp['name']} with value #{resp['value']} created.\n"
+      end
+
+      desc "destroy user/repository token_name",
+           "destroy a master token for the specified repository"
+      def destroy(repo_name, token_name)
+        ARGV.clear # otherwise gets explodes
+
+        if token_name == "default"
+          abort("You can't delete the default master_token.".color(:red))
+        end
+
+        print "Looking for repository at #{repo_name}... "
+        repo = client.repository(repo_name)
+        print "success!\n"
+
+        token = repo.master_tokens.detect do |token|
+          token.name == token_name
+        end
+
+        if token
+          msg = "\nAre you sure you want to delete #{token_name}?"
+          msg << " #{token.read_tokens.length} read tokens will no longer work afterwards (y/n)" if token.read_tokens.length > 0
+          answer = get_valid(msg) do |s|
+            s == "y" || s == "n"
+          end
+          if answer == "y"
+            print "Attempting to destroy token named #{token_name}... "
+            token.destroy
+            print "success!\n".color(:green)
+          else
+            puts "Aborting...".color(:red)
+          end
+        else
+          puts "Wasn't able to find a token named #{token_name}.".color(:red)
+          exit(1)
+        end
+      end
+    end
+  end
+end

data/lib/package_cloud/cli/read_token.rb

@@ -0,0 +1,39 @@
+module PackageCloud
+  module CLI
+    class ReadToken < Base
+      desc "destroy user/repository mastertoken/readtoken",
+           "revokes read token associated to a master token"
+      def destroy(repo_name, master_and_read_token)
+        print "Looking for repository at #{repo_name}... "
+        repo = client.repository(repo_name)
+
+        given_master_token, given_read_token = master_and_read_token.split("/")
+
+        if given_master_token.nil? || given_read_token.nil?
+          print "invalid master token and/or read token!\n".color(:red)
+          exit(127)
+        end
+
+        master_token = repo.master_tokens.detect { |t| t.name == given_master_token }
+
+        if master_token.nil?
+          print "couldn't find master token named #{given_master_token}\n".color(:red)
+          exit(127)
+        end
+
+        read_token = master_token.read_tokens.detect { |t| t.name == given_read_token }
+
+        if read_token.nil?
+          print "couldn't find read token named #{given_read_token} for #{given_master_token}\n".color(:red)
+          exit(127)
+        end
+
+        master_token_path = master_token.paths["self"]
+
+        read_token.destroy(master_token_path, read_token.id)
+
+        print "success!\n"
+      end
+    end
+  end
+end

data/lib/package_cloud/cli/repository.rb

@@ -0,0 +1,52 @@
+module PackageCloud
+  module CLI
+    class Repository < Base
+      option :private
+      desc "create repo_name",
+           "create repository called repo_name"
+      def create(repo_name)
+        config.read_or_create
+        client = Client.new(config)
+
+        print "Looking for repository at #{repo_name}... "
+        repo = client.create_repo(repo_name, options[:private])
+        print "success!\n".color(:green)
+        puts "Your repository has been created at:"
+        puts "    #{repo["url"]}"
+      end
+
+      desc "list",
+           "list your repositories"
+      def list
+        repos = client.repositories
+        if repos.length == 0
+          puts "You have no repositories at the moment. Create one!"
+        else
+          puts "Your repositories:"
+          puts ""
+          repos.each_with_index do |repo, i|
+            puts "  #{repo.fqname} (#{repo.private_human})"
+            puts "  last push: #{repo.last_push_human} | packages: #{repo.package_count_human}"
+            puts "" unless i == repos.length - 1
+          end
+        end
+      end
+
+      desc "install user/repo package_type",
+           "install user/repo for package_type"
+      def install(repo, package_type)
+        if Process.uid != 0 && package_type != "gem"
+          abort("You have to run install as root.".color(:red))
+        end
+
+        print "Locating repository at #{repo}... "
+        repo = client.repository(repo)
+        print "success!\n"
+        script = repo.install_script(package_type)
+        pid = fork { exec("/bin/bash", script.path) }
+        Process.waitpid(pid)
+        script.unlink
+      end
+    end
+  end
+end