artifact_tools 0.0.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 289e3ecd5615e51b01f2a42a2fb09420a52f21f4c0467fce0b0718b6f81da06b
+  data.tar.gz: c7cd3a30aca98803c044ab2aef7df0086f517f5cb18d13ef542bda889ba60544
+SHA512:
+  metadata.gz: e6b7ae6fd0ad77a9521bb8cd7eefc61af5346817956efcbbca52e6e52ca090ee8ee6d7ede6760acfba5a28170eae4a3d9db4109e8bbac684ee19d1b1422b06d0
+  data.tar.gz: d823934f9d66748ac73c040d778905b455a71bbf37143a1c47a7287062f1ebdca435f204f5fc52d0466438d74cc22b2f44f87fde229fd83da5d546b622854de7

data/bin/artifact_download

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'artifact_tools/downloader'
+
+ArtifactTools::Downloader.new(**ArtifactTools::Downloader.parse(ARGV))

data/bin/artifact_upload

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'artifact_tools/uploader'
+
+ArtifactTools::Uploader.new(**ArtifactTools::Uploader.parse(ARGV))

data/lib/artifact_tools/client.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'net/ssh'
+require 'net/scp'
+require 'fileutils'
+require 'digest'
+require 'artifact_tools/hasher'
+
+module ArtifactTools
+  # Notifies that there was a mismatch between expected hash of the
+  # file(according to the configuration file) and the actual hash of the
+  # fetched file
+  class HashMismatchError < RuntimeError
+  end
+
+  # Use an object of this class to put/fetch files from storage specified with {ConfigFile}
+  class Client
+    include ArtifactTools::Hasher
+
+    # @param config [Hash] Configuration
+    # @param user [String] User name to connect to server with, overrides
+    #   ARTIFACT_STORAGE_USER and the on stored in config
+    def initialize(config:, user: nil)
+      @config = config
+      user ||= ENV['ARTIFACT_STORAGE_USER'] || @config['user']
+      @ssh = Net::SSH.start(@config['server'], user, non_interactive: true)
+    end
+
+    # Fetch a file from store
+    #
+    # @param file [String] Path to file to fetch. Fetches all files from config if omitted.
+    # @param dest [String] Optional prefix to add to local path of the file being fetched. Uses cwd if omitted.
+    # @param match [Regexp] Optionally fetch only files matching this pattern.
+    # @param verify [Boolean] Whether to verify the checksum after the file is received. Slows the fetch.
+    #
+    # @raise [HashMismatchError] In case checksum doesn't match the one stored in the config file.
+    def fetch(file: nil, dest: nil, match: nil, verify: false, force: false)
+      files = @config['files'].keys
+      files = [file] if file
+      files.each do |entry|
+        next if match && !entry.match?(match)
+
+        entry_hash = @config['files'][entry]['hash']
+        remote = compose_remote(entry, entry_hash)
+        local = compose_local(dest, entry)
+        next if !force && local_file_matches(local, entry_hash)
+
+        @ssh.scp.download!(remote, local)
+        verify(entry_hash, local) if verify
+      end
+    end
+
+    # Put a file to storage
+    #
+    # @param file [String] Path to the file to store.
+    def put(file:)
+      hash = file_hash(file)
+      remote = compose_remote(file, hash)
+      ensure_remote_path_exists(remote)
+      @ssh.scp.upload!(file, remote)
+    end
+
+    private
+
+    def compose_remote(file, hash)
+      basename = File.basename(file)
+      "#{@config['dir']}/#{hash}/#{basename}"
+    end
+
+    def ensure_path_exists(local)
+      dirname = File.dirname(local)
+      return if File.directory?(dirname)
+
+      FileUtils.mkdir_p(dirname)
+    end
+
+    def ensure_remote_path_exists(remote)
+      dirname = File.dirname(remote)
+      return if File.directory?(dirname)
+
+      @ssh.exec!("mkdir -p #{dirname}")
+    end
+
+    def compose_local(dest, file)
+      local = file
+      local = "#{dest}/#{local}" if dest
+      ensure_path_exists(local)
+      local
+    end
+
+    def verify(expected_hash, path)
+      actual_hash = file_hash(path)
+      return unless expected_hash != actual_hash
+
+      raise HashMismatchError, "File #{path} has hash: #{actual_hash} while it should have: #{expected_hash}"
+    end
+
+    def local_file_matches(local_file, expected_hash)
+      File.exist?(local_file) && file_hash(local_file) == expected_hash
+    end
+  end
+end

data/lib/artifact_tools/config_file.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'artifact_tools/hasher'
+
+module ArtifactTools
+  # Store configuration information about artifacts and where they are stored.
+  #
+  # It has to contain at least the fields from {REQUIRED_FIELDS} while allowing
+  # any key/value which has a value for the user.
+  class ConfigFile
+    include ArtifactTools::Hasher
+    attr_reader :config
+
+    REQUIRED_FIELDS = %w[server dir files].freeze
+
+    # Initialize config file
+    #
+    # @param config [Hash] Provide configuration. Mandatory fields are {REQUIRED_FIELDS}
+    def initialize(config:)
+      raise 'Invalid config' unless REQUIRED_FIELDS.all? { |k| config.keys.include?(k) }
+
+      raise 'Invalid config' unless [NilClass, Hash].any? { |klass| config['files'].is_a?(klass) }
+
+      @config = config
+    end
+
+    # Create ConfigFile from file in YAML format
+    #
+    # @param file [String] Path to file in YAML format.
+    def self.from_file(file)
+      ConfigFile.new(config: YAML.load_file(file))
+      # Leave error propagation as this is development tool
+    end
+
+    # Saves configuration to file
+    #
+    # @param file [String] Save in this file. Overwrites the file if present.
+    def save(file)
+      File.write(file, @config.to_yaml)
+      # Leave error propagation as this is development tool
+    end
+
+    # Append file to configuration.
+    #
+    # @param file [String] Path to the file to store in the configuration
+    # @param store_path [String] Use this path as key in the configuration. Optional, if omitted uses file
+    # @param opts [Hash] Additional fields to store for the file
+    #
+    # @note If file exists in the config with key *store_path* then its
+    #   properties will be merged, where new ones will have priority.
+    def append_file(file:, store_path: nil, **opts)
+      store_path ||= file
+
+      # Convert symbols to String
+      opts = hash_keys_to_strings(opts)
+
+      @config['files'] ||= {}
+      @config['files'][store_path] = opts
+      @config['files'][store_path]['hash'] ||= file_hash(file)
+    end
+
+    private
+
+    def hash_keys_to_strings(hash)
+      hash.transform_keys(&:to_s)
+    end
+  end
+end

data/lib/artifact_tools/downloader.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'artifact_tools/client'
+require 'artifact_tools/config_file'
+require 'optparse'
+require 'yaml'
+
+module ArtifactTools
+  # Downloader allows the user to fetch files from a store specified. All this
+  # information is provided by {ConfigFile}.
+  class Downloader
+    # Downloads requested files
+    #
+    # @param [Hash] args the arguments for downloading artifacts
+    # @argument args :config_file [String] Path to configuration file
+    # @argument args :user [String] User to use for download connection
+    # @argument args :dest_dir [String] Where to download artifacts to
+    # @argument args :verify [Boolean] Whether to verify checksums after download.
+    # @argument args :force [Boolean] Whether to download files even if they are already
+    #   present with the exected hash
+    # @argument args :match [Regexp] Whether to verify checksums after download.
+    def initialize(args = { verify: true, force: false })
+      config = load_config(args[:config_file])
+      c = ArtifactTools::Client.new(config: config.config, user: args[:user])
+      c.fetch(dest: args[:dest_dir], verify: args[:verify], match: args[:match], force: args[:force])
+    end
+
+    @default_opts = {
+      verify: true,
+      force: false,
+      dest_dir: '.'
+    }
+    @parse_opts_handlers = {
+      ['-c FILE', '--configuration=FILE', 'Pass configuration file'] => lambda { |f, options, _opts|
+        options[:config_file] = f
+      },
+      ['-d DIR', '--destination=DIR', 'Store files in directory'] => lambda { |d, options, _opts|
+        options[:dest_dir] = d
+      },
+      ['-v', '--[no-]verify', TrueClass, "Verify hash on downloaded files. Default: #{@default_opts[:verify]}."] =>
+      lambda { |v, options, _opts|
+        options[:verify] = v
+      },
+      [
+        '-f', '--[no-]force', TrueClass,
+        "Force download of files if they are present with expected hash. Default: #{@default_opts[:force]}."
+      ] => lambda { |v, options, _opts|
+        options[:force] = v
+      },
+      ['-u USER', '--user=USER', 'Access server with this username'] => ->(u, options, _opts) { options[:user] = u },
+      ['-m REGEXP', '--match=REGEXP', Regexp, 'Download only file which match regular expression'] =>
+      lambda { |v, options, _opts|
+        options[:match] = v
+      },
+      ['-h', '--help', 'Show this message'] => lambda { |_h, _options, opts|
+        puts opts
+        exit
+      }
+    }
+    # Parse command line options to options suitable to Downloader.new
+    #
+    # @param arguments [Array(String)] Command line options to parse and use.
+    #   Hint: pass ARGV
+    def self.parse(arguments)
+      options = @default_opts
+      arguments << '-h' if arguments.empty?
+      OptionParser.new do |opts|
+        opts.banner = "Usage: #{__FILE__} [options]"
+        @parse_opts_handlers.each do |args, handler|
+          opts.on(*args) { |v|  handler.call(v, options, opts) }
+        end
+      end.parse!(arguments)
+
+      raise OptionParser::MissingArgument, 'Missing -c/--configuration option' unless options.key?(:config_file)
+
+      options
+    end
+
+    private
+
+    def load_config(config_file)
+      ArtifactTools::ConfigFile.from_file(config_file)
+      # TODO: error check
+    end
+  end
+end

data/lib/artifact_tools/hasher.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ArtifactTools
+  # wrapper for the hashing algo used
+  module Hasher
+    # Calculate hash of a file
+    #
+    # @param path [String] Path to file to hash.
+    def file_hash(path)
+      hash_algo.file(path).hexdigest
+    end
+
+    private
+
+    def hash_algo
+      # TODO: decide on used algorithm
+      Digest::SHA1
+    end
+  end
+end

data/lib/artifact_tools/uploader.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'artifact_tools/client'
+require 'artifact_tools/config_file'
+require 'optparse'
+require 'yaml'
+
+module ArtifactTools
+  # Uploader allows the user to upload files to a store specified by
+  # {ConfigFile}.
+  class Uploader
+    include ArtifactTools::Hasher
+    # Upload requested files
+    #
+    # @param config_file [String] Path to configuration file
+    # @param append [Boolean] Whether to append files to config file
+    # @param files [Array(String)] Paths to files to upload
+    def initialize(config_file:, files:, append: false)
+      # TODO: check for clashes of files, do hash checks?
+      @config_file = config_file
+      @append = append
+      @config = load_config(@config_file)
+      c = ArtifactTools::Client.new(config: @config.config)
+      files.each do |file|
+        update_file(c, file)
+      end
+      @config.save(config_file)
+    end
+
+    @default_append_opt = false
+    @parse_opts_handlers = {
+      ['-c FILE', '--configuration=FILE', 'Pass configuration file.'] => lambda { |_opts, v, options|
+        options[:config_file] = v
+      },
+      ['-a', '--append',
+       "Append uploaded files to configuration file, if missing. Default: #{@default_append_opt}."] =>
+      ->(_opts, v, options) { options[:append] = v },
+      ['-h', '--help', 'Show this message'] => lambda { |opts, _v, _options|
+        puts opts
+        exit
+      }
+    }
+
+    # Parse command line options to options suitable to Downloader.new
+    #
+    # @param arguments [Array(String)] Command line options to parse and use.
+    #   Hint: pass ARGV
+    def self.parse(arguments)
+      options = { append: @default_append_opt }
+      arguments << '-h' if arguments.empty?
+      OptionParser.new do |opts|
+        opts.banner = "Usage: #{__FILE__} [options]"
+        @parse_opts_handlers.each do |args, handler|
+          opts.on(*args) { |v|  handler.call(opts, v, options) }
+        end
+      end.parse!(arguments)
+
+      raise OptionParser::MissingArgument, 'Missing -c/--configuration option' unless options.key?(:config_file)
+
+      options.merge({ files: arguments.dup })
+    end
+
+    private
+
+    def load_config(config_file)
+      ArtifactTools::ConfigFile.from_file(config_file)
+    end
+
+    def relative_to_config(file, config_file)
+      file = File.expand_path(file)
+      config_file = File.expand_path(config_file)
+      config_file_dirname = File.dirname(config_file)
+      return nil unless file.start_with?(config_file_dirname)
+
+      file[(config_file_dirname.length + 1)..]
+    end
+
+    # update the current file remotely and append it to the config if needed
+    def update_file(client, file)
+      client.put(file: file)
+      hash = file_hash(file)
+      puts "#{hash} #{file}"
+      return unless @append
+
+      rel_path = relative_to_config(file, @config_file)
+      raise "#{file} is not relative to config: #{@config_file}" unless rel_path
+
+      @config.append_file(file: file, store_path: rel_path, hash: hash)
+    end
+  end
+end

data/lib/artifact_tools/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ArtifactTools
+  VERSION = '0.0.5'
+end

data/lib/artifact_tools.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'artifact_tools/client'
+require 'artifact_tools/version'

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: artifact_tools
+version: !ruby/object:Gem::Version
+  version: 0.0.5
+platform: ruby
+authors:
+- Vitosha Labs Open Source team
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-12-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+description:
+email:
+executables:
+- artifact_download
+- artifact_upload
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/artifact_download
+- bin/artifact_upload
+- lib/artifact_tools.rb
+- lib/artifact_tools/client.rb
+- lib/artifact_tools/config_file.rb
+- lib/artifact_tools/downloader.rb
+- lib/artifact_tools/hasher.rb
+- lib/artifact_tools/uploader.rb
+- lib/artifact_tools/version.rb
+homepage: https://github.com/vitoshalabs/artifact_tools
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Provides tools to manage repository artifacts.
+test_files: []