picdown 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a75b84f17db156d3c6cca720730d2083e50dff9053e978815ee207cfdf9dcf57
+  data.tar.gz: 920d7ac38131c39c5169d241760fd045bea326e4c120505a8ddd86dd7853ed92
+SHA512:
+  metadata.gz: 97a63677439f6b0b48e9c09e1ae2c16b5256cf41b59b36df52cddb4a0444e4ee78ac80b09f4ec1d0cceb9bdcae5638a112d5360568fe476552ba04423109ef74
+  data.tar.gz: 6935d87ded479c182e7e579fed07677f0196ef7b01cc3ef52c5b917f6f840e4e52748e75454a10c2e13ad4a9713087aa9158b68cc2e58528fbe9943fd2c36e9c

data/README.md

@@ -0,0 +1,111 @@
+Picdown ![pipeline](https://gitlab.com/htkaslan/picdown/badges/master/pipeline.svg) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/349123cc896d48cb97f54f9393465f0c)](https://www.codacy.com/gl/htkaslan/picdown/dashboard?utm_source=gitlab.com&utm_medium=referral&utm_content=htkaslan/picdown&utm_campaign=Badge_Grade)
+=======
+
+Picdown is a command line tool that downloads the given images from a source file.
+It also allows you to make parallel requests to download images. Picdown provides
+fancy outputs such as progress bar for each request to inform the user.
+
+[![asciicast](https://asciinema.org/a/ljSjsRVrkroGLHpPBFzltIIDx.svg)](https://asciinema.org/a/ljSjsRVrkroGLHpPBFzltIIDx)
+
+Installation
+------------
+
+Picdown is dockerised, which means you can experiment it by building the image
+from Dockerfile. Please run the below command to do that.
+
+Note that you need to have a proper docker environment first.
+
+```sh
+docker build . -t picdown
+```
+
+If you want to build it manually, please follow the following instructions.
+
+```sh
+bundle config --global silence_root_warning true
+bundle config --global set path vendor/bundle
+bundle install
+bundle exec rake install
+gem install pkg/picdown-*.gem
+```
+
+Picdown is also a gem published on GitLab. You can have it by skipping installation
+steps. Please ensure that you have set up [GitLab Rubygems Registry](https://docs.gitlab.com/ce/user/packages/rubygems_registry/index.html)
+before running the following command.
+
+```sh
+gem install picdown --version <VERSION>
+```
+
+Usage
+-----
+
+Please have a look at the help text of `picdown` for more information.
+
+```
+➜ picdown -h
+Commands:
+  picdown download FILE            # Download files from the specified source file.
+  picdown validate PATH            # Validate source file
+  picdown version                  # Print version
+```
+
+### `validate`
+
+It validates the given source file if it contains valid URLs.
+
+```sh
+picdown validate SOURCE
+```
+
+### `download`
+
+It downloads the images given by the source file. It needs a valid file path.
+
+```sh
+picdown download SOURCE
+```
+
+To download the images parallel:
+
+```sh
+picdown download SOURCE --parallel=4
+```
+
+You are also able to configure the download parameters. Please check the below
+help text for additional parameters.
+
+```sh
+➜ picdown download -h
+...
+
+Options:
+  --destination=VALUE               # Destination where files to be downloaded., default: "."
+  --parallel=VALUE                  # Number of parallel download processes., default: 1
+  --open-timeout=VALUE              # Open timeout for connections. (secs), default: 15
+  --read-timeout=VALUE              # Read timeout for connections. (secs), default: 15
+  --max-size=VALUE                  # Max file size. (bytes), default: 5242880
+  --max-redirects=VALUE             # Max redirections., default: 3
+
+...
+```
+
+Demo
+----
+
+Picdown includes a demo file, which is located in `assets`, that you can use.
+
+```sh
+picdown download assets/test.txt
+```
+
+or
+
+```sh
+docker run -t picdown download assets/test.txt
+```
+
+License
+-------
+
+Not licensed yet.

data/Rakefile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

data/bin/picdown

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'picdown/cli'
+
+def run
+  Dry::CLI.new(Picdown::CLI).call
+rescue StandardError => e
+  abort e.message
+end
+
+run

data/lib/picdown.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'picdown/version'

data/lib/picdown/cli.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'dry/cli'
+
+require 'picdown/error'
+require 'picdown/support'
+require 'picdown/config'
+require 'picdown/source'
+require 'picdown/source_file'
+require 'picdown/multithread'
+require 'picdown/download'
+
+module Picdown
+  module CLI
+    extend Dry::CLI::Registry
+
+    Dir[File.join(__dir__, 'cli/*.rb')].sort.each { |command| require command }
+
+    register 'download', Download
+    register 'validate', Validate
+    register 'version',  Version, aliases: ['v', '-v', '--version']
+  end
+end

data/lib/picdown/cli/download.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Picdown
+  module CLI
+    class Download < Dry::CLI::Command
+      desc 'Download files from the specified source file.'
+
+      option :destination,   type: :string, default: '.',             desc: 'Destination where files to be downloaded.'
+      option :parallel,      type: :string, default: 1,               desc: 'Number of parallel download processes.'
+      option :open_timeout,  type: :string, default: 15,              desc: 'Open timeout for connections. (secs)'
+      option :read_timeout,  type: :string, default: 15,              desc: 'Read timeout for connections. (secs)'
+      option :max_size,      type: :string, default: 5 * 1024 * 1024, desc: 'Max file size. (bytes)'
+      option :max_redirects, type: :string, default: 3,               desc: 'Max redirections.'
+
+      argument :file, type: :string, required: true, desc: 'File to be used as a URL source.'
+
+      example [
+        '/path/to/file                               # Download files normally.',
+        '/path/to/file --parallel=2                  # Perform parallel downloads.',
+        '/path/to/file --destination=/path/to/target # Download to a specific destination.'
+      ]
+
+      def call(file:, **options)
+        Picdown::Download.from_file(file, **options)
+      end
+    end
+  end
+end

data/lib/picdown/cli/validate.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Picdown
+  module CLI
+    class Validate < Dry::CLI::Command
+      desc 'Validate source file'
+
+      argument :path, type: :string, required: true, desc: 'File path to be validated.'
+
+      def call(path:, **)
+        sources = Picdown::SourceFile.load(path)
+        print_results(sources)
+      end
+
+      protected
+
+      def print_results(sources)
+        return warn 'The source file is valid!' if sources.nondownloadable.empty?
+
+        warn 'The source file contains invalid sources.'
+        sources.nondownloadable.each do |src|
+          warn ''
+          warn "on #{sources.path} line #{src.line}:"
+          warn "  #{src.url}"
+        end
+      end
+    end
+  end
+end

data/lib/picdown/cli/version.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'picdown'
+
+module Picdown
+  module CLI
+    class Version < Dry::CLI::Command
+      desc 'Print version'
+
+      def call(*)
+        puts Picdown::VERSION
+      end
+    end
+  end
+end

data/lib/picdown/config.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Picdown
+  Config = Struct.new(
+    :destination,
+    :parallel,
+    :open_timeout,
+    :read_timeout,
+    :max_size,
+    :max_redirects,
+    keyword_init: true
+  ) do
+    def initialize(**)
+      super
+      self.parallel      = parallel.to_i
+      self.open_timeout  = open_timeout.to_i
+      self.read_timeout  = read_timeout.to_i
+      self.max_size      = max_size.to_i
+      self.max_redirects = max_redirects.to_i
+    end
+  end
+end

data/lib/picdown/download.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'down'
+require 'tty-progressbar'
+
+module Picdown
+  class Download
+    using Support::Refinements
+
+    attr_reader :source, :config, :bars
+
+    def initialize(file, **opts)
+      @source = SourceFile.load(file)
+      @config = Config.new(opts)
+      @bars   = TTY::ProgressBar::Multi.new
+    end
+
+    def call
+      config.parallel <= 1 ? perform_sequential : perform_parallel
+    end
+
+    def perform_sequential
+      source.downloadable.each { |src| download(src) }
+    end
+
+    def perform_parallel
+      threads = MultiThread.new(config.parallel)
+      source.downloadable.each { |src| threads.add { download(src) } }
+      threads.invoke
+    end
+
+    class << self
+      def from_file(file, **opts)
+        new(file, **opts).call
+      end
+    end
+
+    private
+
+    def download(src) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      bar = bars.register "#{src.url.shorten} [:bar] :percent", total: 100, width: 30
+      total_size = 0
+
+      tempfile = Down.download(
+        src.url,
+        **config.to_h.except(:destination, :parallel),
+        content_length_proc: ->(content_length) { total_size += content_length },
+        progress_proc: lambda do |progress|
+          break if progress == total_size
+
+          bar.current = (progress.to_f / total_size) * 100
+        end
+      )
+
+      # Safely interrupt the download if the file is not an image.
+      content_type = tempfile.content_type
+      raise MimeTypeError, 'The file is not an image.' unless Support::MIME_TYPES.include?(content_type)
+
+      # Move the tempfile to the destination.
+      name = tempfile.original_filename || File.basename(tempfile.path)
+      dest = File.join(config.destination, name)
+      FileUtils.mv(tempfile.path, dest)
+    rescue Down::Error => e
+      raise DownloadError, "FAILED: #{src.url}: #{e.message}"
+    ensure
+      bar.finish
+      tempfile&.close
+    end
+  end
+end

data/lib/picdown/error.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Picdown
+  Error         = Class.new(StandardError)
+  MimeTypeError = Class.new(Error)
+  DownloadError = Class.new(Error)
+end

data/lib/picdown/multithread.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Picdown
+  class MultiThread
+    Thread.report_on_exception = false
+
+    attr_reader :pool, :threads, :queue
+
+    def initialize(pool)
+      @pool    = pool
+      @threads = []
+      @queue   = Queue.new
+    end
+
+    def add(*args, &block)
+      queue << [block, args]
+    end
+
+    def invoke
+      1.upto(pool) { threads << spawn_thread }
+      threads.each do |t|
+        trap('INT') { Thread.kill(t) }
+        t.join
+      end
+    end
+
+    private
+
+    def spawn_thread
+      Thread.new do
+        until queue.empty?
+          block, args = queue.pop
+          block&.call(*args)
+        end
+      end
+    end
+  end
+end

data/lib/picdown/source.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Picdown
+  Source = Struct.new(:line, :url) do
+    def valid?
+      url.match?(URI::DEFAULT_PARSER.make_regexp)
+    end
+  end
+end

data/lib/picdown/source_file.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Picdown
+  class SourceFile
+    include Enumerable
+
+    attr_reader :path, :sources
+
+    def initialize(path)
+      @path    = path
+      @sources = []
+    end
+
+    def <<(src)
+      sources << src
+    end
+
+    def each(&block)
+      sources.each(&block)
+    end
+
+    def load
+      File.readlines(path).each_with_index do |line, i|
+        sources << Source.new(i + 1, line.chomp)
+      end
+
+      self
+    end
+
+    def downloadable
+      select(&:valid?)
+    end
+
+    def nondownloadable
+      sources - downloadable
+    end
+
+    class << self
+      def load(path)
+        new(path).load
+      end
+    end
+  end
+end

data/lib/picdown/support.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Picdown
+  module Support
+    # Adapted from: https://www.iana.org/assignments/media-types/media-types.xhtml#image
+    MIME_TYPES = %w[
+      image/aces
+      image/avci
+      image/avcs
+      image/avif
+      image/bmp
+      image/cgm
+      image/dicom-rle
+      image/emf
+      image/example
+      image/fits
+      image/g3fax
+      image/gif
+      image/heic
+      image/heic-sequence
+      image/heif
+      image/heif-sequence
+      image/hej2k
+      image/hsj2
+      image/jls
+      image/jp2
+      image/jpeg
+      image/jph
+      image/jphc
+      image/jpm
+      image/jpx
+      image/jxr
+      image/jxrA
+      image/jxrS
+      image/jxs
+      image/jxsc
+      image/jxsi
+      image/jxss
+      image/ktx
+      image/ktx2
+      image/naplps
+      image/png
+      image/prs.btif
+      image/prs.pti
+      image/pwg-raster
+      image/svg+xml
+      image/t38
+      image/tiff
+      image/tiff-fx
+      image/vnd.adobe.photoshop
+      image/vnd.airzip.accelerator.azv
+      image/vnd.cns.inf2
+      image/vnd.dece.graphic
+      image/vnd.djvu
+      image/vnd.dwg
+      image/vnd.dxf
+      image/vnd.dvb.subtitle
+      image/vnd.fastbidsheet
+      image/vnd.fpx
+      image/vnd.fst
+      image/vnd.fujixerox.edmics-mmr
+      image/vnd.fujixerox.edmics-rlc
+      image/vnd.globalgraphics.pgb
+      image/vnd.microsoft.icon
+      image/vnd.mix
+      image/vnd.ms-modi
+      image/vnd.mozilla.apng
+      image/vnd.net-fpx
+      image/vnd.pco.b16
+      image/vnd.radiance
+      image/vnd.sealed.png
+      image/vnd.sealedmedia.softseal.gif
+      image/vnd.sealedmedia.softseal.jpg
+      image/vnd.svf
+      image/vnd.tencent.tap
+      image/vnd.valve.source.texture
+      image/vnd.wap.wbmp
+      image/vnd.xiff
+      image/vnd.zbrush.pcx
+      image/wmf
+      image/emf
+      image/wmf
+    ].freeze
+
+    module Refinements
+      refine String do
+        def shorten(limit: 15)
+          "#{self[..(limit - 1)]}...#{self[-limit..]}"
+        end
+      end
+
+      refine Hash do
+        def except(*keys)
+          slice(*self.keys - keys)
+        end
+      end
+    end
+  end
+end

data/lib/picdown/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Picdown
+  VERSION = '0.1.1'
+end

data/picdown.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path('lib', __dir__)
+require 'picdown/version'
+
+Gem::Specification.new do |spec|
+  spec.name                  = 'picdown'
+  spec.author                = 'Hüseyin Tekinaslan'
+  spec.email                 = 'htkaslan@gmail.com'
+  spec.summary               = 'Download the given images from a source file.'
+  spec.description           = spec.summary
+  spec.homepage              = 'https://github.com/huseyin/picdown'
+  spec.files                 = Dir['README.md', 'Rakefile', 'picdown.gemspec', 'lib/**/*']
+  spec.require_paths         = ['lib']
+  spec.bindir                = 'bin'
+  spec.executables           = ['picdown']
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.7')
+  spec.version               = Picdown::VERSION
+
+  spec.add_dependency 'down', '~> 5.0'
+  spec.add_dependency 'dry-cli'
+  spec.add_dependency 'tty-progressbar'
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'minitest-focus'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-minitest'
+  spec.add_development_dependency 'rubocop-performance'
+  spec.add_development_dependency 'rubocop-rake'
+  spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'webmock'
+end

metadata

@@ -0,0 +1,227 @@
+--- !ruby/object:Gem::Specification
+name: picdown
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Hüseyin Tekinaslan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-05-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: down
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: dry-cli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tty-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-focus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Download the given images from a source file.
+email: htkaslan@gmail.com
+executables:
+- picdown
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- bin/picdown
+- lib/picdown.rb
+- lib/picdown/cli.rb
+- lib/picdown/cli/download.rb
+- lib/picdown/cli/validate.rb
+- lib/picdown/cli/version.rb
+- lib/picdown/config.rb
+- lib/picdown/download.rb
+- lib/picdown/error.rb
+- lib/picdown/multithread.rb
+- lib/picdown/source.rb
+- lib/picdown/source_file.rb
+- lib/picdown/support.rb
+- lib/picdown/version.rb
+- picdown.gemspec
+homepage: https://github.com/huseyin/picdown
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Download the given images from a source file.
+test_files: []