gs_img_fetcher 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3ce17816770496a81e56ee290c9eda21bacddfb0e81c3fbce712ae50d477b64
-  data.tar.gz: e239f232dddbb0c201633837a0d680c245813b26420d0e316b0bf7baaeaec1de
+  metadata.gz: f61655b14e1a59cfe87809b12cbd0ba98e2886cf8bbcb7a1e15681c99d5c53d4
+  data.tar.gz: 319618a2329511a0c680deb8016eddcc668f9630e5d07131ebeda8440e479392
 SHA512:
-  metadata.gz: 732e6099d8c80fc0bad6bb891acdcaa77b262d717944b33e0ab8aaa699802312138765e6a278afa66af44653bb8e1ff5669c0198a15eda3976faf14fc8c38a68
-  data.tar.gz: 905ea2430a1aeca45d2d07ffd71ffeeaa9937381dde32e0a32213c756825057fcbaf31d0942f2f1003e11154d82c3bf815305c7618bb3ecf803da6563225c79b
+  metadata.gz: e5c1fe6b7dc0a7983ad209b0e790e647eded6799141423312dd91d2ba303f93a6e22b94a5fe81da55ca6561c7e72d5f01172005d4cdef857e82a419142710e1e
+  data.tar.gz: 63d3769db8e5268d81a099f9d3512ef23e5c41557defab7b012cb98b7a3157dde5353fd0061d012cd5fc5a29aea5719a59dad9d6c118efb3fa1af6c3c12301be

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gs_img_fetcher (0.1.1)
+    gs_img_fetcher (0.1.3)
       activesupport (~> 6.0)
       down (~> 5.1)
       dry-cli (~> 0.6)

data/README.md

@@ -21,6 +21,18 @@ And then execute:
 Or install it yourself as:
 
     $ gem install gs_img_fetcher
+    
+## Features
+
+### Concurrency
+`gs_img_fetcher` is designed with concurrency in mind. It can be configured to fetch images either asynchronously or synchronously.
+By default, it runs asynchronously and the maximum number of threads depends on what your machine allows.
+For a relatively small input, it would be better to specify `--no-async` option.
+Check out the options `--async` and `--max_threads`.
+
+### File size limit
+You can set a limit on the maximum size of each downloaded image to avoid downloading unexpectedly large files and filling up your storage.
+By default, it runs without a limit. Use `--max_size` option to set one.
 
 ## Usage
 
@@ -33,7 +45,7 @@ http://example.com/image1.jpg
 http://example.com/image1.png
 http://example.com/image1.svg
 
-$ gs_img_fetcher run urls.txt output
+$ gs_img_fetcher run urls.txt output --max_size=5
 I, [2020-05-17T13:09:01.420214 #87392]  INFO -- : Processing 3 URLs (3 valid, 0 invalid)
 ...
 I, [2020-05-17T13:09:02.709097 #87392]  INFO -- : Fetch complete (3 successful, 0 failed)

data/exe/gs_img_fetcher

@@ -35,9 +35,19 @@ module Commands
            required: false,
            type: :integer,
            desc: 'Maximum number of threads.'
+    option :max_size,
+           required: false,
+           type: :integer,
+           desc: 'Maximum size of each downloaded image in MB.'
 
     def call(input_path:, **opts)
-      GsImgFetcher::Manager.fetch(input_path, **opts.slice(:output_dir, :async, :max_threads))
+      puts opts
+      GsImgFetcher::Manager.fetch(
+        input_path,
+        max_threads: opts[:max_threads]&.to_i,
+        max_size: opts[:max_size]&.to_i,
+        **opts.slice(:output_dir, :async).compact
+      )
     end
   end
 

data/lib/gs_img_fetcher/fetcher.rb

@@ -13,15 +13,16 @@ module GsImgFetcher
 
     attr_reader :state, :uuid
 
-    def initialize(entry, output_dir)
+    def initialize(entry, output_dir, max_size: nil)
       @entry = entry
       @output_dir = output_dir
+      @max_size = max_size.yield_self { |s| s.present? ? s * 1024 * 1024 : nil }
       @uuid = SecureRandom.uuid
       @state = INITIALIZED
     end
 
     def fetch
-      @tempfile = Down.download(@entry.url)
+      @tempfile = Down.download(@entry.url, **{ max_size: @max_size }.compact)
       @state = FETCHED
       log_fetched
     rescue Down::Error => e

data/lib/gs_img_fetcher/manager.rb

@@ -15,11 +15,12 @@ module GsImgFetcher
 
     DEFAULT_MAX_THREADS = Concurrent.processor_count
 
-    def initialize(entry_set, output_dir: nil, async: true, max_threads: nil)
+    def initialize(entry_set, output_dir: nil, async: true, max_threads: nil, max_size: nil)
       @entry_set = entry_set
       @output_dir = output_dir || Dir.pwd
       @async = async
       @max_threads = max_threads || DEFAULT_MAX_THREADS
+      @max_size = max_size
       @entries = Queue.new
       @successful_fetches = Queue.new
       @failed_fetches = Queue.new
@@ -66,7 +67,7 @@ module GsImgFetcher
     end
 
     def fetch_one(entry, dir)
-      Fetcher.new(entry, dir).tap do |fetcher|
+      Fetcher.new(entry, dir, max_size: @max_size).tap do |fetcher|
         fetcher.fetch && fetcher.save
 
         if fetcher.successful?

data/lib/gs_img_fetcher/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GsImgFetcher
-  VERSION = '0.1.2'
+  VERSION = '0.1.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gs_img_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Akihiko Ito