gs_img_fetcher 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b3ce17816770496a81e56ee290c9eda21bacddfb0e81c3fbce712ae50d477b64
|
4
|
+
data.tar.gz: e239f232dddbb0c201633837a0d680c245813b26420d0e316b0bf7baaeaec1de
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 732e6099d8c80fc0bad6bb891acdcaa77b262d717944b33e0ab8aaa699802312138765e6a278afa66af44653bb8e1ff5669c0198a15eda3976faf14fc8c38a68
|
7
|
+
data.tar.gz: 905ea2430a1aeca45d2d07ffd71ffeeaa9937381dde32e0a32213c756825057fcbaf31d0942f2f1003e11154d82c3bf815305c7618bb3ecf803da6563225c79b
|
data/README.md
CHANGED
@@ -53,7 +53,7 @@ Set the environment variable `NOLOG` to a truthy value to suppress logs.
|
|
53
53
|
#### Fetching a single image
|
54
54
|
```ruby
|
55
55
|
fetcher = GsImgFetcher::Fetcher.new(
|
56
|
-
GsImgFetcher::
|
56
|
+
GsImgFetcher::Entry.new('http://example.com/image.png'),
|
57
57
|
'output'
|
58
58
|
)
|
59
59
|
fetcher.fetch
|
@@ -63,18 +63,18 @@ fetcher.successful?
|
|
63
63
|
|
64
64
|
#### Fetching multiple images
|
65
65
|
```ruby
|
66
|
-
|
66
|
+
entry_set = GsImgFetcher::EntrySet.from_file('urls.txt')
|
67
67
|
# or
|
68
68
|
urls = ['http://example.com/image.png', 'http://example.com/image2.png']
|
69
|
-
entries = urls.map { |url| GsImgFetcher::
|
70
|
-
|
69
|
+
entries = urls.map { |url| GsImgFetcher::Entry.new(url) }
|
70
|
+
entry_set = GsImgFetcher::EntrySet.new(entries)
|
71
71
|
|
72
|
-
manager = GsImgFetcher::Manager.new(
|
72
|
+
manager = GsImgFetcher::Manager.new(entry_set, output_dir: 'output', async: false)
|
73
73
|
manager.setup.fetch
|
74
74
|
```
|
75
75
|
|
76
76
|
- `Manager` is what controls the entire process of handling the input and fetching and saving the images.
|
77
|
-
- `
|
77
|
+
- `EntrySet` is responsible for finding the input file and parsing, sanitizing and validating the list of URLs.
|
78
78
|
- `Fetcher` is responsible for downloading and saving images.
|
79
79
|
|
80
80
|
## Development
|
@@ -3,13 +3,13 @@
|
|
3
3
|
require 'active_support/core_ext/object/blank'
|
4
4
|
|
5
5
|
module GsImgFetcher
|
6
|
-
class
|
6
|
+
class EntrySet
|
7
7
|
class << self
|
8
8
|
def from_file(path)
|
9
9
|
entries = File.open(path, 'r') { |f| f.each_line.map(&:strip).map(&:presence) }
|
10
10
|
.compact
|
11
11
|
.uniq
|
12
|
-
.map { |url|
|
12
|
+
.map { |url| Entry.new(url) }
|
13
13
|
new(entries)
|
14
14
|
end
|
15
15
|
end
|
@@ -6,17 +6,17 @@ module GsImgFetcher
|
|
6
6
|
class Manager
|
7
7
|
class << self
|
8
8
|
def fetch(input_path, **opts)
|
9
|
-
|
10
|
-
new(
|
9
|
+
entry_set = EntrySet.from_file(input_path)
|
10
|
+
new(entry_set, **opts).setup.fetch
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
-
attr_reader :
|
14
|
+
attr_reader :entry_set, :entries, :successful_fetches, :failed_fetches
|
15
15
|
|
16
16
|
DEFAULT_MAX_THREADS = Concurrent.processor_count
|
17
17
|
|
18
|
-
def initialize(
|
19
|
-
@
|
18
|
+
def initialize(entry_set, output_dir: nil, async: true, max_threads: nil)
|
19
|
+
@entry_set = entry_set
|
20
20
|
@output_dir = output_dir || Dir.pwd
|
21
21
|
@async = async
|
22
22
|
@max_threads = max_threads || DEFAULT_MAX_THREADS
|
@@ -31,7 +31,7 @@ module GsImgFetcher
|
|
31
31
|
|
32
32
|
next unless entries.empty?
|
33
33
|
|
34
|
-
|
34
|
+
entry_set.valid_entries.each { |e| entries.push(e) }
|
35
35
|
end
|
36
36
|
end
|
37
37
|
|
@@ -78,7 +78,7 @@ module GsImgFetcher
|
|
78
78
|
end
|
79
79
|
|
80
80
|
def log_entries
|
81
|
-
LOGGER.info("Processing #{
|
81
|
+
LOGGER.info("Processing #{entry_set.entries.count} URLs (#{entry_set.valid_entries.count} valid, #{entry_set.invalid_entries.count} invalid)")
|
82
82
|
end
|
83
83
|
|
84
84
|
def log_result
|
data/lib/gs_img_fetcher.rb
CHANGED
@@ -3,8 +3,8 @@
|
|
3
3
|
require 'logger'
|
4
4
|
require 'gs_img_fetcher/version'
|
5
5
|
require 'gs_img_fetcher/fetcher'
|
6
|
-
require 'gs_img_fetcher/
|
7
|
-
require 'gs_img_fetcher/
|
6
|
+
require 'gs_img_fetcher/entry'
|
7
|
+
require 'gs_img_fetcher/entry_set'
|
8
8
|
require 'gs_img_fetcher/manager'
|
9
9
|
|
10
10
|
module GsImgFetcher
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gs_img_fetcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Akihiko Ito
|
@@ -187,9 +187,9 @@ files:
|
|
187
187
|
- exe/gs_img_fetcher
|
188
188
|
- gs_img_fetcher.gemspec
|
189
189
|
- lib/gs_img_fetcher.rb
|
190
|
+
- lib/gs_img_fetcher/entry.rb
|
191
|
+
- lib/gs_img_fetcher/entry_set.rb
|
190
192
|
- lib/gs_img_fetcher/fetcher.rb
|
191
|
-
- lib/gs_img_fetcher/input.rb
|
192
|
-
- lib/gs_img_fetcher/input_entry.rb
|
193
193
|
- lib/gs_img_fetcher/manager.rb
|
194
194
|
- lib/gs_img_fetcher/version.rb
|
195
195
|
homepage: https://github.com/AkihikoITOH/gs_img_fetcher
|