twitter_images 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/twitter_images +1 -5
- data/lib/twitter_images.rb +2 -0
- data/lib/twitter_images/cli.rb +31 -3
- data/lib/twitter_images/configuration.rb +15 -18
- data/lib/twitter_images/downloader.rb +6 -6
- data/lib/twitter_images/parser.rb +35 -0
- data/lib/twitter_images/requester.rb +21 -32
- data/lib/twitter_images/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f2efcf33670803f3ef9bb96883e19350d8c64aa1
|
4
|
+
data.tar.gz: c00d26f7028c0f8691af0d17175ac2600e5d5587
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62a688ae4b999dc38232e9d4d1cef356ab2b3de75d45545dc4f1cee9ab3b628f5f265e0c792cdb39a924991bc019b701bba0d4b2cd7a8ef65bcc7c4097983639
|
7
|
+
data.tar.gz: 3369306d11cb49cc8cc60cf076cbf558094fdea04dab12d4e16059d860a4c555669d765d81a47635bf0f6bb836267457548508d39dfa32c3f4d1262b3ed82bc8
|
data/bin/twitter_images
CHANGED
@@ -3,8 +3,4 @@
|
|
3
3
|
$LOAD_PATH.unshift(File.expand_path("../lib", __FILE__))
|
4
4
|
require "twitter_images"
|
5
5
|
|
6
|
-
|
7
|
-
requester = TwitterImages::Requester.new(downloader)
|
8
|
-
configuration = TwitterImages::Configuration.new(requester)
|
9
|
-
|
10
|
-
TwitterImages::CLI.new(configuration).run
|
6
|
+
TwitterImages::CLI.new(ARGV).run
|
data/lib/twitter_images.rb
CHANGED
@@ -5,9 +5,11 @@ require "json"
|
|
5
5
|
require "oauth"
|
6
6
|
require "ruby-progressbar"
|
7
7
|
require "typhoeus"
|
8
|
+
require "optparse"
|
8
9
|
|
9
10
|
require "twitter_images/cli"
|
10
11
|
require "twitter_images/configuration"
|
11
12
|
require "twitter_images/requester"
|
13
|
+
require "twitter_images/parser"
|
12
14
|
require "twitter_images/downloader"
|
13
15
|
require "twitter_images/version"
|
data/lib/twitter_images/cli.rb
CHANGED
@@ -2,12 +2,40 @@ module TwitterImages
|
|
2
2
|
class CLI
|
3
3
|
attr_reader :configuration
|
4
4
|
|
5
|
-
def initialize(
|
6
|
-
@
|
5
|
+
def initialize(argv)
|
6
|
+
@argv = argv
|
7
|
+
@configuration = Configuration.new
|
8
|
+
@options = {}
|
7
9
|
end
|
8
10
|
|
9
11
|
def run
|
10
|
-
|
12
|
+
parse_command_line_options
|
13
|
+
configuration.prepare(@options)
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def parse_command_line_options
|
19
|
+
global_options.parse!(@argv)
|
20
|
+
@options[:path] = @argv[0]
|
21
|
+
@options[:term] = @argv[1..-2].join
|
22
|
+
@options[:amount] = @argv[-1]
|
23
|
+
end
|
24
|
+
|
25
|
+
def global_options
|
26
|
+
OptionParser.new do |opts|
|
27
|
+
opts.banner = "usage: twitter_images [-v | --version] [-h | --help] [options] [path] [search terms] [amount]"
|
28
|
+
|
29
|
+
opts.on("-v", "--version", "Display the version and exit") do
|
30
|
+
puts "Version: #{TwitterImages::VERSION}"
|
31
|
+
exit
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on("-h", "--help", "Display this help message and exit") do
|
35
|
+
puts opts
|
36
|
+
exit
|
37
|
+
end
|
38
|
+
end
|
11
39
|
end
|
12
40
|
end
|
13
41
|
end
|
@@ -1,45 +1,42 @@
|
|
1
1
|
module TwitterImages
|
2
2
|
class Configuration
|
3
|
-
attr_reader :requester
|
3
|
+
attr_reader :options, :requester
|
4
4
|
attr_accessor :search, :directory, :amount
|
5
5
|
|
6
|
-
def initialize
|
7
|
-
@requester =
|
6
|
+
def initialize
|
7
|
+
@requester = Requester.new
|
8
8
|
end
|
9
9
|
|
10
|
-
def prepare
|
11
|
-
set_directory
|
12
|
-
get_search
|
13
|
-
get_amount
|
10
|
+
def prepare(options)
|
11
|
+
set_directory(options[:path])
|
12
|
+
get_search(options[:term])
|
13
|
+
get_amount(options[:amount])
|
14
14
|
start
|
15
15
|
end
|
16
16
|
|
17
17
|
private
|
18
18
|
|
19
|
-
def set_directory
|
20
|
-
|
21
|
-
@directory = gets.chomp
|
19
|
+
def set_directory(dir)
|
20
|
+
@directory = File.expand_path(dir)
|
22
21
|
raise StandardError, "Directory doesn't exist" unless directory_exists?
|
23
22
|
change_directory
|
24
23
|
end
|
25
24
|
|
26
25
|
def directory_exists?
|
27
|
-
Dir.exist?(
|
26
|
+
Dir.exist?(@directory)
|
28
27
|
end
|
29
28
|
|
30
29
|
def change_directory
|
31
|
-
Dir.chdir(
|
30
|
+
Dir.chdir(@directory)
|
32
31
|
end
|
33
32
|
|
34
|
-
def get_search
|
35
|
-
|
36
|
-
@search = gets.chomp.gsub(/\s/, "%20")
|
33
|
+
def get_search(term)
|
34
|
+
@search = term.chomp.gsub(/\s/, "%20").gsub(/\#/, "%23")
|
37
35
|
raise StandardError, "The search string is empty" if @search.empty?
|
38
36
|
end
|
39
37
|
|
40
|
-
def get_amount
|
41
|
-
|
42
|
-
@amount = gets.chomp.to_i
|
38
|
+
def get_amount(number)
|
39
|
+
@amount = number.to_i
|
43
40
|
raise StandardError, "The number is too small" if @amount <= 0
|
44
41
|
end
|
45
42
|
|
@@ -2,8 +2,8 @@ module TwitterImages
|
|
2
2
|
class Downloader
|
3
3
|
attr_accessor :output, :images
|
4
4
|
|
5
|
-
def download(
|
6
|
-
save_images(
|
5
|
+
def download(parsed_links)
|
6
|
+
save_images(parsed_links)
|
7
7
|
end
|
8
8
|
|
9
9
|
private
|
@@ -14,11 +14,11 @@ module TwitterImages
|
|
14
14
|
file.close
|
15
15
|
end
|
16
16
|
|
17
|
-
def save_images(
|
18
|
-
progressbar = ProgressBar.create(:total =>
|
17
|
+
def save_images(parsed_links)
|
18
|
+
progressbar = ProgressBar.create(:total => parsed_links.count)
|
19
19
|
hydra = Typhoeus::Hydra.new
|
20
20
|
|
21
|
-
|
21
|
+
parsed_links.each do |src|
|
22
22
|
request = Typhoeus::Request.new(src + ":large")
|
23
23
|
request.on_complete do |response|
|
24
24
|
write_file(src, response.body)
|
@@ -27,7 +27,7 @@ module TwitterImages
|
|
27
27
|
hydra.queue request
|
28
28
|
end
|
29
29
|
hydra.run
|
30
|
-
puts "Downloaded #{
|
30
|
+
puts "Downloaded #{parsed_links.count} pictures!"
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module TwitterImages
|
2
|
+
class Parser
|
3
|
+
attr_reader :response
|
4
|
+
attr_accessor :parsed_links, :max_id
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@parsed_links = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def parse(response)
|
11
|
+
parsed = JSON.parse(response.body)
|
12
|
+
get_max_id(parsed)
|
13
|
+
collect_responses(parsed)
|
14
|
+
end
|
15
|
+
|
16
|
+
def trim_links(amount)
|
17
|
+
@parsed_links = @parsed_links.slice!(0...amount)
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def get_max_id(parsed)
|
23
|
+
ids = []
|
24
|
+
parsed["statuses"].each do |tweet|
|
25
|
+
ids << tweet["id"]
|
26
|
+
end
|
27
|
+
@max_id = ids.min - 1
|
28
|
+
end
|
29
|
+
|
30
|
+
def collect_responses(parsed)
|
31
|
+
@parsed_links += parsed.inspect.scan(/https:\/\/pbs.twimg.com\/media\/\w+\.(?:jpg|png|gif)/)
|
32
|
+
@parsed_links.uniq!
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -1,11 +1,11 @@
|
|
1
1
|
module TwitterImages
|
2
2
|
class Requester
|
3
|
-
attr_reader :consumer_key, :access_token, :downloader, :search
|
4
|
-
attr_accessor :https, :address, :response
|
3
|
+
attr_reader :consumer_key, :access_token, :downloader, :search, :parser
|
4
|
+
attr_accessor :https, :address, :response
|
5
5
|
|
6
|
-
def initialize
|
7
|
-
@downloader =
|
8
|
-
@
|
6
|
+
def initialize
|
7
|
+
@downloader = Downloader.new
|
8
|
+
@parser = Parser.new
|
9
9
|
end
|
10
10
|
|
11
11
|
def start(search, amount)
|
@@ -16,22 +16,26 @@ module TwitterImages
|
|
16
16
|
|
17
17
|
def get_links(search, amount)
|
18
18
|
loop do
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
parse_response
|
23
|
-
break if all_links.count > amount
|
19
|
+
configure_request(search)
|
20
|
+
parse
|
21
|
+
break if @parser.parsed_links.count > amount
|
24
22
|
end
|
25
23
|
trim_links(amount)
|
26
24
|
end
|
27
25
|
|
28
26
|
private
|
29
27
|
|
28
|
+
def configure_request(search)
|
29
|
+
setup_address(search)
|
30
|
+
setup_https
|
31
|
+
issue_request
|
32
|
+
end
|
33
|
+
|
30
34
|
def setup_address(search)
|
31
|
-
unless
|
32
|
-
@address = URI("https://api.twitter.com/1.1/search/tweets.json?q
|
35
|
+
unless parser.max_id.nil?
|
36
|
+
@address = URI("https://api.twitter.com/1.1/search/tweets.json?q=#{search}&result_type=recent&count=100&max_id=#{parser.max_id}")
|
33
37
|
else
|
34
|
-
@address = URI("https://api.twitter.com/1.1/search/tweets.json?q
|
38
|
+
@address = URI("https://api.twitter.com/1.1/search/tweets.json?q=#{search}&result_type=recent&count=100")
|
35
39
|
end
|
36
40
|
end
|
37
41
|
|
@@ -67,31 +71,16 @@ module TwitterImages
|
|
67
71
|
@response = https.request(request)
|
68
72
|
end
|
69
73
|
|
70
|
-
def
|
71
|
-
|
72
|
-
get_max_id(filtered)
|
73
|
-
collect_responses(filtered)
|
74
|
-
end
|
75
|
-
|
76
|
-
def get_max_id(filtered)
|
77
|
-
ids = []
|
78
|
-
filtered["statuses"].each do |tweet|
|
79
|
-
ids << tweet["id"]
|
80
|
-
end
|
81
|
-
@max_id = ids.min - 1
|
82
|
-
end
|
83
|
-
|
84
|
-
def collect_responses(filtered)
|
85
|
-
@all_links += filtered.inspect.scan(/https:\/\/pbs.twimg.com\/media\/\w+\.(?:jpg|png|gif)/)
|
86
|
-
@all_links.uniq!
|
74
|
+
def parse
|
75
|
+
parser.parse(response)
|
87
76
|
end
|
88
77
|
|
89
78
|
def trim_links(amount)
|
90
|
-
|
79
|
+
parser.trim_links(amount)
|
91
80
|
end
|
92
81
|
|
93
82
|
def download
|
94
|
-
downloader.download(
|
83
|
+
downloader.download(parser.parsed_links)
|
95
84
|
end
|
96
85
|
end
|
97
86
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: twitter_images
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexey Zabelin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fileutils
|
@@ -199,6 +199,7 @@ files:
|
|
199
199
|
- lib/twitter_images/cli.rb
|
200
200
|
- lib/twitter_images/configuration.rb
|
201
201
|
- lib/twitter_images/downloader.rb
|
202
|
+
- lib/twitter_images/parser.rb
|
202
203
|
- lib/twitter_images/requester.rb
|
203
204
|
- lib/twitter_images/version.rb
|
204
205
|
homepage: https://github.com/alexeyzab/twitter_images
|