twitter_images 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/twitter_images +1 -5
- data/lib/twitter_images.rb +2 -0
- data/lib/twitter_images/cli.rb +31 -3
- data/lib/twitter_images/configuration.rb +15 -18
- data/lib/twitter_images/downloader.rb +6 -6
- data/lib/twitter_images/parser.rb +35 -0
- data/lib/twitter_images/requester.rb +21 -32
- data/lib/twitter_images/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f2efcf33670803f3ef9bb96883e19350d8c64aa1
|
4
|
+
data.tar.gz: c00d26f7028c0f8691af0d17175ac2600e5d5587
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62a688ae4b999dc38232e9d4d1cef356ab2b3de75d45545dc4f1cee9ab3b628f5f265e0c792cdb39a924991bc019b701bba0d4b2cd7a8ef65bcc7c4097983639
|
7
|
+
data.tar.gz: 3369306d11cb49cc8cc60cf076cbf558094fdea04dab12d4e16059d860a4c555669d765d81a47635bf0f6bb836267457548508d39dfa32c3f4d1262b3ed82bc8
|
data/bin/twitter_images
CHANGED
@@ -3,8 +3,4 @@
|
|
3
3
|
$LOAD_PATH.unshift(File.expand_path("../lib", __FILE__))
|
4
4
|
require "twitter_images"
|
5
5
|
|
6
|
-
|
7
|
-
requester = TwitterImages::Requester.new(downloader)
|
8
|
-
configuration = TwitterImages::Configuration.new(requester)
|
9
|
-
|
10
|
-
TwitterImages::CLI.new(configuration).run
|
6
|
+
TwitterImages::CLI.new(ARGV).run
|
data/lib/twitter_images.rb
CHANGED
@@ -5,9 +5,11 @@ require "json"
|
|
5
5
|
require "oauth"
|
6
6
|
require "ruby-progressbar"
|
7
7
|
require "typhoeus"
|
8
|
+
require "optparse"
|
8
9
|
|
9
10
|
require "twitter_images/cli"
|
10
11
|
require "twitter_images/configuration"
|
11
12
|
require "twitter_images/requester"
|
13
|
+
require "twitter_images/parser"
|
12
14
|
require "twitter_images/downloader"
|
13
15
|
require "twitter_images/version"
|
data/lib/twitter_images/cli.rb
CHANGED
@@ -2,12 +2,40 @@ module TwitterImages
|
|
2
2
|
class CLI
|
3
3
|
attr_reader :configuration
|
4
4
|
|
5
|
-
def initialize(
|
6
|
-
@
|
5
|
+
def initialize(argv)
|
6
|
+
@argv = argv
|
7
|
+
@configuration = Configuration.new
|
8
|
+
@options = {}
|
7
9
|
end
|
8
10
|
|
9
11
|
def run
|
10
|
-
|
12
|
+
parse_command_line_options
|
13
|
+
configuration.prepare(@options)
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def parse_command_line_options
|
19
|
+
global_options.parse!(@argv)
|
20
|
+
@options[:path] = @argv[0]
|
21
|
+
@options[:term] = @argv[1..-2].join
|
22
|
+
@options[:amount] = @argv[-1]
|
23
|
+
end
|
24
|
+
|
25
|
+
def global_options
|
26
|
+
OptionParser.new do |opts|
|
27
|
+
opts.banner = "usage: twitter_images [-v | --version] [-h | --help] [options] [path] [search terms] [amount]"
|
28
|
+
|
29
|
+
opts.on("-v", "--version", "Display the version and exit") do
|
30
|
+
puts "Version: #{TwitterImages::VERSION}"
|
31
|
+
exit
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on("-h", "--help", "Display this help message and exit") do
|
35
|
+
puts opts
|
36
|
+
exit
|
37
|
+
end
|
38
|
+
end
|
11
39
|
end
|
12
40
|
end
|
13
41
|
end
|
@@ -1,45 +1,42 @@
|
|
1
1
|
module TwitterImages
|
2
2
|
class Configuration
|
3
|
-
attr_reader :requester
|
3
|
+
attr_reader :options, :requester
|
4
4
|
attr_accessor :search, :directory, :amount
|
5
5
|
|
6
|
-
def initialize
|
7
|
-
@requester =
|
6
|
+
def initialize
|
7
|
+
@requester = Requester.new
|
8
8
|
end
|
9
9
|
|
10
|
-
def prepare
|
11
|
-
set_directory
|
12
|
-
get_search
|
13
|
-
get_amount
|
10
|
+
def prepare(options)
|
11
|
+
set_directory(options[:path])
|
12
|
+
get_search(options[:term])
|
13
|
+
get_amount(options[:amount])
|
14
14
|
start
|
15
15
|
end
|
16
16
|
|
17
17
|
private
|
18
18
|
|
19
|
-
def set_directory
|
20
|
-
|
21
|
-
@directory = gets.chomp
|
19
|
+
def set_directory(dir)
|
20
|
+
@directory = File.expand_path(dir)
|
22
21
|
raise StandardError, "Directory doesn't exist" unless directory_exists?
|
23
22
|
change_directory
|
24
23
|
end
|
25
24
|
|
26
25
|
def directory_exists?
|
27
|
-
Dir.exist?(
|
26
|
+
Dir.exist?(@directory)
|
28
27
|
end
|
29
28
|
|
30
29
|
def change_directory
|
31
|
-
Dir.chdir(
|
30
|
+
Dir.chdir(@directory)
|
32
31
|
end
|
33
32
|
|
34
|
-
def get_search
|
35
|
-
|
36
|
-
@search = gets.chomp.gsub(/\s/, "%20")
|
33
|
+
def get_search(term)
|
34
|
+
@search = term.chomp.gsub(/\s/, "%20").gsub(/\#/, "%23")
|
37
35
|
raise StandardError, "The search string is empty" if @search.empty?
|
38
36
|
end
|
39
37
|
|
40
|
-
def get_amount
|
41
|
-
|
42
|
-
@amount = gets.chomp.to_i
|
38
|
+
def get_amount(number)
|
39
|
+
@amount = number.to_i
|
43
40
|
raise StandardError, "The number is too small" if @amount <= 0
|
44
41
|
end
|
45
42
|
|
@@ -2,8 +2,8 @@ module TwitterImages
|
|
2
2
|
class Downloader
|
3
3
|
attr_accessor :output, :images
|
4
4
|
|
5
|
-
def download(
|
6
|
-
save_images(
|
5
|
+
def download(parsed_links)
|
6
|
+
save_images(parsed_links)
|
7
7
|
end
|
8
8
|
|
9
9
|
private
|
@@ -14,11 +14,11 @@ module TwitterImages
|
|
14
14
|
file.close
|
15
15
|
end
|
16
16
|
|
17
|
-
def save_images(
|
18
|
-
progressbar = ProgressBar.create(:total =>
|
17
|
+
def save_images(parsed_links)
|
18
|
+
progressbar = ProgressBar.create(:total => parsed_links.count)
|
19
19
|
hydra = Typhoeus::Hydra.new
|
20
20
|
|
21
|
-
|
21
|
+
parsed_links.each do |src|
|
22
22
|
request = Typhoeus::Request.new(src + ":large")
|
23
23
|
request.on_complete do |response|
|
24
24
|
write_file(src, response.body)
|
@@ -27,7 +27,7 @@ module TwitterImages
|
|
27
27
|
hydra.queue request
|
28
28
|
end
|
29
29
|
hydra.run
|
30
|
-
puts "Downloaded #{
|
30
|
+
puts "Downloaded #{parsed_links.count} pictures!"
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module TwitterImages
|
2
|
+
class Parser
|
3
|
+
attr_reader :response
|
4
|
+
attr_accessor :parsed_links, :max_id
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@parsed_links = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def parse(response)
|
11
|
+
parsed = JSON.parse(response.body)
|
12
|
+
get_max_id(parsed)
|
13
|
+
collect_responses(parsed)
|
14
|
+
end
|
15
|
+
|
16
|
+
def trim_links(amount)
|
17
|
+
@parsed_links = @parsed_links.slice!(0...amount)
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def get_max_id(parsed)
|
23
|
+
ids = []
|
24
|
+
parsed["statuses"].each do |tweet|
|
25
|
+
ids << tweet["id"]
|
26
|
+
end
|
27
|
+
@max_id = ids.min - 1
|
28
|
+
end
|
29
|
+
|
30
|
+
def collect_responses(parsed)
|
31
|
+
@parsed_links += parsed.inspect.scan(/https:\/\/pbs.twimg.com\/media\/\w+\.(?:jpg|png|gif)/)
|
32
|
+
@parsed_links.uniq!
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -1,11 +1,11 @@
|
|
1
1
|
module TwitterImages
|
2
2
|
class Requester
|
3
|
-
attr_reader :consumer_key, :access_token, :downloader, :search
|
4
|
-
attr_accessor :https, :address, :response
|
3
|
+
attr_reader :consumer_key, :access_token, :downloader, :search, :parser
|
4
|
+
attr_accessor :https, :address, :response
|
5
5
|
|
6
|
-
def initialize
|
7
|
-
@downloader =
|
8
|
-
@
|
6
|
+
def initialize
|
7
|
+
@downloader = Downloader.new
|
8
|
+
@parser = Parser.new
|
9
9
|
end
|
10
10
|
|
11
11
|
def start(search, amount)
|
@@ -16,22 +16,26 @@ module TwitterImages
|
|
16
16
|
|
17
17
|
def get_links(search, amount)
|
18
18
|
loop do
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
parse_response
|
23
|
-
break if all_links.count > amount
|
19
|
+
configure_request(search)
|
20
|
+
parse
|
21
|
+
break if @parser.parsed_links.count > amount
|
24
22
|
end
|
25
23
|
trim_links(amount)
|
26
24
|
end
|
27
25
|
|
28
26
|
private
|
29
27
|
|
28
|
+
def configure_request(search)
|
29
|
+
setup_address(search)
|
30
|
+
setup_https
|
31
|
+
issue_request
|
32
|
+
end
|
33
|
+
|
30
34
|
def setup_address(search)
|
31
|
-
unless
|
32
|
-
@address = URI("https://api.twitter.com/1.1/search/tweets.json?q
|
35
|
+
unless parser.max_id.nil?
|
36
|
+
@address = URI("https://api.twitter.com/1.1/search/tweets.json?q=#{search}&result_type=recent&count=100&max_id=#{parser.max_id}")
|
33
37
|
else
|
34
|
-
@address = URI("https://api.twitter.com/1.1/search/tweets.json?q
|
38
|
+
@address = URI("https://api.twitter.com/1.1/search/tweets.json?q=#{search}&result_type=recent&count=100")
|
35
39
|
end
|
36
40
|
end
|
37
41
|
|
@@ -67,31 +71,16 @@ module TwitterImages
|
|
67
71
|
@response = https.request(request)
|
68
72
|
end
|
69
73
|
|
70
|
-
def
|
71
|
-
|
72
|
-
get_max_id(filtered)
|
73
|
-
collect_responses(filtered)
|
74
|
-
end
|
75
|
-
|
76
|
-
def get_max_id(filtered)
|
77
|
-
ids = []
|
78
|
-
filtered["statuses"].each do |tweet|
|
79
|
-
ids << tweet["id"]
|
80
|
-
end
|
81
|
-
@max_id = ids.min - 1
|
82
|
-
end
|
83
|
-
|
84
|
-
def collect_responses(filtered)
|
85
|
-
@all_links += filtered.inspect.scan(/https:\/\/pbs.twimg.com\/media\/\w+\.(?:jpg|png|gif)/)
|
86
|
-
@all_links.uniq!
|
74
|
+
def parse
|
75
|
+
parser.parse(response)
|
87
76
|
end
|
88
77
|
|
89
78
|
def trim_links(amount)
|
90
|
-
|
79
|
+
parser.trim_links(amount)
|
91
80
|
end
|
92
81
|
|
93
82
|
def download
|
94
|
-
downloader.download(
|
83
|
+
downloader.download(parser.parsed_links)
|
95
84
|
end
|
96
85
|
end
|
97
86
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: twitter_images
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexey Zabelin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fileutils
|
@@ -199,6 +199,7 @@ files:
|
|
199
199
|
- lib/twitter_images/cli.rb
|
200
200
|
- lib/twitter_images/configuration.rb
|
201
201
|
- lib/twitter_images/downloader.rb
|
202
|
+
- lib/twitter_images/parser.rb
|
202
203
|
- lib/twitter_images/requester.rb
|
203
204
|
- lib/twitter_images/version.rb
|
204
205
|
homepage: https://github.com/alexeyzab/twitter_images
|