twitter_images 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa9d6738e520524fa87bebe3509598b51ff6b61c
-  data.tar.gz: d7021a59087e612655ec50a740ac3d02a559f94f
+  metadata.gz: 20dbabd1b8ed7d7b58e062c56e3a1912ee07b250
+  data.tar.gz: 8de26899a00b93007b8338a588514fbba231f781
 SHA512:
-  metadata.gz: 131398180fdb31450b64b8c1303637ed67b8190272d5e86206458a594c7c5b8222841190d26f0c87412ec8ccb3bcff2b635a48a15b8d10ee9c7ea39d261fd705
-  data.tar.gz: 552d52a07c0525ae55a54777785383c0823823cf21455f9090d0f16f2e3a1d9b0938b1b19b709355527bb20e1c1d06450c101fb964d0872394066d1a2fd7bf86
+  metadata.gz: 62d203269ee201430588c11da10ac95dac38cd514a68ba3e3dd589cff06083d61ec4ff2bc0a3e5b4e46ba324818901f875db37b30e80c1a54c919f4ac2756564
+  data.tar.gz: f91bd4f69ce6bd9fb4e8d33fb151d5a2ef29f21b37b363660d5a8b360b3ec1154fe5525718f8f4d02dc8cdb7118501f210d3518e7a2d2e337ae50bb336581f44

data/bin/twitter_images

@@ -3,4 +3,8 @@
 $LOAD_PATH.unshift(File.expand_path("../lib", __FILE__))
 require "twitter_images"
 
-TwitterImages::CLI.new.run
+downloader = TwitterImages::Downloader.new
+requester = TwitterImages::Requester.new(downloader)
+configuration = TwitterImages::Configuration.new(requester)
+
+TwitterImages::CLI.new(configuration).run

data/lib/twitter_images.rb

@@ -4,6 +4,7 @@ require "fileutils"
 require "json"
 require "oauth"
 require "ruby-progressbar"
+require "typhoeus"
 
 require "twitter_images/cli"
 require "twitter_images/configuration"

data/lib/twitter_images/cli.rb

@@ -1,9 +1,9 @@
 module TwitterImages
   class CLI
-    attr_accessor :configuration
+    attr_reader :configuration
 
-    def initialize()
-      @configuration = Configuration.new
+    def initialize(configuration)
+      @configuration = configuration
     end
 
     def run

data/lib/twitter_images/configuration.rb

@@ -1,11 +1,17 @@
 module TwitterImages
   class Configuration
-    attr_accessor :search, :directory
+    attr_reader   :requester
+    attr_accessor :search, :directory, :amount
+
+    def initialize(requester)
+      @requester = requester
+    end
 
     def prepare
       set_directory
       get_search
-      Requester.new(@search).start
+      get_amount
+      start
     end
 
     private
@@ -30,5 +36,15 @@ module TwitterImages
       @search = gets.chomp.gsub(/\s/, "%20")
       raise StandardError, "The search string is empty" if @search.empty?
     end
+
+    def get_amount
+      puts "Please enter the number of images to download: "
+      @amount = gets.chomp.to_i
+      raise StandardError, "The number is too small" if @amount <= 0
+    end
+
+    def start
+      requester.start(@search, @amount)
+    end
   end
 end

data/lib/twitter_images/downloader.rb

@@ -1,34 +1,34 @@
 module TwitterImages
   class Downloader
-    attr_accessor :response, :output, :images
+    attr_accessor :output, :images
 
-    def initialize(response)
-      @response = response
-    end
-
-    def download
-      get_output
-      parse_output
-      save_images
+    def download(all_links)
+      save_images(all_links)
     end
 
     private
 
-    def get_output
-      @output = JSON.parse(response.body)
+    def write_file(filename, data)
+      file = File.new(File.basename(filename), "wb")
+      file.write(data)
+      file.close
     end
 
-    def parse_output
-      @images = output.inspect.scan(/https:\/\/pbs.twimg.com\/media\/\w+\.(?:jpg|png|gif)/)
-      raise StandardError, "Couldn't find any images" unless @images.count > 0
-    end
+    def save_images(all_links)
+      progressbar = ProgressBar.create(:total => all_links.count)
+      hydra = Typhoeus::Hydra.new
 
-    def save_images
-      progressbar = ProgressBar.create(:total => images.count)
-      images.each do |src|
-        File.open(File.basename(src), "wb") { |f| f.write(open(src + ":large").read) }
-        progressbar.increment
+      all_links.each do |src|
+        request = Typhoeus::Request.new(src + ":large")
+        request.on_complete do |response|
+          write_file(src, response.body)
+          progressbar.increment
+        end
+        hydra.queue request
       end
+      hydra.run
+      puts "Downloaded #{all_links.count} pictures!"
     end
   end
+
 end

data/lib/twitter_images/requester.rb

@@ -1,31 +1,46 @@
 module TwitterImages
   class Requester
-    attr_accessor :search, :consumer_key, :access_token, :address, :https,
-                  :request, :response
+    attr_reader   :consumer_key, :access_token, :downloader, :search
+    attr_accessor :https, :address, :response, :max_id, :all_links
 
-    def initialize(search)
-      @search = search
+    def initialize(downloader)
+      @downloader = downloader
+      @all_links = []
     end
 
-    def start
-      set_address
-      setup_credentials
-      setup_https
-      build_request
-      issue_request
-      Downloader.new(@response).download
+    def start(search, amount)
+      check_env
+      get_links(search, amount)
+      download
+    end
+
+    def get_links(search, amount)
+      loop do
+        setup_address(search)
+        setup_https
+        issue_request
+        parse_response
+        break if all_links.count > amount
+      end
+      trim_links(amount)
     end
 
     private
 
-    def set_address
-      @address = URI("https://api.twitter.com/1.1/search/tweets.json?q=%23#{search}&mode=photos&count=100")
+    def setup_address(search)
+      unless @max_id.nil?
+        @address = URI("https://api.twitter.com/1.1/search/tweets.json?q=%23#{search}&result_type=recent&count=100&max_id=#{@max_id}")
+      else
+        @address = URI("https://api.twitter.com/1.1/search/tweets.json?q=%23#{search}&result_type=recent&count=100")
+      end
+    end
+
+    def consumer_key
+      OAuth::Consumer.new(ENV["CONSUMER_KEY"], ENV["CONSUMER_SECRET"])
     end
 
-    def setup_credentials
-      check_env
-      @consumer_key = OAuth::Consumer.new(ENV["CONSUMER_KEY"], ENV["CONSUMER_SECRET"])
-      @access_token = OAuth::Token.new(ENV["ACCESS_TOKEN"], ENV["ACCESS_SECRET"])
+    def access_token
+      OAuth::Token.new(ENV["ACCESS_TOKEN"], ENV["ACCESS_SECRET"])
     end
 
     def check_env
@@ -40,22 +55,43 @@ module TwitterImages
     end
 
     def setup_https
-      # Set up Net::HTTP to use SSL, which is required by Twitter.
       @https = Net::HTTP.new(address.host, address.port)
       https.use_ssl = true
-      https.verify_mode = OpenSSL::SSL::VERIFY_PEER
-    end
-
-    def build_request
-      # Build the request and authorize it with OAuth.
-      @request = Net::HTTP::Get.new(address.request_uri)
-      request.oauth!(https, consumer_key, access_token)
+      https.verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
 
     def issue_request
-      # Issue the request and return the response.
+      request = Net::HTTP::Get.new(address.request_uri)
+      request.oauth!(https, consumer_key, access_token)
       @https.start
       @response = https.request(request)
     end
+
+    def parse_response
+      filtered = JSON.parse(@response.body)
+      get_max_id(filtered)
+      collect_responses(filtered)
+    end
+
+    def get_max_id(filtered)
+      ids = []
+      filtered["statuses"].each do |tweet|
+        ids << tweet["id"]
+      end
+      @max_id = ids.min - 1
+    end
+
+    def collect_responses(filtered)
+      @all_links += filtered.inspect.scan(/https:\/\/pbs.twimg.com\/media\/\w+\.(?:jpg|png|gif)/)
+      @all_links.uniq!
+    end
+
+    def trim_links(amount)
+      @all_links = @all_links.slice!(0...amount)
+    end
+
+    def download
+      downloader.download(@all_links)
+    end
   end
 end

data/lib/twitter_images/version.rb

@@ -1,3 +1,3 @@
 module TwitterImages
-  VERSION = "0.0.4"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitter_images
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - Alexey Zabelin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-07 00:00:00.000000000 Z
+date: 2016-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fileutils
@@ -84,6 +84,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.7.5
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +168,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.22.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.22.6
 description: |-
   A CLI tool that downloads the most recent images from twitter based on the search terms provided. Please remember that you need to provide your own
   consumer key and consumer secret as well as the access token and access token secret. You