oii_twitter_goodies 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17ae17effe9efc8b71ad239c5031e867d29251ce
-  data.tar.gz: 31092a565c077b78bf3e6085f368c3f4ce4d50c2
+  metadata.gz: ff7c76ac31bcf7f46a4776f0e5d60df17e544056
+  data.tar.gz: a07e57b3b8d5d4d0c1e7fb5783b949519eb1e5f6
 SHA512:
-  metadata.gz: 5ab600404a84ac7d4bd9ae9f78d1d8a2d8d6f2a96d489d93cceeb4d6747c0336e32f79a5c059fc717947742419d7b0a9ab7f44e438d19b3d9302cdd0685f2a10
-  data.tar.gz: 83b45e92c3014c84b1d4d692bf149873bba357b486648a8896254364fa6e31e19129d8f5b3624546558599621704fe471289703895b3947a463061a4cbc99652
+  metadata.gz: a115f0566025572ac5f1b36258c7d0d9f6858662cab58b90409f921022a51fa1496d16c833e958799411ced02613fb8d306d085667a50379a2ef0f70f68249d0
+  data.tar.gz: 6962758ff69a74a06d4e6a33cbf5b0e7db49bd1642845c511533fb4a1d0d45e0b14916283d06d769f5b9d0744f06a81381dc3d7ba59c1ec7a717d86170c37ed6

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Devin Gaffney
+Copyright (c) 2012 Devin Gaffney
 
 MIT License
 
@@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/oii_twitter_goodies/extensions/twitter_api_utils.rb

@@ -1,30 +1,30 @@
-module Twitter
-  module API
-    module Utils
-      def objects_from_response(klass, request_method, path, options={})
-        response = send(request_method.to_sym, path, options)
-        if response.class == Hash && response[:body]
-          response = response[:body]
-          objects_from_array(klass, response)
-        else
-          return response
-        end
-      end
-
-      def object_from_response(klass, request_method, path, options={})
-        response = send(request_method.to_sym, path, options)
-        return klass.from_response(response) || response
-      end
-      
-      def cursor_from_response(collection_name, klass, request_method, path, options, method_name)
-        merge_default_cursor!(options)
-        response = send(request_method.to_sym, path, options)
-        if (response.class == Hash || response.class == BSON::OrderedHash) && !response.keys.include?(:body)
-          Twitter::Cursor.from_response({:body => response}, collection_name.to_sym, klass, self, method_name, options)
-        else
-          Twitter::Cursor.from_response(response, collection_name.to_sym, klass, self, method_name, options)
-        end
-      end
-    end
-  end
-end
+# module Twitter
+#   module API
+#     module Utils
+#       def objects_from_response(klass, request_method, path, options={})
+#         response = send(request_method.to_sym, path, options)
+#         if response.class == Hash && response[:body]
+#           response = response[:body]
+#           objects_from_array(klass, response)
+#         else
+#           return response
+#         end
+#       end
+# 
+#       def object_from_response(klass, request_method, path, options={})
+#         response = send(request_method.to_sym, path, options)
+#         return klass.from_response(response) || response
+#       end
+#       
+#       def cursor_from_response(collection_name, klass, request_method, path, options, method_name)
+#         merge_default_cursor!(options)
+#         response = send(request_method.to_sym, path, options)
+#         if (response.class == Hash || response.class == BSON::OrderedHash) && !response.keys.include?(:body)
+#           Twitter::Cursor.from_response({:body => response}, collection_name.to_sym, klass, self, method_name, options)
+#         else
+#           Twitter::Cursor.from_response(response, collection_name.to_sym, klass, self, method_name, options)
+#         end
+#       end
+#     end
+#   end
+# end

data/lib/oii_twitter_goodies/extensions/twitter_client.rb

@@ -1,17 +1,46 @@
-module Twitter
-  class Client
-    def get(path, params={})
-      lookup = TwitterAPICall.first(:url => path.to_s, :params => params)
-      if lookup
-        return lookup.data
-      else
-        response = request(:get, path, params)
-        if response[:status] && response[:status] == 200
-          lookup = TwitterAPICall.new(:url => path.to_s, :params => params, :data => response[:body])
-          lookup.save!
-        end
-        return response
-      end
-    end
-  end
-end
+# module Twitter
+#   class Client
+#     def get(path, params={})
+#       lookup = TwitterAPICall.first(:url => path.to_s, :params => params)
+#       if lookup
+#         return lookup.data
+#       else
+#         response = request(:get, path, params)
+#         if response[:status] && response[:status] == 200
+#           lookup = TwitterAPICall.new(:url => path.to_s, :params => params, :data => response[:body])
+#           lookup.save!
+#         end
+#         return response
+#       end
+#     end
+#     
+#     def post(path, params={})
+#       lookup = TwitterAPICall.first(:url => path.to_s, :params => params)
+#       if lookup
+#         return lookup.data
+#       else
+#         signature_params = params.values.any?{|value| value.respond_to?(:to_io)} ? {} : params
+#         response = request(:post, path, params, signature_params)
+#         if response[:status] && response[:status] == 200
+#           lookup = TwitterAPICall.new(:url => path.to_s, :params => params, :data => response[:body])
+#           lookup.save!
+#         end
+#         return response
+#       end
+#     end
+# 
+#     def put(path, params={})
+#       lookup = TwitterAPICall.first(:url => path.to_s, :params => params)
+#       if lookup
+#         return lookup.data
+#       else
+#         response = request(:put, path, params)
+#         if response[:status] && response[:status] == 200
+#           lookup = TwitterAPICall.new(:url => path.to_s, :params => params, :data => response[:body])
+#           lookup.save!
+#         end
+#         return response
+#       end
+#     end
+#   end
+# end

data/lib/oii_twitter_goodies/lib/followers_for_users.rb

@@ -32,7 +32,19 @@ class FollowersForUsers
     puts "Done!"
     return results
   end
-    
+  
+  def grab_followers_iterative(&block)
+    @screen_names.each do |screen_name|
+      puts "Current progress: #{@screen_names.index(screen_name)}/#{@screen_names.length} started"
+      user = user_data_for(screen_name).first
+      follower_ids = follower_ids_for(screen_name)
+      follower_ids.each_slice(100) do |follower_id_slice|
+        block.call([user, user_data_for_set(follower_id_slice)])
+      end
+      puts "Current progress: ##{@screen_names.index(screen_name)}/#{@screen_names.length} complete"      
+    end
+  end
+  
   def follower_ids_for(screen_name)
     return direction_ids_for screen_name, "follower"
   end
@@ -46,19 +58,68 @@ class FollowersForUsers
     ids = []
     while cursor != 0
       puts cursor
-      data = Hashie::Mash[@client.send(direction+"_ids", screen_name, :cursor => cursor).attrs]
-      ids = ids|data.ids
+      begin
+        data = Hashie::Mash[@client.send(direction+"_ids", screen_name, :cursor => cursor).attrs]
+      rescue Twitter::Error::BadGateway
+        puts "Got Twitter::Error::BadGateway error - usually this is just a missed connect from Twitter."
+        sleep(10)
+        retry
+      rescue Twitter::Error::TooManyRequests
+        puts "Got Twitter::Error::TooManyRequests error - we are rate limited and will sleep for 15 minutes. See you in a bit."
+        sleep(15*60)
+        retry
+      rescue Twitter::Error::ClientError
+        puts "Got Twitter::Error::ClientError error - something weird happened and we're going to retry."
+        sleep(5)
+        retry
+      rescue Twitter::Error::InternalServerError
+        puts "Got Twitter::Error::InternalServerError error - something weird happened and we're going to retry."
+        sleep(5)
+        retry
+      rescue Twitter::Error::ServiceUnavailable
+        puts "Got Twitter::Error::ServiceUnavailable error - Fail whaled. We'll hit those guys again in 10"
+        sleep(10)
+        retry
+      end
+      ids << data.ids
       cursor = data["next_cursor"]
     end
-    return ids
+    return ids.flatten.uniq
   end
   
   def user_data_for(screen_names)
     screen_names = [screen_names].flatten
     user_data = []
+    size = screen_names.length/100
     screen_names.each_slice(100) do |screen_name_set|
-      user_data = [user_data|@client.users(screen_name_set)].flatten
+      user_data << user_data_for_set(screen_name_set)
+    end
+    user_data.flatten
+  end
+  
+  def user_data_for_set(screen_name_set)
+    begin
+      return @client.users(screen_name_set)
+    rescue Twitter::Error::BadGateway
+      puts "Got Twitter::Error::BadGateway error - usually this is just a missed connect from Twitter."
+      sleep(10)
+      retry
+    rescue Twitter::Error::TooManyRequests
+      puts "Got Twitter::Error::TooManyRequests error - we are rate limited and will sleep for 15 minutes. See you in a bit."
+      sleep(15*60)
+      retry
+    rescue Twitter::Error::ClientError
+      puts "Got Twitter::Error::ClientError error - something weird happened and we're going to retry."
+      sleep(5)
+      retry
+    rescue Twitter::Error::InternalServerError
+      puts "Got Twitter::Error::InternalServerError error - something weird happened and we're going to retry."
+      sleep(5)
+      retry
+    rescue Twitter::Error::ServiceUnavailable
+      puts "Got Twitter::Error::ServiceUnavailable error - Fail whaled. We'll hit those guys again in 10"
+      sleep(10)
+      retry
     end
-    user_data
   end
 end

data/lib/oii_twitter_goodies/model/place.rb

@@ -23,6 +23,7 @@ class Place
     
   def self.new_from_raw(place, tweet_id)
     return if place.nil?
+    place = place.attrs if place.class == Twitter::Place
     place = Hashie::Mash[place]
     obj = self.new
     obj.twitter_id   = place["twitter_id"]
@@ -32,6 +33,7 @@ class Place
     obj.full_name    = place["full_name"]
     obj.country_code = place["country_code"]
     obj.country      = place["country"]
+    place["bounding_box"] = place["bounding_box"].attrs if Twitter::Place
     bounding_box = BoundingBox.new_from_raw(place["bounding_box"], obj._id)
     if bounding_box
       obj.bounding_box = bounding_box

data/lib/oii_twitter_goodies/model/tweet.rb

@@ -48,6 +48,7 @@ class Tweet
 
   def self.new_from_raw(tweet, user_id)
     return if tweet.nil?
+    tweet = tweet.attrs if tweet.class == Twitter::Tweet
     tweet = Hashie::Mash[tweet]
     obj = self.new
     obj.created_at                = tweet["created_at"]

data/lib/oii_twitter_goodies/model/user.rb

@@ -47,6 +47,7 @@ class User
   end
   
   def self.new_from_raw(status)
+    status = status.attrs if status.class == Twitter::User
     status = Hashie::Mash[status]
     obj = self.new
     obj.twitter_id                         = status["user"].nil? ? status["id"]                                 : status["user"]["id"]

data/lib/oii_twitter_goodies/version.rb

@@ -1,3 +1,3 @@
 module OIITwitterGoodies
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/lib/oii_twitter_goodies.rb

@@ -7,8 +7,8 @@ require 'hashie'
 require 'typhoeus'
 require 'json'
 
-require "oii_twitter_goodies/version"
+require File.dirname(__FILE__)+'/oii_twitter_goodies/version.rb'
 
-Dir[File.dirname(__FILE__)+'/oii_twitter_goodies/extensions/*.rb'].each {|file| require file }
-Dir[File.dirname(__FILE__)+'/oii_twitter_goodies/lib/*.rb'].each {|file| require file }
-Dir[File.dirname(__FILE__)+'/oii_twitter_goodies/model/*.rb'].each {|file| require file }
+Dir[File.dirname(__FILE__) + '/oii_twitter_goodies/extensions/*.rb'].each {|file| require file }
+Dir[File.dirname(__FILE__) + '/oii_twitter_goodies/lib/*.rb'].each {|file| require file }
+Dir[File.dirname(__FILE__) + '/oii_twitter_goodies/model/*.rb'].each {|file| require file }

data/oii_twitter_goodies.gemspec

@@ -22,17 +22,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "hashie"
   gem.add_development_dependency "typhoeus"
   gem.add_development_dependency "json"
-  gem.add_dependency "bundler", ">= 1.0.0"
-  gem.add_dependency "mongo_mapper"
-  gem.add_dependency "bcrypt-ruby"
-  gem.add_dependency "bson_ext"
-  gem.add_dependency "oauth"
-  gem.add_dependency "twitter"
-  gem.add_dependency "tweetstream"
-  gem.add_dependency "minitest"
-  gem.add_dependency "hashie"
-  gem.add_dependency "typhoeus"
-  gem.add_dependency "json"
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oii_twitter_goodies
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Devin Gaffney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-14 00:00:00.000000000 Z
+date: 2013-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,160 +164,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: mongo_mapper
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: bcrypt-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: bson_ext
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: oauth
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: twitter
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: tweetstream
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: hashie
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: typhoeus
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: OII Twitter Goodies!
 email:
 - itsme@devingaffney.com
@@ -325,9 +171,6 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .DS_Store
-- .gitignore
-- Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -358,6 +201,7 @@ files:
 - lib/oii_twitter_goodies/model/user_mention.rb
 - lib/oii_twitter_goodies/version.rb
 - oii_twitter_goodies.gemspec
+- pkg/oii_twitter_goodies-0.0.1.gem
 homepage: http://github.com/DGaffney/oii_twitter_goodies
 licenses: []
 metadata: {}

data/.gitignore

@@ -1,17 +0,0 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp

data/Gemfile

@@ -1,12 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in oii_twitter_goodies.gemspec
-gem 'bundler'
-gem 'mongo_mapper'
-gem 'oauth'
-gem 'twitter'
-gem 'tweetstream'
-gem 'hashie'
-gem 'typhoeus'
-gem 'json'
-gemspec