oii_twitter_goodies 0.0.74 → 0.0.81

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76eec990a606faf8e1ad18ad5422e272456bd921
-  data.tar.gz: f0fe5a6822059182401d4dd5f322b1909f2a2ba7
+  metadata.gz: caa9222364f19589980c52ebf5964db505e2e116
+  data.tar.gz: 7622951d0744f2b7ed1a7625cd6fc43ff84b7106
 SHA512:
-  metadata.gz: d16b0bfc149a3fd52dd66fdfcc9d580ce6cf3cc21997101c9f6a65686753535f2d350960ba9f4f06002f8d1c0898fb043495235fe47c63aa2aab8f3ab77115aa
-  data.tar.gz: 4e734866c7abe41aafb18dd0c5f3d62bc9ad6ebe5c2d867ddd8846d299c00ca1e73129e972ee59f65ded16bfea9eeb0b230c2054e6c111eb4dc664b11dbe8091
+  metadata.gz: 26ed52798666b5bb72f008ad22374968a5a19aaa6dcd4961284a91c6a5bcf21bf152c6625094142ec50182eeb0658cc733fb992e12d16550aa4e7d7e1e5cfe3a
+  data.tar.gz: 35e00579537cffa047068a2ee2afa87bdbd361bb494df5d3d7fcfd8957f468d381090fb8dabe425c0177d5b7a4de6f4a2c2867dd4adda3cecea583fe92318237

data/Gemfile

@@ -1,6 +1,3 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in oii_twitter_goodies.gemspec
 gem 'bundler'
 gem 'mongo_mapper'
 gem 'oauth'
@@ -9,4 +6,4 @@ gem 'tweetstream'
 gem 'hashie'
 gem 'typhoeus'
 gem 'json'
-gemspec
+

data/Gemfile.lock

@@ -0,0 +1,75 @@
+GEM
+  specs:
+    activemodel (3.2.12)
+      activesupport (= 3.2.12)
+      builder (~> 3.0.0)
+    activesupport (3.2.12)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    addressable (2.3.3)
+    bson (1.8.3)
+    builder (3.0.4)
+    cookiejar (0.3.0)
+    daemons (1.1.9)
+    em-http-request (1.0.3)
+      addressable (>= 2.2.3)
+      cookiejar
+      em-socksify
+      eventmachine (>= 1.0.0.beta.4)
+      http_parser.rb (>= 0.5.3)
+    em-socksify (0.2.1)
+      eventmachine (>= 1.0.0.beta.4)
+    em-twitter (0.2.1)
+      eventmachine (~> 1.0)
+      http_parser.rb (~> 0.5)
+      simple_oauth (~> 0.1)
+    ethon (0.5.10)
+      ffi (~> 1.3.0)
+      mime-types (~> 1.18)
+    eventmachine (1.0.3)
+    faraday (0.8.6)
+      multipart-post (~> 1.1)
+    ffi (1.3.1)
+    hashie (2.0.2)
+    http_parser.rb (0.5.3)
+    i18n (0.6.4)
+    json (1.7.7)
+    mime-types (1.21)
+    mongo (1.8.3)
+      bson (~> 1.8.3)
+    mongo_mapper (0.12.0)
+      activemodel (~> 3.0)
+      activesupport (~> 3.0)
+      plucky (~> 0.5.2)
+    multi_json (1.6.1)
+    multipart-post (1.2.0)
+    oauth (0.4.7)
+    plucky (0.5.2)
+      mongo (~> 1.5)
+    simple_oauth (0.2.0)
+    tweetstream (2.4.0)
+      daemons (~> 1.1)
+      em-http-request (~> 1.0.2)
+      em-twitter (~> 0.2)
+      twitter (~> 4.0)
+      yajl-ruby (~> 1.1)
+    twitter (4.6.1)
+      faraday (~> 0.8, < 0.10)
+      multi_json (~> 1.0)
+      simple_oauth (~> 0.2)
+    typhoeus (0.6.2)
+      ethon (~> 0.5.10)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  hashie
+  json
+  mongo_mapper
+  oauth
+  tweetstream
+  twitter
+  typhoeus

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Devin Gaffney
+Copyright (c) 2012 Devin Gaffney
 
 MIT License
 
@@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/oii_twitter_goodies.rb

@@ -7,8 +7,8 @@ require 'hashie'
 require 'typhoeus'
 require 'json'
 
-require "oii_twitter_goodies/version"
+require File.dirname(__FILE__)+'/oii_twitter_goodies/version.rb'
 
-Dir[File.dirname(__FILE__)+'/oii_twitter_goodies/extensions/*.rb'].each {|file| require file }
-Dir[File.dirname(__FILE__)+'/oii_twitter_goodies/lib/*.rb'].each {|file| require file }
-Dir[File.dirname(__FILE__)+'/oii_twitter_goodies/model/*.rb'].each {|file| require file }
+Dir[File.dirname(__FILE__) + '/oii_twitter_goodies/extensions/*.rb'].each {|file| require file }
+Dir[File.dirname(__FILE__) + '/oii_twitter_goodies/lib/*.rb'].each {|file| require file }
+Dir[File.dirname(__FILE__) + '/oii_twitter_goodies/model/*.rb'].each {|file| require file }

data/lib/oii_twitter_goodies/extensions/twitter_api_utils.rb

@@ -1,30 +1,30 @@
-module Twitter
-  module API
-    module Utils
-      def objects_from_response(klass, request_method, path, options={})
-        response = send(request_method.to_sym, path, options)
-        if response.class == Hash && response[:body]
-          response = response[:body]
-          objects_from_array(klass, response)
-        else
-          return response
-        end
-      end
-
-      def object_from_response(klass, request_method, path, options={})
-        response = send(request_method.to_sym, path, options)
-        return klass.from_response(response) || response
-      end
-      
-      def cursor_from_response(collection_name, klass, request_method, path, options, method_name)
-        merge_default_cursor!(options)
-        response = send(request_method.to_sym, path, options)
-        if (response.class == Hash || response.class == BSON::OrderedHash) && !response.keys.include?(:body)
-          Twitter::Cursor.from_response({:body => response}, collection_name.to_sym, klass, self, method_name, options)
-        else
-          Twitter::Cursor.from_response(response, collection_name.to_sym, klass, self, method_name, options)
-        end
-      end
-    end
-  end
-end
+# module Twitter
+#   module API
+#     module Utils
+#       def objects_from_response(klass, request_method, path, options={})
+#         response = send(request_method.to_sym, path, options)
+#         if response.class == Hash && response[:body]
+#           response = response[:body]
+#           objects_from_array(klass, response)
+#         else
+#           return response
+#         end
+#       end
+# 
+#       def object_from_response(klass, request_method, path, options={})
+#         response = send(request_method.to_sym, path, options)
+#         return klass.from_response(response) || response
+#       end
+#       
+#       def cursor_from_response(collection_name, klass, request_method, path, options, method_name)
+#         merge_default_cursor!(options)
+#         response = send(request_method.to_sym, path, options)
+#         if (response.class == Hash || response.class == BSON::OrderedHash) && !response.keys.include?(:body)
+#           Twitter::Cursor.from_response({:body => response}, collection_name.to_sym, klass, self, method_name, options)
+#         else
+#           Twitter::Cursor.from_response(response, collection_name.to_sym, klass, self, method_name, options)
+#         end
+#       end
+#     end
+#   end
+# end

data/lib/oii_twitter_goodies/extensions/twitter_client.rb

@@ -1,17 +1,46 @@
-module Twitter
-  class Client
-    def get(path, params={})
-      lookup = TwitterAPICall.first(:url => path.to_s, :params => params)
-      if lookup
-        return lookup.data
-      else
-        response = request(:get, path, params)
-        if response[:status] && response[:status] == 200
-          lookup = TwitterAPICall.new(:url => path.to_s, :params => params, :data => response[:body])
-          lookup.save!
-        end
-        return response
-      end
-    end
-  end
-end
+# module Twitter
+#   class Client
+#     def get(path, params={})
+#       lookup = TwitterAPICall.first(:url => path.to_s, :params => params)
+#       if lookup
+#         return lookup.data
+#       else
+#         response = request(:get, path, params)
+#         if response[:status] && response[:status] == 200
+#           lookup = TwitterAPICall.new(:url => path.to_s, :params => params, :data => response[:body])
+#           lookup.save!
+#         end
+#         return response
+#       end
+#     end
+#     
+#     def post(path, params={})
+#       lookup = TwitterAPICall.first(:url => path.to_s, :params => params)
+#       if lookup
+#         return lookup.data
+#       else
+#         signature_params = params.values.any?{|value| value.respond_to?(:to_io)} ? {} : params
+#         response = request(:post, path, params, signature_params)
+#         if response[:status] && response[:status] == 200
+#           lookup = TwitterAPICall.new(:url => path.to_s, :params => params, :data => response[:body])
+#           lookup.save!
+#         end
+#         return response
+#       end
+#     end
+# 
+#     def put(path, params={})
+#       lookup = TwitterAPICall.first(:url => path.to_s, :params => params)
+#       if lookup
+#         return lookup.data
+#       else
+#         response = request(:put, path, params)
+#         if response[:status] && response[:status] == 200
+#           lookup = TwitterAPICall.new(:url => path.to_s, :params => params, :data => response[:body])
+#           lookup.save!
+#         end
+#         return response
+#       end
+#     end
+#   end
+# end

data/lib/oii_twitter_goodies/lib/followers_for_users.rb

@@ -32,7 +32,19 @@ class FollowersForUsers
     puts "Done!"
     return results
   end
-    
+  
+  def grab_followers_iterative(&block)
+    @screen_names.each do |screen_name|
+      puts "Current progress: #{@screen_names.index(screen_name)}/#{@screen_names.length} started"
+      user = user_data_for(screen_name).first
+      follower_ids = follower_ids_for(screen_name)
+      follower_ids.each_slice(100) do |follower_id_slice|
+        block.call([user, user_data_for_set(follower_id_slice)])
+      end
+      puts "Current progress: ##{@screen_names.index(screen_name)}/#{@screen_names.length} complete"      
+    end
+  end
+  
   def follower_ids_for(screen_name)
     return direction_ids_for screen_name, "follower"
   end
@@ -46,19 +58,68 @@ class FollowersForUsers
     ids = []
     while cursor != 0
       puts cursor
-      data = Hashie::Mash[@client.send(direction+"_ids", screen_name, :cursor => cursor).attrs]
-      ids = ids|data.ids
+      begin
+        data = Hashie::Mash[@client.send(direction+"_ids", screen_name, :cursor => cursor).attrs]
+      rescue Twitter::Error::BadGateway
+        puts "Got Twitter::Error::BadGateway error - usually this is just a missed connect from Twitter."
+        sleep(10)
+        retry
+      rescue Twitter::Error::TooManyRequests
+        puts "Got Twitter::Error::TooManyRequests error - we are rate limited and will sleep for 15 minutes. See you in a bit."
+        sleep(15*60)
+        retry
+      rescue Twitter::Error::ClientError
+        puts "Got Twitter::Error::ClientError error - something weird happened and we're going to retry."
+        sleep(5)
+        retry
+      rescue Twitter::Error::InternalServerError
+        puts "Got Twitter::Error::InternalServerError error - something weird happened and we're going to retry."
+        sleep(5)
+        retry
+      rescue Twitter::Error::ServiceUnavailable
+        puts "Got Twitter::Error::ServiceUnavailable error - Fail whaled. We'll hit those guys again in 10"
+        sleep(10)
+        retry
+      end
+      ids << data.ids
       cursor = data["next_cursor"]
     end
-    return ids
+    return ids.flatten.uniq
   end
   
   def user_data_for(screen_names)
     screen_names = [screen_names].flatten
     user_data = []
+    size = screen_names.length/100
     screen_names.each_slice(100) do |screen_name_set|
-      user_data = [user_data|@client.users(screen_name_set)].flatten
+      user_data << user_data_for_set(screen_name_set)
+    end
+    user_data.flatten
+  end
+  
+  def user_data_for_set(screen_name_set)
+    begin
+      return @client.users(screen_name_set)
+    rescue Twitter::Error::BadGateway
+      puts "Got Twitter::Error::BadGateway error - usually this is just a missed connect from Twitter."
+      sleep(10)
+      retry
+    rescue Twitter::Error::TooManyRequests
+      puts "Got Twitter::Error::TooManyRequests error - we are rate limited and will sleep for 15 minutes. See you in a bit."
+      sleep(15*60)
+      retry
+    rescue Twitter::Error::ClientError
+      puts "Got Twitter::Error::ClientError error - something weird happened and we're going to retry."
+      sleep(5)
+      retry
+    rescue Twitter::Error::InternalServerError
+      puts "Got Twitter::Error::InternalServerError error - something weird happened and we're going to retry."
+      sleep(5)
+      retry
+    rescue Twitter::Error::ServiceUnavailable
+      puts "Got Twitter::Error::ServiceUnavailable error - Fail whaled. We'll hit those guys again in 10"
+      sleep(10)
+      retry
     end
-    user_data
   end
 end

data/lib/oii_twitter_goodies/model/bounding_box.rb

@@ -13,7 +13,7 @@ class BoundingBox
     return nil if bounding_box.nil?
     bounding_box = Hashie::Mash[bounding_box]
     obj = self.new
-    obj.type = bounding_box["type"] || "Not Specified"
+    obj.type = bounding_box["type"]
     obj.coordinates = bounding_box["coordinates"]
     obj.place_id = place_id
     obj.save!

data/lib/oii_twitter_goodies/model/place.rb

@@ -23,6 +23,7 @@ class Place
     
   def self.new_from_raw(place, tweet_id)
     return if place.nil?
+    place = place.attrs if place.class == Twitter::Place
     place = Hashie::Mash[place]
     obj = self.new
     obj.twitter_id   = place["twitter_id"]
@@ -32,7 +33,8 @@ class Place
     obj.full_name    = place["full_name"]
     obj.country_code = place["country_code"]
     obj.country      = place["country"]
-    bounding_box = BoundingBox.new_from_raw(place["bounding_box"].attrs, obj._id)
+    place["bounding_box"] = place["bounding_box"].attrs if Twitter::Place
+    bounding_box = BoundingBox.new_from_raw(place["bounding_box"], obj._id)
     if bounding_box
       obj.bounding_box = bounding_box
     end

data/lib/oii_twitter_goodies/model/tweet.rb

@@ -48,6 +48,7 @@ class Tweet
 
   def self.new_from_raw(tweet, user_id)
     return if tweet.nil?
+    tweet = tweet.attrs if tweet.class == Twitter::Tweet
     tweet = Hashie::Mash[tweet]
     obj = self.new
     obj.created_at                = tweet["created_at"]

data/lib/oii_twitter_goodies/model/user.rb

@@ -47,6 +47,7 @@ class User
   end
   
   def self.new_from_raw(status)
+    status = status.attrs if status.class == Twitter::User
     status = Hashie::Mash[status]
     obj = self.new
     obj.twitter_id                         = status["user"].nil? ? status["id"]                                 : status["user"]["id"]

data/lib/oii_twitter_goodies/version.rb

@@ -1,3 +1,3 @@
 module OIITwitterGoodies
-  VERSION = "0.0.74"
+  VERSION = "0.0.81"
 end

data/oii_twitter_goodies.gemspec

@@ -22,17 +22,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "hashie"
   gem.add_development_dependency "typhoeus"
   gem.add_development_dependency "json"
-  gem.add_dependency "bundler", ">= 1.0.0"
-  gem.add_dependency "mongo_mapper"
-  gem.add_dependency "bcrypt-ruby"
-  gem.add_dependency "bson_ext"
-  gem.add_dependency "oauth"
-  gem.add_dependency "twitter"
-  gem.add_dependency "tweetstream"
-  gem.add_dependency "minitest"
-  gem.add_dependency "hashie"
-  gem.add_dependency "typhoeus"
-  gem.add_dependency "json"
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oii_twitter_goodies
 version: !ruby/object:Gem::Version
-  version: 0.0.74
+  version: 0.0.81
 platform: ruby
 authors:
 - Devin Gaffney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-14 00:00:00.000000000 Z
+date: 2013-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,160 +164,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: mongo_mapper
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: bcrypt-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: bson_ext
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: oauth
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: twitter
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: tweetstream
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: hashie
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: typhoeus
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: OII Twitter Goodies!
 email:
 - itsme@devingaffney.com
@@ -325,9 +171,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .DS_Store
-- .gitignore
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -357,7 +202,12 @@ files:
 - lib/oii_twitter_goodies/model/user.rb
 - lib/oii_twitter_goodies/model/user_mention.rb
 - lib/oii_twitter_goodies/version.rb
+- oii_twitter_goodies-0.0.8.gem
 - oii_twitter_goodies.gemspec
+- pkg/oii_twitter_goodies-0.0.1.gem
+- pkg/oii_twitter_goodies-0.0.2.gem
+- pkg/oii_twitter_goodies-0.0.8.gem
+- pkg/oii_twitter_goodies-0.0.81.gem
 homepage: http://github.com/DGaffney/oii_twitter_goodies
 licenses: []
 metadata: {}

data/.gitignore

@@ -1,17 +0,0 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp