neo4apis-twitter 0.7.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7c8be4b5bfaae76a81157797e34dd4565a939ec2
-  data.tar.gz: 10542478383cdcf161003cc746ec4e20414d5f2d
+  metadata.gz: 4af92daa4a378b1ced8de1deb45ca1a9b551e637
+  data.tar.gz: dd5db05d098883b4699674761ad2801232ecbc2c
 SHA512:
-  metadata.gz: 503ba611328dc21c6fcfb03e84c1efb100899a356a83dcb44303003fd7cfc8857a859806c6ae48a0119965d99147328687f981c43a4e4e8893675dad5ac93b65
-  data.tar.gz: cdf50c105718208e443888792dae59c50099f27f6b00f4ac11c07e9275f6673af63017198db9b826d07dac70e38cf44671488df2835c8d4a996e934f090536cb
+  metadata.gz: ecc8240fed4ca7038db29ebb7bda7e1848aa0a6e5b7ca365604c839bd56ff0b33fefa5d0ae491454938c8625c193999d843ef24234bd402f8ab57be60e930a21
+  data.tar.gz: 6004713148e4a3848b3f45c7f006648186a5872e7e4c2048869b7d0d490530e0d29d3cbe72dc152ffa2147258ca854f0817d953534660e10f92b4ab9d57ed6aa

data/config/twitter.yml

@@ -0,0 +1,7 @@
+---
+:consumer_key: iwmENsqBZNAeN07hsrGuGwKMa
+:consumer_secret: ioVqcH3pxTrTq5or5cpXdj4WJpUO1ZNjMRr3tmhFg4xT1mOQXa
+:access_token: 2776665793-QPDyINltYv3hGa9RjmMeAbxFSHXJ764n0ycOu1E
+:access_token_secret: hlxs1wONx6lJ7GLLY0nXm630mv6NzPHOhDlirCvqetCip
+
+

data/lib/neo4apis/cli/twitter.rb

@@ -4,7 +4,7 @@ require 'colorize'
 
 module Neo4Apis
   module CLI
-    class Twitter < Thor
+    class Twitter < Base
       class_option :config_path, type: :string,  default: 'config/twitter.yml'
 
       class_option :import_retweets, type: :boolean, default: false
@@ -27,18 +27,93 @@ module Neo4Apis
       desc "search QUERY COUNT", "Import tweets via a search query"
       def search(query, count)
         neo4apis_client.batch do
-          twitter_client(false).search(query, options).take(count.to_i).each(&tweet_importer)
+          twitter_client.search(query, options).take(count.to_i).each(&tweet_importer)
+        end
+      end
+
+      desc "followers USERNAME DEPTH", "Imports followers (and optionally followers of followers if depth specified and with a value > 1"
+      def followers(user, depth = 1)
+        neo4apis_client.batch do
+          follower_importer(neo4apis_client, user, depth.to_i)
+        end
+      end
+
+      desc "friends USERNAME DEPTH", "Imports friends (and optionally friends of friends if depth specified and with a value > 1"
+      def friends(user, depth = 1)
+        neo4apis_client.batch do
+          friend_importer(neo4apis_client, user, depth.to_i)
         end
       end
 
       private
 
+      def follower_importer(neo4apis_client, user, depth)
+        friend_follower_importer(:followers, neo4apis_client, user, depth)
+      end
+
+      def friend_importer(neo4apis_client, user, depth)
+        friend_follower_importer(:friends, neo4apis_client, user, depth)
+      end
+
+      def friend_follower_importer(type, neo4apis_client, user, depth)
+        twitter_user = if user == ::Twitter::User
+                         user
+                       else
+                         throttled_block { twitter_client.user(user) }
+                       end
+        user_node = user_importer.call(twitter_user)
+
+        relationship = (type == :friends ? :friended : :follows)
+        get_friends_or_followers(type, twitter_user).each do |other_user|
+          other_user_node = user_importer.call(other_user)
+
+          neo4apis_client.add_relationship(relationship, other_user_node, user_node)
+
+          if depth > 1
+            friend_follower_importer(type, neo4apis_client, other_user, depth - 1)
+          end
+        end
+      end
+
+      def get_friends_or_followers(type, user)
+        [].tap do |user_list|
+          next_cursor = -1
+          while next_cursor != 0
+            result = throttled_block { twitter_client.send(type, user, cursor: next_cursor, count: 5000) }
+            throttled_block { user_list.concat(result.take(5000)) }
+            next_cursor = result.send(:next_cursor)
+          end
+        end
+      end
+
+      def throttled_block
+        yield
+      rescue ::Twitter::Error::TooManyRequests => error
+        reset_in = error.rate_limit.reset_in
+        puts "Rate limit exceeded.  Sleeping for #{reset_in} seconds"
+        sleep reset_in
+        retry
+      end
+
       def tweet_importer
+        @tweet_importer ||= object_importer(::Twitter::Tweet, :Tweet)
+      end
+
+      def user_importer
+        @user_importer ||= object_importer(::Twitter::User, :User)
+      end
+
+      def object_importer(gem_class, label)
         Proc.new do |object|
           case object
-          when ::Twitter::Tweet
-            say "got tweet from @#{object.user.screen_name.colorize(:light_blue)}: #{object.text}"
-            neo4apis_client.import :Tweet, object
+          when gem_class
+            case label
+            when :Tweet
+              say "got tweet from @#{object.user.screen_name.colorize(:light_blue)}: #{object.text}"
+            when :User
+              say "get user ##{object.id}"
+            end
+            neo4apis_client.import label, object
           when ::Twitter::Streaming::StallWarning
             say "FALLING BEHIND!".colorize(:red)
           end
@@ -48,10 +123,10 @@ module Neo4Apis
       NEO4APIS_CLIENT_CLASS = ::Neo4Apis::Twitter
 
       def neo4apis_client
-        @neo4apis_client ||= NEO4APIS_CLIENT_CLASS.new(Neo4j::Session.open(:server_db, parent_options[:neo4j_url]), import_retweets: options[:import_retweets], import_hashtags: options[:import_hashtags], import_user_mentions: options[:import_user_mentions])
+        @neo4apis_client ||= NEO4APIS_CLIENT_CLASS.new(neo4j_session, import_retweets: options[:import_retweets], import_hashtags: options[:import_hashtags], import_user_mentions: options[:import_user_mentions])
       end
 
-      def twitter_client(streaming)
+      def twitter_client(streaming = false)
         return @twitter_client if @twitter_client
 
         twitter_client_class = streaming ? ::Twitter::Streaming::Client : ::Twitter::REST::Client

data/lib/neo4apis/twitter.rb

@@ -3,7 +3,7 @@ require 'ostruct'
 
 module Neo4Apis
   class Twitter < Base
-    prefix :Twitter
+    common_label :Twitter
 
     uuid :Tweet, :id
     uuid :User, :id

data/neo4apis-twitter.gemspec

@@ -3,7 +3,7 @@ $:.unshift lib unless $:.include?(lib)
 
 Gem::Specification.new do |s|
   s.name     = "neo4apis-twitter"
-  s.version  = '0.7.0'
+  s.version  = '0.8.1'
   s.required_ruby_version = ">= 1.9.1"
 
   s.authors  = "Brian Underwood"
@@ -18,7 +18,7 @@ A ruby gem using neo4apis to make importing twitter data to neo4j easy
   s.require_path = 'lib'
   s.files = Dir.glob("{bin,lib,config}/**/*") + %w(README.md Gemfile neo4apis-twitter.gemspec)
 
-  s.add_dependency('neo4apis', '~> 0.4.0')
+  s.add_dependency('neo4apis', '>= 0.8.0')
   s.add_dependency('twitter', '~> 5.12.0')
 
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: neo4apis-twitter
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Brian Underwood
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-09 00:00:00.000000000 Z
+date: 2015-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: neo4apis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: twitter
   requirement: !ruby/object:Gem::Requirement
@@ -47,6 +47,7 @@ extra_rdoc_files: []
 files:
 - Gemfile
 - README.md
+- config/twitter.yml
 - lib/neo4apis-twitter.rb
 - lib/neo4apis/cli/twitter.rb
 - lib/neo4apis/twitter.rb
@@ -71,7 +72,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: An ruby gem to import twitter data to neo4j