twitter_ebooks 2.3.2 → 3.0.0

data/lib/twitter_ebooks.rb

@@ -1,7 +1,7 @@
 $debug = false
 
 def log(*args)
-  STDERR.puts args.map(&:to_s).join(' ')
+  STDERR.print args.map(&:to_s).join(' ') + "\n"
   STDERR.flush
 end
 
@@ -11,11 +11,11 @@ module Ebooks
   SKELETON_PATH = File.join(GEM_PATH, 'skeleton')
   TEST_PATH = File.join(GEM_PATH, 'test')
   TEST_CORPUS_PATH = File.join(TEST_PATH, 'corpus/0xabad1dea.tweets')
+  INTERIM = :interim
 end
 
 require 'twitter_ebooks/nlp'
 require 'twitter_ebooks/archive'
-require 'twitter_ebooks/markov'
 require 'twitter_ebooks/suffix'
 require 'twitter_ebooks/model'
 require 'twitter_ebooks/bot'

data/lib/twitter_ebooks/archive.rb

@@ -21,9 +21,9 @@ module Ebooks
         @config[:consumer_key] = STDIN.gets.chomp
         print "Consumer secret: "
         @config[:consumer_secret] = STDIN.gets.chomp
-        print "Oauth token: "
+        print "Access token: "
         @config[:oauth_token] = STDIN.gets.chomp
-        print "Oauth secret: "
+        print "Access secret: "
         @config[:oauth_token_secret] = STDIN.gets.chomp
 
         File.open(CONFIG_PATH, 'w') do |f|
@@ -31,19 +31,22 @@ module Ebooks
         end
       end
 
-      Twitter.configure do |config|
+      Twitter::REST::Client.new do |config|
         config.consumer_key = @config[:consumer_key]
         config.consumer_secret = @config[:consumer_secret]
-        config.oauth_token = @config[:oauth_token]
-        config.oauth_token_secret = @config[:oauth_token_secret]
+        config.access_token = @config[:oauth_token]
+        config.access_token_secret = @config[:oauth_token_secret]
       end
-
-      Twitter::Client.new
     end
 
-    def initialize(username, path, client=nil)
+    def initialize(username, path=nil, client=nil)
       @username = username
-      @path = path || "#{username}.json"
+      @path = path || "corpus/#{username}.json"
+
+      if File.directory?(@path)
+        @path = File.join(@path, "#{username}.json")
+      end
+
       @client = client || make_client
 
       if File.exists?(@path)

data/lib/twitter_ebooks/bot.rb

@@ -1,180 +1,414 @@
-#!/usr/bin/env ruby
 # encoding: utf-8
 require 'twitter'
-require 'tweetstream'
 require 'rufus/scheduler'
 
 module Ebooks
-  class Bot
-    attr_accessor :consumer_key, :consumer_secret,
-                  :oauth_token, :oauth_token_secret
+  class ConfigurationError < Exception
+  end
 
-    attr_accessor :username
+  # Represents a single reply tree of tweets
+  class Conversation
+    attr_reader :last_update
 
-    attr_reader :twitter, :stream
+    # @param bot [Ebooks::Bot]
+    def initialize(bot)
+      @bot = bot
+      @tweets = []
+      @last_update = Time.now
+    end
 
-    @@all = [] # List of all defined bots
-    def self.all; @@all; end
+    # @param tweet [Twitter::Tweet] tweet to add
+    def add(tweet)
+      @tweets << tweet
+      @last_update = Time.now
+    end
+
+    # Make an informed guess as to whether a user is a bot based
+    # on their behavior in this conversation
+    def is_bot?(username)
+      usertweets = @tweets.select { |t| t.user.screen_name == username }
 
-    def self.get(name)
-      all.find { |bot| bot.username == name }
+      if usertweets.length > 2
+        if (usertweets[-1].created_at - usertweets[-3].created_at) < 30
+          return true
+        end
+      end
+
+      username.include?("ebooks")
     end
 
-    def initialize(username, &b)
-      # Set defaults
-      @username = username
+    # Figure out whether to keep this user in the reply prefix
+    # We want to avoid spamming non-participating users
+    def can_include?(username)
+      @tweets.length <= 4 ||
+        !@tweets[-4..-1].select { |t| t.user.screen_name == username }.empty?
+    end
+  end
 
-      # Override with callback
-      b.call(self)
+  # Meta information about a tweet that we calculate for ourselves
+  class TweetMeta
+    # @return [Array<String>] usernames mentioned in tweet
+    attr_accessor :mentions
+    # @return [String] text of tweets with mentions removed
+    attr_accessor :mentionless
+    # @return [Array<String>] usernames to include in a reply
+    attr_accessor :reply_mentions
+    # @return [String] mentions to start reply with
+    attr_accessor :reply_prefix
+    # @return [Integer] available chars for reply
+    attr_accessor :limit
+
+    # @return [Ebooks::Bot] associated bot
+    attr_accessor :bot
+    # @return [Twitter::Tweet] associated tweet
+    attr_accessor :tweet
+
+    # Check whether this tweet mentions our bot
+    # @return [Boolean]
+    def mentions_bot?
+      # To check if this is someone talking to us, ensure:
+      # - The tweet mentions list contains our username
+      # - The tweet is not being retweeted by somebody else
+      # - Or soft-retweeted by somebody else
+      @mentions.map(&:downcase).include?(@bot.username.downcase) && !@tweet.retweeted_status? && !@tweet.text.start_with?('RT ')
+    end
+
+    # @param bot [Ebooks::Bot]
+    # @param ev [Twitter::Tweet]
+    def initialize(bot, ev)
+      @bot = bot
+      @tweet = ev
+
+      @mentions = ev.attrs[:entities][:user_mentions].map { |x| x[:screen_name] }
+
+      # Process mentions to figure out who to reply to
+      # i.e. not self and nobody who has seen too many secondary mentions
+      reply_mentions = @mentions.reject do |m|
+        username = m.downcase
+        username == @bot.username || !@bot.conversation(ev).can_include?(username)
+      end
+      @reply_mentions = ([ev.user.screen_name] + reply_mentions).uniq
+
+      @reply_prefix = @reply_mentions.map { |m| '@'+m }.join(' ') + ' '
+      @limit = 140 - @reply_prefix.length
 
-      Bot.all.push(self)
+      mless = ev.text
+      begin
+        ev.attrs[:entities][:user_mentions].reverse.each do |entity|
+          last = mless[entity[:indices][1]..-1]||''
+          mless = mless[0...entity[:indices][0]] + last.strip
+        end
+      rescue Exception
+        p ev.attrs[:entities][:user_mentions]
+        p ev.text
+        raise
+      end
+      @mentionless = mless
     end
+  end
 
+  class Bot
+    # @return [String] OAuth consumer key for a Twitter app
+    attr_accessor :consumer_key
+    # @return [String] OAuth consumer secret for a Twitter app
+    attr_accessor :consumer_secret
+    # @return [String] OAuth access token from `ebooks auth`
+    attr_accessor :access_token
+    # @return [String] OAuth access secret from `ebooks auth`
+    attr_accessor :access_token_secret
+    # @return [String] Twitter username of bot
+    attr_accessor :username
+    # @return [Array<String>] list of usernames to block on contact
+    attr_accessor :blacklist
+    # @return [Hash{String => Ebooks::Conversation}] maps tweet ids to their conversation contexts
+    attr_accessor :conversations
+    # @return [Range, Integer] range of seconds to delay in delay method
+    attr_accessor :delay_range
+
+    # @return [Array] list of all defined bots
+    def self.all; @@all ||= []; end
+
+    # Fetches a bot by username
+    # @param username [String]
+    # @return [Ebooks::Bot]
+    def self.get(username)
+      all.find { |bot| bot.username == username }
+    end
+
+    # Logs info to stdout in the context of this bot
     def log(*args)
-      STDOUT.puts "@#{@username}: " + args.map(&:to_s).join(' ')
+      STDOUT.print "@#{@username}: " + args.map(&:to_s).join(' ') + "\n"
       STDOUT.flush
     end
 
-    def configure
-      TweetStream.configure do |config|
+    # Initializes and configures bot
+    # @param args Arguments passed to configure method
+    # @param b Block to call with new bot
+    def initialize(username, &b)
+      @blacklist ||= []
+      @conversations ||= {}
+      # Tweet ids we've already observed, to avoid duplication
+      @seen_tweets ||= {}
+
+      @username = username
+      configure
+
+      b.call(self) unless b.nil?
+      Bot.all << self
+    end
+
+    # Find or create the conversation context for this tweet
+    # @param tweet [Twitter::Tweet]
+    # @return [Ebooks::Conversation]
+    def conversation(tweet)
+      conv = if tweet.in_reply_to_status_id?
+        @conversations[tweet.in_reply_to_status_id]
+      end
+
+      if conv.nil?
+        conv = @conversations[tweet.id] || Conversation.new(self)
+      end
+
+      if tweet.in_reply_to_status_id?
+        @conversations[tweet.in_reply_to_status_id] = conv
+      end
+      @conversations[tweet.id] = conv
+
+      # Expire any old conversations to prevent memory growth
+      @conversations.each do |k,v|
+        if v != conv && Time.now - v.last_update > 3600
+          @conversations.delete(k)
+        end
+      end
+
+      conv
+    end
+
+    # @return [Twitter::REST::Client] underlying REST client from twitter gem
+    def twitter
+      @twitter ||= Twitter::REST::Client.new do |config|
         config.consumer_key = @consumer_key
         config.consumer_secret = @consumer_secret
-        config.oauth_token = @oauth_token
-        config.oauth_token_secret = @oauth_token_secret
+        config.access_token = @access_token
+        config.access_token_secret = @access_token_secret
       end
+    end
 
-      Twitter.configure do |config|
+    # @return [Twitter::Streaming::Client] underlying streaming client from twitter gem
+    def stream
+      @stream ||= Twitter::Streaming::Client.new do |config|
         config.consumer_key = @consumer_key
         config.consumer_secret = @consumer_secret
-        config.oauth_token = @oauth_token
-        config.oauth_token_secret = @oauth_token_secret
+        config.access_token = @access_token
+        config.access_token_secret = @access_token_secret
+      end
+    end
+
+    # Calculate some meta information about a tweet relevant for replying
+    # @param ev [Twitter::Tweet]
+    # @return [Ebooks::TweetMeta]
+    def meta(ev)
+      TweetMeta.new(self, ev)
+    end
+
+    # Receive an event from the twitter stream
+    # @param ev [Object] Twitter streaming event
+    def receive_event(ev)
+      if ev.is_a? Array # Initial array sent on first connection
+        log "Online!"
+        return
       end
 
-      @twitter = Twitter::Client.new
+      if ev.is_a? Twitter::DirectMessage
+        return if ev.sender.screen_name == @username # Don't reply to self
+        log "DM from @#{ev.sender.screen_name}: #{ev.text}"
+        fire(:direct_message, ev)
 
-      needs_stream = [@on_follow, @on_message, @on_mention, @on_timeline].any? {|e| !e.nil?}
+      elsif ev.respond_to?(:name) && ev.name == :follow
+        return if ev.source.screen_name == @username
+        log "Followed by #{ev.source.screen_name}"
+        fire(:follow, ev.source)
 
-      @stream = TweetStream::Client.new if needs_stream
-    end
+      elsif ev.is_a? Twitter::Tweet
+        return unless ev.text # If it's not a text-containing tweet, ignore it
+        return if ev.user.screen_name == @username # Ignore our own tweets
 
-    # Connects to tweetstream and opens event handlers for this bot
-    def start
-      configure
+        meta = meta(ev)
 
-      @on_startup.call if @on_startup
+        if blacklisted?(ev.user.screen_name)
+          log "Blocking blacklisted user @#{ev.user.screen_name}"
+          @twitter.block(ev.user.screen_name)
+        end
 
-      if not @stream
-        log "not bothering with stream for #@username"
-        return
-      end
+        # Avoid responding to duplicate tweets
+        if @seen_tweets[ev.id]
+          log "Not firing event for duplicate tweet #{ev.id}"
+          return
+        else
+          @seen_tweets[ev.id] = true
+        end
 
-      log "starting stream for #@username"
-      @stream.on_error do |msg|
-        log "ERROR: #{msg}"
-      end
+        if meta.mentions_bot?
+          log "Mention from @#{ev.user.screen_name}: #{ev.text}"
+          conversation(ev).add(ev)
+          fire(:mention, ev)
+        else
+          fire(:timeline, ev)
+        end
 
-      @stream.on_inited do
-        log "Online!"
+      elsif ev.is_a?(Twitter::Streaming::DeletedTweet) ||
+            ev.is_a?(Twitter::Streaming::Event)
+        # pass
+      else
+        log ev
       end
+    end
 
-      @stream.on_event(:follow) do |event|
-        next if event[:source][:screen_name] == @username
-        log "Followed by #{event[:source][:screen_name]}"
-        @on_follow.call(event[:source]) if @on_follow
+    # Configures client and fires startup event
+    def prepare
+      # Sanity check
+      if @username.nil?
+        raise ConfigurationError, "bot username cannot be nil"
       end
 
-      @stream.on_direct_message do |dm|
-        next if dm[:sender][:screen_name] == @username # Don't reply to self
-        log "DM from @#{dm[:sender][:screen_name]}: #{dm[:text]}"
-        @on_message.call(dm) if @on_message
+      if @consumer_key.nil? || @consumer_key.empty? ||
+         @consumer_secret.nil? || @consumer_key.empty?
+        log "Missing consumer_key or consumer_secret. These details can be acquired by registering a Twitter app at https://apps.twitter.com/"
+        exit 1
       end
 
-      @stream.userstream do |ev|
-        next unless ev[:text] # If it's not a text-containing tweet, ignore it
-        next if ev[:user][:screen_name] == @username # Ignore our own tweets
+      if @access_token.nil? || @access_token.empty? ||
+         @access_token_secret.nil? || @access_token_secret.empty?
+        log "Missing access_token or access_token_secret. Please run `ebooks auth`."
+        exit 1
+      end
 
-        meta = {}
-        mentions = ev.attrs[:entities][:user_mentions].map { |x| x[:screen_name] }
+      twitter
+      fire(:startup)
+    end
 
-        reply_mentions = mentions.reject { |m| m.downcase == @username.downcase }
-        reply_mentions = [ev[:user][:screen_name]] + reply_mentions
+    # Start running user event stream
+    def start
+      log "starting tweet stream"
 
-        meta[:reply_prefix] = reply_mentions.uniq.map { |m| '@'+m }.join(' ') + ' '
-        meta[:limit] = 140 - meta[:reply_prefix].length
+      stream.user do |ev|
+        receive_event ev
+      end
+    end
 
-        mless = ev[:text]
-        begin
-          ev.attrs[:entities][:user_mentions].reverse.each do |entity|
-            last = mless[entity[:indices][1]..-1]||''
-            mless = mless[0...entity[:indices][0]] + last.strip
-          end
-        rescue Exception
-          p ev.attrs[:entities][:user_mentions]
-          p ev[:text]
-          raise
-        end
-        meta[:mentionless] = mless
-
-        # To check if this is a mention, ensure:
-        # - The tweet mentions list contains our username
-        # - The tweet is not being retweeted by somebody else
-        # - Or soft-retweeted by somebody else
-        if mentions.map(&:downcase).include?(@username.downcase) && !ev[:retweeted_status] && !ev[:text].start_with?('RT ')
-          log "Mention from @#{ev[:user][:screen_name]}: #{ev[:text]}"
-          @on_mention.call(ev, meta) if @on_mention
-        else
-          @on_timeline.call(ev, meta) if @on_timeline
-        end
+    # Fire an event
+    # @param event [Symbol] event to fire
+    # @param args arguments for event handler
+    def fire(event, *args)
+      handler = "on_#{event}".to_sym
+      if respond_to? handler
+        self.send(handler, *args)
       end
     end
 
-    # Wrapper for EM.add_timer
-    # Delays add a greater sense of humanity to bot behaviour
-    def delay(time, &b)
-      time = time.to_a.sample unless time.is_a? Integer
-      EM.add_timer(time, &b)
+    # Delay an action for a variable period of time
+    # @param range [Range, Integer] range of seconds to choose for delay
+    def delay(range=@delay_range, &b)
+      time = range.to_a.sample unless range.is_a? Integer
+      sleep time
+      b.call
+    end
+
+    # Check if a username is blacklisted
+    # @param username [String]
+    # @return [Boolean]
+    def blacklisted?(username)
+      if @blacklist.include?(username)
+        true
+      else
+        false
+      end
     end
 
     # Reply to a tweet or a DM.
-    # Applies configurable @reply_delay range
+    # @param ev [Twitter::Tweet, Twitter::DirectMessage]
+    # @param text [String] contents of reply excluding reply_prefix
+    # @param opts [Hash] additional params to pass to twitter gem
     def reply(ev, text, opts={})
       opts = opts.clone
 
       if ev.is_a? Twitter::DirectMessage
-        log "Sending DM to @#{ev[:sender][:screen_name]}: #{text}"
-        @twitter.direct_message_create(ev[:sender][:screen_name], text, opts)
+        log "Sending DM to @#{ev.sender.screen_name}: #{text}"
+        twitter.create_direct_message(ev.sender.screen_name, text, opts)
       elsif ev.is_a? Twitter::Tweet
-        log "Replying to @#{ev[:user][:screen_name]} with: #{text}"
-        @twitter.update(text, in_reply_to_status_id: ev[:id])
+        meta = meta(ev)
+
+        if conversation(ev).is_bot?(ev.user.screen_name)
+          log "Not replying to suspected bot @#{ev.user.screen_name}"
+          return false
+        end
+
+        log "Replying to @#{ev.user.screen_name} with: #{meta.reply_prefix + text}"
+        tweet = twitter.update(meta.reply_prefix + text, in_reply_to_status_id: ev.id)
+        conversation(tweet).add(tweet)
+        tweet
       else
         raise Exception("Don't know how to reply to a #{ev.class}")
       end
     end
 
-    def scheduler
-      @scheduler ||= Rufus::Scheduler.new
+    # Favorite a tweet
+    # @param tweet [Twitter::Tweet]
+    def favorite(tweet)
+      log "Favoriting @#{tweet.user.screen_name}: #{tweet.text}"
+
+      begin
+        twitter.favorite(tweet.id)
+      rescue Twitter::Error::Forbidden
+        log "Already favorited: #{tweet.user.screen_name}: #{tweet.text}"
+      end
+    end
+
+    # Retweet a tweet
+    # @param tweet [Twitter::Tweet]
+    def retweet(tweet)
+      log "Retweeting @#{tweet.user.screen_name}: #{tweet.text}"
+
+      begin
+        twitter.retweet(tweet.id)
+      rescue Twitter::Error::Forbidden
+        log "Already retweeted: #{tweet.user.screen_name}: #{tweet.text}"
+      end
+    end
+
+    # Follow a user
+    # @param user [String] username or user id
+    def follow(user, *args)
+      log "Following #{user}"
+      twitter.follow(user, *args)
+    end
+
+    # Unfollow a user
+    # @param user [String] username or user id
+    def unfollow(user, *args)
+      log "Unfollowing #{user}"
+      twiter.unfollow(user, *args)
     end
 
-    def follow(*args)
-      log "Following #{args}"
-      @twitter.follow(*args)
+    # Tweet something
+    # @param text [String]
+    def tweet(text, *args)
+      log "Tweeting '#{text}'"
+      twitter.update(text, *args)
     end
 
-    def tweet(*args)
-      log "Tweeting #{args.inspect}"
-      @twitter.update(*args)
+    # Get a scheduler for this bot
+    # @return [Rufus::Scheduler]
+    def scheduler
+      @scheduler ||= Rufus::Scheduler.new
     end
 
-    # could easily just be *args however the separation keeps it clean.
+    # Tweet some text with an image
+    # @param txt [String]
+    # @param pic [String] filename
     def pictweet(txt, pic, *args)
       log "Tweeting #{txt.inspect} - #{pic} #{args}"
-      @twitter.update_with_media(txt, File.new(pic), *args)
+      twitter.update_with_media(txt, File.new(pic), *args)
     end
-
-    def on_startup(&b); @on_startup = b; end
-    def on_follow(&b); @on_follow = b; end
-    def on_mention(&b); @on_mention = b; end
-    def on_timeline(&b); @on_timeline = b; end
-    def on_message(&b); @on_message = b; end
   end
 end