twitter_ebooks 2.3.2 → 3.0.0

data/spec/bot_spec.rb

@@ -0,0 +1,178 @@
+require 'spec_helper'
+require 'memory_profiler'
+require 'tempfile'
+require 'timecop'
+
+class TestBot < Ebooks::Bot
+  attr_accessor :twitter
+
+  def configure
+  end
+
+  def on_direct_message(dm)
+    reply dm, "echo: #{dm.text}"
+  end
+
+  def on_mention(tweet)
+    reply tweet, "echo: #{meta(tweet).mentionless}"
+  end
+
+  def on_timeline(tweet)
+    reply tweet, "fine tweet good sir"
+  end
+end
+
+module Ebooks::Test
+  # Generates a random twitter id
+  def twitter_id
+    (rand*10**18).to_i
+  end
+
+  # Creates a mock direct message
+  # @param username User sending the DM
+  # @param text DM content
+  def mock_dm(username, text)
+    Twitter::DirectMessage.new(id: twitter_id,
+                               sender: { id: twitter_id, screen_name: username},
+                               text: text)
+  end
+
+  # Creates a mock tweet
+  # @param username User sending the tweet
+  # @param text Tweet content
+  def mock_tweet(username, text, extra={})
+    mentions = text.split.find_all { |x| x.start_with?('@') }
+    tweet = Twitter::Tweet.new({
+      id: twitter_id,
+      in_reply_to_status_id: 'mock-link',
+      user: { id: twitter_id, screen_name: username },
+      text: text,
+      created_at: Time.now.to_s,
+      entities: {
+        user_mentions: mentions.map { |m|
+          { screen_name: m.split('@')[1],
+            indices: [text.index(m), text.index(m)+m.length] }
+        }
+      }
+    }.merge!(extra))
+    tweet
+  end
+
+  def twitter_spy(bot)
+    twitter = spy("twitter")
+    allow(twitter).to receive(:update).and_return(mock_tweet(bot.username, "test tweet"))
+    twitter
+  end
+
+  def simulate(bot, &b)
+    bot.twitter = twitter_spy(bot)
+    b.call
+  end
+
+  def expect_direct_message(bot, content)
+    expect(bot.twitter).to have_received(:create_direct_message).with(anything(), content, {})
+    bot.twitter = twitter_spy(bot)
+  end
+
+  def expect_tweet(bot, content)
+    expect(bot.twitter).to have_received(:update).with(content, anything())
+    bot.twitter = twitter_spy(bot)
+  end
+end
+
+
+describe Ebooks::Bot do
+  include Ebooks::Test
+  let(:bot) { TestBot.new('test_ebooks') }
+
+  before { Timecop.freeze }
+  after { Timecop.return }
+
+  it "responds to dms" do
+    simulate(bot) do
+      bot.receive_event(mock_dm("m1sp", "this is a dm"))
+      expect_direct_message(bot, "echo: this is a dm")
+    end
+  end
+
+  it "responds to mentions" do
+    simulate(bot) do
+      bot.receive_event(mock_tweet("m1sp", "@test_ebooks this is a mention"))
+      expect_tweet(bot, "@m1sp echo: this is a mention")
+    end
+  end
+
+  it "responds to timeline tweets" do
+    simulate(bot) do
+      bot.receive_event(mock_tweet("m1sp", "some excellent tweet"))
+      expect_tweet(bot, "@m1sp fine tweet good sir")
+    end
+  end
+
+  it "links tweets to conversations correctly" do
+    tweet1 = mock_tweet("m1sp", "tweet 1", id: 1, in_reply_to_status_id: nil)
+
+    tweet2 = mock_tweet("m1sp", "tweet 2", id: 2, in_reply_to_status_id: 1)
+
+    tweet3 = mock_tweet("m1sp", "tweet 3", id: 3, in_reply_to_status_id: nil)
+
+    bot.conversation(tweet1).add(tweet1)
+    expect(bot.conversation(tweet2)).to eq(bot.conversation(tweet1))
+
+    bot.conversation(tweet2).add(tweet2)
+    expect(bot.conversation(tweet3)).to_not eq(bot.conversation(tweet2))
+  end
+
+  it "stops mentioning people after a certain limit" do
+    simulate(bot) do
+      bot.receive_event(mock_tweet("spammer", "@test_ebooks @m1sp 1"))
+      expect_tweet(bot, "@spammer @m1sp echo: 1")
+
+      Timecop.travel(Time.now + 60)
+      bot.receive_event(mock_tweet("spammer", "@test_ebooks @m1sp 2"))
+      expect_tweet(bot, "@spammer @m1sp echo: 2")
+
+      Timecop.travel(Time.now + 60)
+      bot.receive_event(mock_tweet("spammer", "@test_ebooks @m1sp 3"))
+      expect_tweet(bot, "@spammer echo: 3")
+    end
+  end
+
+  it "doesn't stop mentioning them if they reply" do
+    simulate(bot) do
+      bot.receive_event(mock_tweet("spammer", "@test_ebooks @m1sp 4"))
+      expect_tweet(bot, "@spammer @m1sp echo: 4")
+
+      Timecop.travel(Time.now + 60)
+      bot.receive_event(mock_tweet("m1sp", "@spammer @test_ebooks 5"))
+      expect_tweet(bot, "@m1sp @spammer echo: 5")
+
+      Timecop.travel(Time.now + 60)
+      bot.receive_event(mock_tweet("spammer", "@test_ebooks @m1sp 6"))
+      expect_tweet(bot, "@spammer @m1sp echo: 6")
+    end
+  end
+
+  it "doesn't get into infinite bot conversations" do
+    simulate(bot) do
+      bot.receive_event(mock_tweet("spammer", "@test_ebooks @m1sp 7"))
+      expect_tweet(bot, "@spammer @m1sp echo: 7")
+
+      Timecop.travel(Time.now + 10)
+      bot.receive_event(mock_tweet("spammer", "@test_ebooks @m1sp 8"))
+      expect_tweet(bot, "@spammer @m1sp echo: 8")
+
+      Timecop.travel(Time.now + 10)
+      bot.receive_event(mock_tweet("spammer", "@test_ebooks @m1sp 9"))
+      expect(bot.twitter).to_not have_received(:update)
+    end
+  end
+
+  it "blocks blacklisted users on contact" do
+    simulate(bot) do
+      bot.blacklist = ["spammer"]
+      bot.receive_event(mock_tweet("spammer", "@test_ebooks @m1sp 7"))
+      expect(bot.twitter).to have_received(:block).with("spammer")
+    end
+  end
+end

data/spec/model_spec.rb

@@ -22,12 +22,28 @@ describe Ebooks::Model do
     end
   end
 
-  it "does not use a ridiculous amount of memory" do
+  it "consumes, saves and loads models correctly" do
+    model = nil
+
     report = MemoryUsage.report do
       model = Ebooks::Model.consume(path("data/0xabad1dea.json"))
     end
+    expect(report.total_memsize).to be < 200000000
+
+    file = Tempfile.new("0xabad1dea")
+    model.save(file.path)
+
+    report2 = MemoryUsage.report do
+      model = Ebooks::Model.load(file.path)
+    end
+    expect(report2.total_memsize).to be < 3000000
+
+    expect(model.tokens[0]).to be_a String
+    expect(model.sentences[0][0]).to be_a Fixnum
+    expect(model.mentions[0][0]).to be_a Fixnum
+    expect(model.keywords[0]).to be_a String
 
-    expect(report.total_memsize).to be < 1000000000
+    puts "0xabad1dea.model uses #{report2.total_memsize} bytes in memory"
   end
 
   describe '.consume' do

data/twitter_ebooks.gemspec

@@ -16,16 +16,20 @@ Gem::Specification.new do |gem|
   gem.version       = Ebooks::VERSION
 
   gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rspec-mocks'
   gem.add_development_dependency 'memory_profiler'
+  gem.add_development_dependency 'timecop'
   gem.add_development_dependency 'pry-byebug'
+  gem.add_development_dependency 'yard'
 
-  gem.add_runtime_dependency 'twitter', '~> 4.0'
-  gem.add_runtime_dependency 'simple_oauth', '~> 0.2.0'
-  gem.add_runtime_dependency 'tweetstream'
+  gem.add_runtime_dependency 'twitter', '~> 5.0'
+  gem.add_runtime_dependency 'simple_oauth'
+  gem.add_runtime_dependency 'eventmachine', '~> 1.0.3'
   gem.add_runtime_dependency 'rufus-scheduler'
   gem.add_runtime_dependency 'gingerice'
   gem.add_runtime_dependency 'htmlentities'
   gem.add_runtime_dependency 'engtagger'
   gem.add_runtime_dependency 'fast-stemmer'
   gem.add_runtime_dependency 'highscore'
+  gem.add_runtime_dependency 'pry'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitter_ebooks
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 3.0.0
 platform: ruby
 authors:
 - Jaiden Mispy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-25 00:00:00.000000000 Z
+date: 2014-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: memory_profiler
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -53,35 +81,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: twitter
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: simple_oauth
+  name: twitter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: tweetstream
+  name: simple_oauth
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -94,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.3
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +220,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Markov chains for all your friends~
 email:
 - "^_^@mispy.me"
@@ -188,6 +244,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -199,7 +256,6 @@ files:
 - lib/twitter_ebooks.rb
 - lib/twitter_ebooks/archive.rb
 - lib/twitter_ebooks/bot.rb
-- lib/twitter_ebooks/markov.rb
 - lib/twitter_ebooks/model.rb
 - lib/twitter_ebooks/nlp.rb
 - lib/twitter_ebooks/suffix.rb
@@ -210,15 +266,12 @@ files:
 - skeleton/bots.rb
 - skeleton/corpus/.gitignore
 - skeleton/model/.gitignore
-- skeleton/run.rb
+- spec/bot_spec.rb
 - spec/data/0xabad1dea.json
 - spec/data/0xabad1dea.model
 - spec/memprof.rb
 - spec/model_spec.rb
 - spec/spec_helper.rb
-- test/corpus/0xabad1dea.tweets
-- test/keywords.rb
-- test/tokenize.rb
 - twitter_ebooks.gemspec
 homepage: ''
 licenses: []
@@ -244,11 +297,10 @@ signing_key:
 specification_version: 4
 summary: Markov chains for all your friends~
 test_files:
+- spec/bot_spec.rb
 - spec/data/0xabad1dea.json
 - spec/data/0xabad1dea.model
 - spec/memprof.rb
 - spec/model_spec.rb
 - spec/spec_helper.rb
-- test/corpus/0xabad1dea.tweets
-- test/keywords.rb
-- test/tokenize.rb
+has_rdoc: 

data/lib/twitter_ebooks/markov.rb

@@ -1,82 +0,0 @@
-module Ebooks
-  # Special INTERIM token represents sentence boundaries
-  # This is so we can include start and end of statements in model
-  # Due to the way the sentence tokenizer works, can correspond
-  # to multiple actual parts of text (such as ^, $, \n and .?!)
-  INTERIM = :interim
-
-  # This is an ngram-based Markov model optimized to build from a
-  # tokenized sentence list without requiring too much transformation
-  class MarkovModel
-    def self.build(sentences)
-      MarkovModel.new.consume(sentences)
-    end
-
-    def consume(sentences)
-      # These models are of the form ngram => [[sentence_pos, token_pos] || INTERIM, ...]
-      # We map by both bigrams and unigrams so we can fall back to the latter in
-      # cases where an input bigram is unavailable, such as starting a sentence
-      @sentences = sentences
-      @unigrams = {}
-      @bigrams = {}
-
-      sentences.each_with_index do |tokens, i|
-        last_token = INTERIM
-        tokens.each_with_index do |token, j|
-          @unigrams[last_token] ||= []
-          @unigrams[last_token] << [i, j]
-
-          @bigrams[last_token] ||= {}
-          @bigrams[last_token][token] ||= []
-
-          if j == tokens.length-1 # Mark sentence endings
-            @unigrams[token] ||= []
-            @unigrams[token] << INTERIM
-            @bigrams[last_token][token] << INTERIM
-          else
-            @bigrams[last_token][token] << [i, j+1]
-          end
-
-          last_token = token
-        end
-      end
-
-      self
-    end
-
-    def find_token(index)
-      if index == INTERIM
-        INTERIM
-      else
-        @sentences[index[0]][index[1]]
-      end
-    end
-
-    def chain(tokens)
-      if tokens.length == 1
-        matches = @unigrams[tokens[-1]]
-      else
-        matches = @bigrams[tokens[-2]][tokens[-1]]
-        matches = @unigrams[tokens[-1]] if matches.length < 2
-      end
-
-      if matches.empty?
-        # This should never happen unless a strange token is
-        # supplied from outside the dataset
-        raise ArgumentError, "Unable to continue chain for: #{tokens.inspect}"
-      end
-
-      next_token = find_token(matches.sample)
-
-      if next_token == INTERIM # We chose to end the sentence
-        return tokens
-      else
-        return chain(tokens + [next_token])
-      end
-    end
-
-    def generate
-      NLP.reconstruct(chain([INTERIM]))
-    end
-  end
-end