twitter_ebooks 2.2.9 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50a5dca9c31287964724b38cb022c6273a242a59
-  data.tar.gz: 7c2f8e441bc119f1bc29b8e7ece6650812785fa4
+  metadata.gz: a33310c52cb154361bfa00ccdf9cba3b9850af3b
+  data.tar.gz: c6c29fd59ae7a7926b5e649ca2858eed204f0cdb
 SHA512:
-  metadata.gz: fdd9dee8a8f53bb421761a0a485b0c5bcb2677355e6c4186c69c8e60d9f15dbd13a4c1cd77c09243e609bc53b02cecc3d2c6a2f81e2fadc4da0cc8b527558df3
-  data.tar.gz: 6a0beb91162f03bfd3ddc1edb386940919b066c13ef323d370b3314b9e4cdd3cca4463f22719ce0cecfd5f5f33fe5b5352268d71d99fa2f15f2b8f41749845a7
+  metadata.gz: 807bdfd51ac33fdb4ae25687e74ff89da02ca8004a7cae576d1fa159df2e7801d4f98195983ae8f8f3c5bff0136b4e880ee4bd67b1b219ede0e1aaf350b6e627
+  data.tar.gz: 082ef0b1c815c30d535174c19a7f21c21d9988486f3c64759d908b04d5b7379a077cf54cfdf39bff2c11346fea7654187a80f097c938d0fc26a3128d49eeb51b

data/README.md

@@ -1,4 +1,4 @@
-# twitter\_ebooks 2.2.9
+# twitter\_ebooks 2.3.0
 
 Rewrite of my twitter\_ebooks code. While the original was solely a tweeting Markov generator, this framework helps you build any kind of interactive twitterbot which responds to mentions/DMs. See [ebooks\_example](https://github.com/mispy/ebooks_example) for an example of a full bot.
 

data/lib/twitter_ebooks/model.rb

@@ -18,18 +18,31 @@ module Ebooks
       Marshal.load(File.open(path, 'rb') { |f| f.read })
     end
 
-    def mass_tokenize(text)
+    def initialize
+      # This is the only source of actual strings in the model. It is
+      # an array of unique tokens. Manipulation of a token is mostly done
+      # using its index in this array, which we call a "tiki"
+      @tokens = []
+
+      # Reverse lookup tiki by token, for faster generation
+      @tikis = {}
+    end
+
+    def tikify(token)
+      @tikis[token] or (@tokens << token and @tikis[token] = @tokens.length-1)
+    end
+
+    def mass_tikify(text)
       sentences = NLP.sentences(text)
-      tokens = []
 
-      sentences.each do |s|
-        tokens << NLP.tokenize(s).reject do |t|
+      sentences.map do |s|
+        tokens = NLP.tokenize(s).reject do |t|
           # Don't include usernames/urls as tokens
           t.include?('@') || t.include?('http')
         end
-      end
 
-      tokens
+        tokens.map { |t| tikify(t) }
+      end
     end
 
     def consume(path)
@@ -63,9 +76,9 @@ module Ebooks
         next if l.include?('RT') || l.include?('MT') # Remove soft retweets
 
         if l.include?('@')
-          statements << NLP.normalize(l)
-        else
           mentions << NLP.normalize(l)
+        else
+          statements << NLP.normalize(l)
         end
       end
 
@@ -76,11 +89,11 @@ module Ebooks
 
       log "Tokenizing #{text.count('\n')} statements and #{mention_text.count('\n')} mentions"
 
-      @sentences = mass_tokenize(text)
-      @mentions = mass_tokenize(mention_text)
+      @sentences = mass_tikify(text)
+      @mentions = mass_tikify(mention_text)
 
       log "Ranking keywords"
-      @keywords = NLP.keywords(@sentences)
+      @keywords = NLP.keywords(text)
 
       self
     end
@@ -106,8 +119,8 @@ module Ebooks
       NLP.htmlentities.decode tweet
     end
 
-    def valid_tweet?(tokens, limit)
-      tweet = NLP.reconstruct(tokens)
+    def valid_tweet?(tikis, limit)
+      tweet = NLP.reconstruct(tikis, @tokens)
       tweet.length <= limit && !NLP.unmatched_enclosers?(tweet)
     end
 
@@ -118,24 +131,24 @@ module Ebooks
       retries = 0
       tweet = ""
 
-      while (tokens = generator.generate(3, :bigrams)) do
-        next if tokens.length <= 3 && !responding
-        break if valid_tweet?(tokens, limit)
+      while (tikis = generator.generate(3, :bigrams)) do
+        next if tikis.length <= 3 && !responding
+        break if valid_tweet?(tikis, limit)
 
         retries += 1
         break if retries >= retry_limit
       end
 
-      if verbatim?(tokens) && tokens.length > 3 # We made a verbatim tweet by accident
-        while (tokens = generator.generate(3, :unigrams)) do
-          break if valid_tweet?(tokens, limit) && !verbatim?(tokens)
+      if verbatim?(tikis) && tikis.length > 3 # We made a verbatim tweet by accident
+        while (tikis = generator.generate(3, :unigrams)) do
+          break if valid_tweet?(tikis, limit) && !verbatim?(tikis)
 
           retries += 1
           break if retries >= retry_limit
         end
       end
 
-      tweet = NLP.reconstruct(tokens)
+      tweet = NLP.reconstruct(tikis, @tokens)
 
       if retries >= retry_limit
         log "Unable to produce valid non-verbatim tweet; using \"#{tweet}\""
@@ -159,7 +172,7 @@ module Ebooks
 
       sentences.each do |sent|
         tokenized.each do |token|
-          if sent.map(&:downcase).include?(token)
+          if sent.map { |tiki| @tokens[tiki].downcase }.include?(token)
             relevant << sent unless NLP.stopword?(token)
             slightly_relevant << sent
           end

data/lib/twitter_ebooks/nlp.rb

@@ -69,9 +69,9 @@ module Ebooks
       Stemmer::stem_word(word.downcase)
     end
 
-    def self.keywords(sentences)
+    def self.keywords(text)
       # Preprocess to remove stopwords (highscore's blacklist is v. slow)
-      text = sentences.flatten.reject { |t| stopword?(t) }.join(' ')
+      text = NLP.tokenize(text).reject { |t| stopword?(t) }.join(' ')
 
       text = Highscore::Content.new(text)
 
@@ -91,11 +91,12 @@ module Ebooks
     end
 
     # Takes a list of tokens and builds a nice-looking sentence
-    def self.reconstruct(tokens)
+    def self.reconstruct(tikis, tokens)
       text = ""
       last_token = nil
-      tokens.each do |token|
-        next if token == INTERIM
+      tikis.each do |tiki|
+        next if tiki == INTERIM
+        token = tokens[tiki]
         text += ' ' if last_token && space_between?(last_token, token)
         text += token
         last_token = token

data/lib/twitter_ebooks/suffix.rb

@@ -15,24 +15,24 @@ module Ebooks
       @unigrams = {}
       @bigrams = {}
 
-      @sentences.each_with_index do |tokens, i|
-        last_token = INTERIM
-        tokens.each_with_index do |token, j|
-          @unigrams[last_token] ||= []
-          @unigrams[last_token] << [i, j]
-
-          @bigrams[last_token] ||= {}
-          @bigrams[last_token][token] ||= []
-
-          if j == tokens.length-1 # Mark sentence endings
-            @unigrams[token] ||= []
-            @unigrams[token] << [i, INTERIM]
-            @bigrams[last_token][token] << [i, INTERIM]
+      @sentences.each_with_index do |tikis, i|
+        last_tiki = INTERIM
+        tikis.each_with_index do |tiki, j|
+          @unigrams[last_tiki] ||= []
+          @unigrams[last_tiki] << [i, j]
+
+          @bigrams[last_tiki] ||= {}
+          @bigrams[last_tiki][tiki] ||= []
+
+          if j == tikis.length-1 # Mark sentence endings
+            @unigrams[tiki] ||= []
+            @unigrams[tiki] << [i, INTERIM]
+            @bigrams[last_tiki][tiki] << [i, INTERIM]
           else
-            @bigrams[last_token][token] << [i, j+1]
+            @bigrams[last_tiki][tiki] << [i, j+1]
           end
 
-          last_token = token
+          last_tiki = tiki
         end
       end
 
@@ -41,19 +41,18 @@ module Ebooks
 
     def generate(passes=5, n=:unigrams)
       index = rand(@sentences.length)
-      tokens = @sentences[index]
+      tikis = @sentences[index]
       used = [index] # Sentences we've already used
-      verbatim = [tokens] # Verbatim sentences to avoid reproducing
+      verbatim = [tikis] # Verbatim sentences to avoid reproducing
 
       0.upto(passes-1) do
-        log NLP.reconstruct(tokens) if $debug
-        varsites = {} # Map bigram start site => next token alternatives
+        varsites = {} # Map bigram start site => next tiki alternatives
 
-        tokens.each_with_index do |token, i|
-          next_token = tokens[i+1]
-          break if next_token.nil?
+        tikis.each_with_index do |tiki, i|
+          next_tiki = tikis[i+1]
+          break if next_tiki.nil?
 
-          alternatives = (n == :unigrams) ? @unigrams[next_token] : @bigrams[token][next_token]
+          alternatives = (n == :unigrams) ? @unigrams[next_tiki] : @bigrams[tiki][next_tiki]
           # Filter out suffixes from previous sentences
           alternatives.reject! { |a| a[1] == INTERIM || used.include?(a[0]) }
           varsites[i] = alternatives unless alternatives.empty?
@@ -67,7 +66,7 @@ module Ebooks
             start, alt = site[0], site[1].sample
             verbatim << @sentences[alt[0]]
             suffix = @sentences[alt[0]][alt[1]..-1]
-            potential = tokens[0..start+1] + suffix
+            potential = tikis[0..start+1] + suffix
 
             # Ensure we're not just rebuilding some segment of another sentence
             unless verbatim.find { |v| NLP.subseq?(v, potential) || NLP.subseq?(potential, v) }
@@ -80,10 +79,10 @@ module Ebooks
           break if variant
         end
 
-        tokens = variant if variant
+        tikis = variant if variant
       end
 
-      tokens
+      tikis
     end
   end
 end

data/lib/twitter_ebooks/version.rb

@@ -1,3 +1,3 @@
 module Ebooks
-  VERSION = "2.2.9"
+  VERSION = "2.3.0"
 end

data/spec/model_spec.rb

@@ -1,9 +1,27 @@
 require 'spec_helper'
 require 'memory_profiler'
+require 'tempfile'
 
 def Process.rss; `ps -o rss= -p #{Process.pid}`.chomp.to_i; end
 
 describe Ebooks::Model do
+  describe 'making tweets' do
+    before(:all) { @model = Ebooks::Model.consume(path("data/0xabad1dea.json")) }
+
+    it "generates a tweet" do
+      s = @model.make_statement
+      expect(s.length).to be <= 140
+      puts s
+    end
+
+    it "generates an appropriate response" do
+      s = @model.make_response("hi")
+      expect(s.length).to be <= 140
+      expect(s.downcase).to include("hi")
+      puts s
+    end
+  end
+
   it "does not use a ridiculous amount of memory" do
     report = MemoryUsage.report do
       model = Ebooks::Model.consume(path("data/0xabad1dea.json"))
@@ -11,4 +29,30 @@ describe Ebooks::Model do
 
     expect(report.total_memsize).to be < 1000000000
   end
+
+  describe '.consume' do
+    it 'interprets lines with @ as mentions' do
+      file = Tempfile.new('mentions')
+      file.write('@m1spy hello!')
+      file.close
+
+      model = Ebooks::Model.consume(file.path)
+      expect(model.sentences.count).to eq 0
+      expect(model.mentions.count).to eq 1
+
+      file.unlink
+    end
+
+    it 'interprets lines without @ as statements' do
+      file = Tempfile.new('statements')
+      file.write('hello!')
+      file.close
+
+      model = Ebooks::Model.consume(file.path)
+      expect(model.mentions.count).to eq 0
+      expect(model.sentences.count).to eq 1
+
+      file.unlink
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitter_ebooks
 version: !ruby/object:Gem::Version
-  version: 2.2.9
+  version: 2.3.0
 platform: ruby
 authors:
 - Jaiden Mispy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-20 00:00:00.000000000 Z
+date: 2014-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec