twitter_ebooks 3.1.0 → 3.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0bf168a080a0aabf939715c3dcb59655fca83463
-  data.tar.gz: 88ddb4ead32b0ca21db59b6edb4dcd97cf0d0594
+  metadata.gz: eb109277da9591a48da1231fe81690a569a94e16
+  data.tar.gz: 84a3a30a0e8bdec98e6998a22897617d6349807a
 SHA512:
-  metadata.gz: 13873b6ea1ba04b79e0b5b44e735aa41a97b8aeaee798eee2e3cc43795dabb8bc4149ecc37418d341818d671bee7fd91f10316d19e53c941a8d652fc7dae1cc7
-  data.tar.gz: 48b0e54e02267df3e5cc12c7a75d2deea62adfeaed9cf1e8274b6e30917cda55041f428ff3cd5bd5cf62eb1f8c84fe0880b1bb002e0c0db650892885ff221978
+  metadata.gz: e492aa786f3d29d16fa878cc3bcb615ba7258ee01d5b5f4a1817279bc08bb4d0d485e78696cd9f39ebe2cc6b81aa01187aa69b735c9ea5a9b9fe496f00954b1c
+  data.tar.gz: 5208f5d31d895ecc60ab6829f26cea43c80d59b39f53063911593db7914de80cf6d7cd852a4ba65f6bd6435bac09d27de5e5f089d1427344cbe3d04c5d4e5146

data/README.md

@@ -135,10 +135,10 @@ The secondary function is the "interesting keywords" list. For example, I use th
 
 ``` ruby
 top100 = model.keywords.take(100)
-tokens = Ebooks::NLP.tokenize(tweet[:text])
+tokens = Ebooks::NLP.tokenize(tweet.text)
 
 if tokens.find { |t| top100.include?(t) }
-  bot.favorite(tweet[:id])
+  favorite(tweet)
 end
 ```
 

data/bin/ebooks

@@ -25,9 +25,11 @@ Usage:
      ebooks auth
      ebooks consume <corpus_path> [corpus_path2] [...]
      ebooks consume-all <model_name> <corpus_path> [corpus_path2] [...]
+     ebooks append <model_name> <corpus_path>
      ebooks gen <model_path> [input]
      ebooks archive <username> [path]
      ebooks tweet <model_path> <botname>
+     ebooks version
 STR
 
   def self.help(command=nil)
@@ -115,6 +117,24 @@ STR
     log "Corpuses consumed to #{outpath}"
   end
 
+  HELP.append = <<-STR
+    Usage: ebooks append <model_name> <corpus_path>
+
+    Process then append the provided corpus to the model 
+    instead of overwriting.
+  STR
+
+  def self.append(name, path)
+    if !name || !path
+      help :append
+      exit 1
+    end
+
+    Ebooks::Model.consume(path).append(File.join(APP_PATH,'model',"#{name}.model"))
+    log "Corpus appended to #{name}.model"
+  end
+ 
+
   HELP.jsonify = <<-STR
     Usage: ebooks jsonify <tweets.csv> [tweets.csv2] [...]
 
@@ -275,6 +295,17 @@ STR
     require 'pry'; Ebooks.module_exec { pry }
   end
 
+  HELP.version = <<-STR
+    Usage: ebooks version
+
+    Shows you twitter_ebooks' version number.
+  STR
+  
+  def self.version
+    require File.expand_path('../../lib/twitter_ebooks/version', __FILE__)
+    log Ebooks::VERSION
+  end
+
   HELP.start = <<-STR
     Usage: ebooks s[tart] [botname]
 
@@ -368,6 +399,7 @@ STR
     when "new" then new(args[1])
     when "consume" then consume(args[1..-1])
     when "consume-all" then consume_all(args[1], args[2..-1])
+    when "append" then append(args[1],args[2])
     when "gen" then gen(args[1], args[2..-1].join(' '))
     when "archive" then archive(args[1], args[2])
     when "tweet" then tweet(args[1], args[2])
@@ -378,6 +410,7 @@ STR
     when "start" then start(args[1])
     when "s" then start(args[1])
     when "help" then help(args[1])
+    when "version" then version
     else
       log "No such command '#{args[0]}'"
       help

data/lib/twitter_ebooks/model.rb

@@ -69,6 +69,35 @@ module Ebooks
       self
     end
 
+    # Append a generated model to existing model file instead of overwriting it
+    # @param path [String]
+    def append(path)
+      existing = File.file?(path)
+      if !existing
+        log "No existing model found at #{path}"
+        return
+      else
+        #read-in and deserialize existing model
+        props = Marshal.load(File.open(path,'rb') { |old| old.read })
+        old_tokens = props[:tokens]
+        old_sentences = props[:sentences]
+        old_mentions = props[:mentions]
+        old_keywords = props[:keywords]
+
+        #append existing properties to new ones and overwrite with new model
+        File.open(path, 'wb') do |f|
+          f.write(Marshal.dump({
+            tokens: @tokens.concat(old_tokens),
+            sentences: @sentences.concat(old_sentences),
+            mentions: @mentions.concat(old_mentions),
+            keywords: @keywords.concat(old_keywords)
+          }))
+        end
+      end
+      self
+    end
+     
+
     def initialize
       @tokens = []
 
@@ -80,7 +109,13 @@ module Ebooks
     # @param token [String]
     # @return [Integer]
     def tikify(token)
-      @tikis[token] or (@tokens << token and @tikis[token] = @tokens.length-1)
+      if @tikis.has_key?(token) then
+        return @tikis[token]
+      else
+        (@tokens.length+1)%1000 == 0 and puts "#{@tokens.length+1} tokens"
+        @tokens << token 
+        return @tikis[token] = @tokens.length-1
+      end
     end
 
     # Convert a body of text into arrays of tikis
@@ -143,8 +178,8 @@ module Ebooks
         end
       end
 
-      text = statements.join("\n")
-      mention_text = mentions.join("\n")
+      text = statements.join("\n").encode('UTF-8', :invalid => :replace)
+      mention_text = mentions.join("\n").encode('UTF-8', :invalid => :replace)
 
       lines = nil; statements = nil; mentions = nil # Allow garbage collection
 
@@ -155,6 +190,7 @@ module Ebooks
 
       log "Ranking keywords"
       @keywords = NLP.keywords(text).top(200).map(&:to_s)
+      log "Top keywords: #{@keywords[0]} #{@keywords[1]} #{@keywords[2]}"
 
       self
     end
@@ -218,6 +254,7 @@ module Ebooks
       tweet = ""
 
       while (tikis = generator.generate(3, :bigrams)) do
+        log "Attempting to produce tweet try #{retries+1}/#{retry_limit}"
         next if tikis.length <= 3 && !responding
         break if valid_tweet?(tikis, limit)
 
@@ -226,6 +263,7 @@ module Ebooks
       end
 
       if verbatim?(tikis) && tikis.length > 3 # We made a verbatim tweet by accident
+        log "Attempting to produce unigram tweet try #{retries+1}/#{retry_limit}"
         while (tikis = generator.generate(3, :unigrams)) do
           break if valid_tweet?(tikis, limit) && !verbatim?(tikis)
 

data/lib/twitter_ebooks/nlp.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require 'fast-stemmer'
 require 'highscore'
+require 'htmlentities'
 
 module Ebooks
   module NLP
@@ -42,7 +43,6 @@ module Ebooks
     # Lazily load HTML entity decoder
     # @return [HTMLEntities]
     def self.htmlentities
-      require 'htmlentities'
       @htmlentities ||= HTMLEntities.new
     end
 

data/lib/twitter_ebooks/suffix.rb

@@ -19,6 +19,9 @@ module Ebooks
       @bigrams = {}
 
       @sentences.each_with_index do |tikis, i|
+        if (i % 10000 == 0) then
+          log ("Building: sentence #{i} of #{sentences.length}")
+        end
         last_tiki = INTERIM
         tikis.each_with_index do |tiki, j|
           @unigrams[last_tiki] ||= []

data/lib/twitter_ebooks/version.rb

@@ -1,3 +1,3 @@
 module Ebooks
-  VERSION = "3.1.0"
+  VERSION = "3.1.1"
 end

data/twitter_ebooks.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'pry-byebug'
   gem.add_development_dependency 'yard'
 
-  gem.add_runtime_dependency 'twitter', '~> 5.0'
+  gem.add_runtime_dependency 'twitter', '~> 5.15'
   gem.add_runtime_dependency 'rufus-scheduler'
   gem.add_runtime_dependency 'gingerice'
   gem.add_runtime_dependency 'htmlentities'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitter_ebooks
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.1.1
 platform: ruby
 authors:
 - Jaiden Mispy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-24 00:00:00.000000000 Z
+date: 2015-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.15'
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
@@ -278,7 +278,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Markov chains for all your friends~
@@ -289,3 +289,4 @@ test_files:
 - spec/memprof.rb
 - spec/model_spec.rb
 - spec/spec_helper.rb
+has_rdoc: