twitter_ebooks 2.0.4 → 2.0.5

data/lib/twitter_ebooks/model.rb

@@ -40,15 +40,8 @@ module Ebooks
       log "Tokenizing #{sentences.length} sentences"
       @sentences = sentences.map { |sent| NLP.tokenize(sent) }
 
-      log "Building markov model"
-      @markov = MarkovModel.build(@sentences)
-
       log "Ranking keywords"
-      require 'benchmark'
-      puts Benchmark.measure {
-        @keywords = NLP.keywords(@sentences)
-        p @keywords.top(100)
-      }
+      @keywords = NLP.keywords(@sentences)
 
       self
     end
@@ -75,10 +68,10 @@ module Ebooks
     end
 
     def markov_statement(limit=140, markov=nil)
-      markov ||= @markov
+      markov ||= MarkovModel.build(@sentences)
       tweet = ""
 
-      while (tweet = markov.generate) do
+      while (tweet = markov.generate(@sentences)) do
         next if tweet.length > limit
         next if NLP.unmatched_enclosers?(tweet)
         break if tweet.length > limit*0.4 || rand > 0.8
@@ -113,9 +106,6 @@ module Ebooks
       # First try 
       relevant, slightly_relevant = relevant_sentences(input)
 
-      p relevant
-      p slightly_relevant.length
-
       if relevant.length >= 3
         markov = MarkovModel.new.consume(relevant)
         markov_statement(limit, markov)

data/lib/twitter_ebooks/nlp.rb

@@ -23,10 +23,6 @@ module Ebooks
     def self.adjectives
       @adjectives ||= File.read(File.join(DATA_PATH, 'adjectives.txt')).split
     end
-    
-    def self.wordfreq
-      @wordfreq ||= JSON.load(File.read(File.join(DATA_PATH, 'wordfreq.json')))
-    end
 
     # POS tagger
     def self.tagger
@@ -94,25 +90,6 @@ module Ebooks
       text.keywords
     end
 
-    def self.stemset(sentence)
-      tokens = sentence.is_a?(Array) ? sentence : tokenize(sentence)
-      tokens.map(&:downcase)
-            .reject { |token| stopwords.include?(token) }
-            .map { |t| stemmer.stem(t) }
-            .to_set
-    end
-
-    # Builds a token stem frequency map
-    def self.stemfreq(sentences)
-      freqmap = {}
-      sentences.flatten.each do |token|
-        stem = NLP.stem(token)
-        freqmap[stem] ||= 0
-        freqmap[stem] += 1
-      end
-      freqmap
-    end
-
     # Takes a list of tokens and builds a nice-looking sentence
     def self.reconstruct(tokens)
       text = ""

data/lib/twitter_ebooks/version.rb

@@ -1,3 +1,3 @@
 module Ebooks
-  VERSION = "2.0.4"
+  VERSION = "2.0.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitter_ebooks
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -171,11 +171,9 @@ files:
 - README.md
 - Rakefile
 - bin/ebooks
-- data/ANC-all-count.txt
 - data/adjectives.txt
 - data/nouns.txt
 - data/stopwords.txt
-- data/wordfreq.json
 - lib/twitter_ebooks.rb
 - lib/twitter_ebooks/archiver.rb
 - lib/twitter_ebooks/bot.rb