twitter_ebooks 2.0.3 → 2.0.4

data/test/keywords.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'twitter_ebooks'
+require 'minitest/autorun'
+require 'benchmark'
+
+module Ebooks
+  class TestKeywords < Minitest::Test
+    corpus = NLP.normalize(File.read(ARGV[0]))
+    puts "Finding and ranking keywords"
+    puts Benchmark.measure {
+      NLP.keywords(corpus).top(50).each do |keyword|
+        puts "#{keyword.text} #{keyword.weight}"
+      end
+    }
+  end
+end

data/twitter_ebooks.gemspec

@@ -19,12 +19,10 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency 'twitter'
   gem.add_runtime_dependency 'tweetstream'
-  gem.add_runtime_dependency 'tactful_tokenizer'
-  gem.add_runtime_dependency 'tokenizer'
-  gem.add_runtime_dependency 'ruby-stemmer'
-  gem.add_runtime_dependency 'engtagger'
-  gem.add_runtime_dependency 'linguistics'
   gem.add_runtime_dependency 'rufus-scheduler'
   gem.add_runtime_dependency 'gingerice'
   gem.add_runtime_dependency 'htmlentities'
+  gem.add_runtime_dependency 'engtagger'
+  gem.add_runtime_dependency 'fast-stemmer'
+  gem.add_runtime_dependency 'highscore'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitter_ebooks
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-05 00:00:00.000000000 Z
+date: 2013-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -60,7 +60,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: tactful_tokenizer
+  name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -76,7 +76,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: tokenizer
+  name: gingerice
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -92,7 +92,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ruby-stemmer
+  name: htmlentities
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -124,39 +124,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: linguistics
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rufus-scheduler
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: gingerice
+  name: fast-stemmer
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -172,7 +140,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: htmlentities
+  name: highscore
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -203,9 +171,11 @@ files:
 - README.md
 - Rakefile
 - bin/ebooks
+- data/ANC-all-count.txt
 - data/adjectives.txt
 - data/nouns.txt
 - data/stopwords.txt
+- data/wordfreq.json
 - lib/twitter_ebooks.rb
 - lib/twitter_ebooks/archiver.rb
 - lib/twitter_ebooks/bot.rb
@@ -213,12 +183,14 @@ files:
 - lib/twitter_ebooks/model.rb
 - lib/twitter_ebooks/nlp.rb
 - lib/twitter_ebooks/version.rb
+- script/process_anc_data.rb
 - skeleton/.gitignore
 - skeleton/Procfile
 - skeleton/bots.rb
 - skeleton/corpus/README.md
-- skeleton/model/README.md
+- skeleton/run.rb
 - test/corpus/0xabad1dea.tweets
+- test/keywords.rb
 - test/tokenize.rb
 - twitter_ebooks.gemspec
 homepage: ''
@@ -247,4 +219,5 @@ specification_version: 3
 summary: Markov chains for all your friends~
 test_files:
 - test/corpus/0xabad1dea.tweets
+- test/keywords.rb
 - test/tokenize.rb

data/skeleton/model/README.md

@@ -1 +0,0 @@
-This is where the output of `ebooks consume <corpus_path>` goes. You can load these files using Model.load(path), and `ebooks gen <path>` for testing.