twitter_ebooks 2.2.6 → 2.2.7

data/spec/memprof.rb

@@ -0,0 +1,37 @@
+require 'objspace'
+
+module MemoryUsage
+  MemoryReport = Struct.new(:total_memsize)
+
+  def self.full_gc
+    GC.start(full_mark: true)
+  end
+
+  def self.report(&block)
+    rvalue_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]
+
+    full_gc
+    GC.disable
+
+    total_memsize = 0
+
+    generation = nil
+    ObjectSpace.trace_object_allocations do
+      generation = GC.count
+      block.call
+    end
+
+    ObjectSpace.each_object do |obj|
+      next unless generation == ObjectSpace.allocation_generation(obj)
+      memsize = ObjectSpace.memsize_of(obj) + rvalue_size
+      # compensate for API bug
+      memsize = rvalue_size if memsize > 100_000_000_000
+      total_memsize += memsize
+    end
+
+    GC.enable
+    full_gc
+
+    return MemoryReport.new(total_memsize)
+  end
+end

data/spec/model_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+require 'memory_profiler'
+
+def Process.rss; `ps -o rss= -p #{Process.pid}`.chomp.to_i; end
+
+describe Ebooks::Model do
+  it "does not use a ridiculous amount of memory" do
+    report = MemoryUsage.report do
+      model = Ebooks::Model.consume(path("data/0xabad1dea.json"))
+    end
+
+    expect(report.total_memsize).to be < 1000000000
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require 'twitter_ebooks'
+require_relative 'memprof'
+
+def path(relpath)
+  File.join(File.dirname(__FILE__), relpath)
+end

data/twitter_ebooks.gemspec

@@ -15,10 +15,13 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Ebooks::VERSION
 
-  gem.add_runtime_dependency 'minitest'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'memory_profiler'
+  gem.add_development_dependency 'pry-byebug'
 
-  gem.add_runtime_dependency 'twitter', '~> 4.5'
-  gem.add_runtime_dependency 'tweetstream', '= 2.5'
+
+  gem.add_runtime_dependency 'twitter', '~> 5.1'
+  gem.add_runtime_dependency 'tweetstream'
   gem.add_runtime_dependency 'rufus-scheduler'
   gem.add_runtime_dependency 'gingerice'
   gem.add_runtime_dependency 'htmlentities'

metadata

@@ -1,23 +1,51 @@
 --- !ruby/object:Gem::Specification
 name: twitter_ebooks
 version: !ruby/object:Gem::Version
-  version: 2.2.6
+  version: 2.2.7
 platform: ruby
 authors:
 - Jaiden Mispy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-29 00:00:00.000000000 Z
+date: 2014-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: memory_profiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -30,28 +58,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.5'
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.5'
+        version: '5.1'
 - !ruby/object:Gem::Dependency
   name: tweetstream
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
@@ -145,6 +173,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - Gemfile
 - LICENSE
 - README.md
@@ -161,7 +190,6 @@ files:
 - lib/twitter_ebooks/nlp.rb
 - lib/twitter_ebooks/suffix.rb
 - lib/twitter_ebooks/version.rb
-- script/process_anc_data.rb
 - skeleton/.gitignore
 - skeleton/Gemfile
 - skeleton/Procfile
@@ -169,6 +197,11 @@ files:
 - skeleton/corpus/.gitignore
 - skeleton/model/.gitignore
 - skeleton/run.rb
+- spec/data/0xabad1dea.json
+- spec/data/0xabad1dea.model
+- spec/memprof.rb
+- spec/model_spec.rb
+- spec/spec_helper.rb
 - test/corpus/0xabad1dea.tweets
 - test/keywords.rb
 - test/tokenize.rb
@@ -197,6 +230,11 @@ signing_key:
 specification_version: 4
 summary: Markov chains for all your friends~
 test_files:
+- spec/data/0xabad1dea.json
+- spec/data/0xabad1dea.model
+- spec/memprof.rb
+- spec/model_spec.rb
+- spec/spec_helper.rb
 - test/corpus/0xabad1dea.tweets
 - test/keywords.rb
 - test/tokenize.rb

data/script/process_anc_data.rb

@@ -1,19 +0,0 @@
-#!/usr/bin/env ruby
-# encoding: utf-8
-
-require 'json'
-
-freqmap = {}
-
-data = File.read("data/ANC-all-count.txt")
-data = data.unpack("C*").pack("U*")
-
-data.lines.each do |l|
-  vals = l.split("\t")
-  
-  freqmap[vals[0]] = vals[-1].to_i
-end
-
-File.open("data/wordfreq.json", 'w') do |f|
-  f.write(JSON.dump(freqmap))
-end