wordtree 0.3.1 → 0.4.0

data/spec/wordtree/book_spec.rb

@@ -13,50 +13,32 @@ describe WordTree::Book do
   end
 
   it "can return cleaned content" do
-    book = WordTree::Book.create("book", {}, "Wi&ld\nContent!")
-    expect(book.content_clean).to eq("wild content\n")
+    content = "Wi&ld\nContent!"
+    book = WordTree::Book.create("book", {}, content)
+    expect(content).to eq("Wi&ld\nContent!")
+    expect(book.content_clean).to eq("wild content.")
   end
 
-  context "ngrams" do
-    let(:content) { "A man. A plan. And a man."}
-    let(:book) { WordTree::Book.create("book", {}, content) }
-    let(:one_grams) { { "a" => 3, "man" => 2, "plan" => 1, "and" => 1, "." => 3 } }
-    let(:two_grams) {
-      {"a man" => 2, "man ." => 2, ". a" => 1, "a plan" => 1,
-       "plan ." => 1, ". and" => 1, "and a" => 1}
-    }
-    describe "#count_ngrams" do
-      it "creates a hash lookup table" do
-        hash = book.count_ngrams(1)
-        expect(hash).to be_a(Hash)
-      end
-
-      it "has counts of ngrams" do
-        hash = book.count_ngrams(1)
-        expect(hash).to eq(one_grams)
-        hash = book.count_ngrams(2)
-        expect(hash).to eq(two_grams)
-      end
-
-      it "memoizes ngrams" do
-        expect(book).to receive(:count_ngrams).with(1).and_return(one_grams)
-        expect(book.ngrams(1)).to eq one_grams
-        expect(book).to_not receive(:count_ngrams)
-        expect(book.ngrams(1)).to eq one_grams
-      end
-    end
-
-    describe "#set_ngrams" do
-      it "sets the lookup hash" do
-        book.set_ngrams(1, {"one" => 1})
-        expect(book.ngrams(1)).to eq("one" => 1)
-        expect(book.ngrams(2)).to eq(two_grams)
-      end
-
-      it "raises an error when not a hash" do
-        expect{ book.set_ngrams(1, "string") }.to raise_error
-        expect{ book.set_ngrams(1, nil) }.to raise_error
-      end
-    end
-  end
+  # context "ngrams" do
+  #   let(:content) { "A man. A plan. And a man."}
+  #   let(:book) { WordTree::Book.create("book", {}, content) }
+  #   let(:one_grams) { { "a" => 3, "man" => 2, "plan" => 1, "and" => 1, "." => 3 } }
+  #   let(:two_grams) {
+  #     {"a man" => 2, "man ." => 2, ". a" => 1, "a plan" => 1,
+  #      "plan ." => 1, ". and" => 1, "and a" => 1}
+  #   }
+  #   describe "#ngrams" do
+  #     it "creates a hash lookup table" do
+  #       hash = book.count_ngrams(1)
+  #       expect(hash).to be_a(Hash)
+  #     end
+
+  #     it "has counts of ngrams" do
+  #       hash = book.count_ngrams(1)
+  #       expect(hash).to eq(one_grams)
+  #       hash = book.count_ngrams(2)
+  #       expect(hash).to eq(two_grams)
+  #     end
+  #   end
+  # end
 end

data/spec/wordtree/disk/librarian_spec.rb

@@ -32,12 +32,6 @@ describe WordTree::Disk::Librarian do
         expect(book.year).to eq(1800)
         expect(book.content).to eq("Book with content")
       end
-
-      it "loads ngrams if available" do
-        book = librarian.find("book")
-        expect(book).to_not receive(:count_ngrams)
-        expect(book.ngrams(1)).to eq("xyz" => 1)
-      end
     end
 
     describe "#each" do
@@ -46,43 +40,5 @@ describe WordTree::Disk::Librarian do
         expect(book_sizes).to contain_exactly(17, 23)
       end
     end
-
-    it "saves ngrams to disk" do
-      tmp_root = Dir.mktmpdir
-      tmp_library = WordTree::Disk::Library.new(tmp_root)
-      tmp_librarian = WordTree::Disk::Librarian.new(tmp_library)
-
-      book = librarian.find("book")
-      book.ngrams(1)
-      book.ngrams(2)
-
-      tmp_librarian.save(book)
-
-      ngrams_filepath = tmp_library.path_to("book", :ngrams, :n => 1)
-      expect(File.exist?(ngrams_filepath)).to be_truthy
-    end
-
-    it "saves to disk (yaml, content)" do
-      tmp_root = Dir.mktmpdir
-      tmp_library = WordTree::Disk::Library.new(tmp_root)
-      tmp_librarian = WordTree::Disk::Librarian.new(tmp_library)
-
-      book = librarian.find_without_ngrams("book")
-
-      book.source = "test"
-      book.content += "."
-
-      tmp_librarian.save(book)
-
-      updated = Preamble.load(tmp_library.path_to("book"))
-      expect(updated.metadata).to eq(
-        :id => "book",
-        :archive_org_id => "book",
-        :year => 1800,
-        :source => "test",
-        :size_bytes => 17)
-      expect(updated.content).to eq("Book with content.")
-    end
-
   end
 end

data/spec/wordtree/text_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+require 'wordtree/text'
+require 'timeout'
+
+describe WordTree::Text do
+  context "#split_near" do
+    it "splits on spaces" do
+      line, rem = WordTree::Text.split_near("it is near", 7)
+      expect(line).to eq("it is")
+      expect(rem).to eq("near")
+    end
+
+    it "removes a space if index lands on one" do
+      line, rem = WordTree::Text.split_near("it is near", 5)
+      expect(line).to eq("it is")
+      expect(rem).to eq("near")
+    end
+
+    it "keeps the whole line if index is >= length of line" do
+      line, rem = WordTree::Text.split_near("it is near", 10)
+      expect(line).to eq("it is near")
+      expect(rem).to eq("")
+
+      line, rem = WordTree::Text.split_near("it is near", 11)
+      expect(line).to eq("it is near")
+      expect(rem).to eq("")
+    end
+
+    it "splits at the index anyway if no spaces are found" do
+      line, rem = WordTree::Text.split_near("itisnear", 4)
+      expect(line).to eq("itis")
+      expect(rem).to eq("near")
+    end
+  end
+
+  describe "#clean" do
+    it "wraps" do
+      sample_text = "This, [here]  is awesome, right"
+      cleaned = WordTree::Text.clean(sample_text)
+      expect(cleaned).to eq("this here is awesome right")
+    end
+
+    it "joins lines ending in -" do
+      sample_text = "What-\never\ndo you\n mean?"
+      cleaned = WordTree::Text.clean(sample_text)
+      expect(cleaned).to eq("whatever do you mean.")
+    end
+
+    it "does not ignore sentence boundaries" do
+      sample_text = "This is a sentence. And so is this? Keep the dots."
+      cleaned = WordTree::Text.clean(sample_text)
+      expect(cleaned).to eq("this is a sentence.and so is this.keep the dots.")
+    end
+
+    it "compresses sentence boundary punctuation and spaces" do
+      sample_text = "words . . and.. stuff"
+      cleaned = WordTree::Text.clean(sample_text)
+      expect(cleaned).to eq("words.and.stuff")
+    end
+  end
+
+  describe "#common_trigrams" do
+    it "returns 0 for strings of len < 3" do
+      expect(WordTree::Text.common_trigrams("")).to eq 0
+      expect(WordTree::Text.common_trigrams("1")).to eq 0
+      expect(WordTree::Text.common_trigrams("12")).to eq 0
+    end
+
+    it "returns 0 for strings without common trigrams" do
+      expect(WordTree::Text.common_trigrams("!{*@*!()}")).to eq 0
+      expect(WordTree::Text.common_trigrams("qwrtypzzx")).to eq 0
+      expect(WordTree::Text.common_trigrams("         ")).to eq 0
+    end
+
+    it "returns correct counts for strings with trigrams" do
+      expect(WordTree::Text.common_trigrams("what")).to eq 1
+      expect(WordTree::Text.common_trigrams("the wall")).to eq 2
+    end
+  end
+
+end

data/wordtree.gemspec

@@ -5,12 +5,12 @@ require 'wordtree/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "wordtree"
-  spec.version       = Wordtree::VERSION
+  spec.version       = WordTree::VERSION
   spec.authors       = ["Duane Johnson"]
   spec.email         = ["duane.johnson@gmail.com"]
   spec.description   = %q{WordTree common library code}
-  spec.summary       = %q{Wordtree common library code}
-  spec.homepage      = ""
+  spec.summary       = %q{WordTree common library code}
+  spec.homepage      = "https://github.com/wordtreefoundation/wordtree-ruby"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files`.split($/)
@@ -18,19 +18,21 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "virtus"
-  spec.add_dependency "preamble", ">= 0.0.3"
-  spec.add_dependency "archivist-client", ">= 0.1.7"
-  spec.add_dependency "retriable"
+  spec.extensions    = %w[ext/extconf.rb]
+
+  spec.add_dependency "virtus",   "~> 1.0"
+  spec.add_dependency "preamble", "0.0.3"
+  spec.add_dependency "archivist-client", "0.1.7"
+  spec.add_dependency "retriable", "1.4.1"
   spec.add_dependency "simhash", "0.2.5"
-  spec.add_dependency "rethinkdb", "~> 1.14.0"
+  spec.add_dependency "rethinkdb", "~> 1.14"
 
   spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "debugger"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "guard"
-  spec.add_development_dependency "guard-rspec"
-  spec.add_development_dependency "vcr"
-  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "rake", "~> 10.3"
+  spec.add_development_dependency "byebug", "~> 3.4"
+  spec.add_development_dependency "rspec", "~> 3.1"
+  spec.add_development_dependency "guard", "~> 2.6"
+  spec.add_development_dependency "guard-rspec", "~> 4.3"
+  spec.add_development_dependency "vcr", "~> 2.9"
+  spec.add_development_dependency "webmock", "~> 1.18"
 end

metadata

@@ -1,84 +1,74 @@
 --- !ruby/object:Gem::Specification
 name: wordtree
 version: !ruby/object:Gem::Version
-  version: 0.3.1
-  prerelease: 
+  version: 0.4.0
 platform: ruby
 authors:
 - Duane Johnson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-08 00:00:00.000000000 Z
+date: 2014-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: virtus
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: preamble
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.0.3
 - !ruby/object:Gem::Dependency
   name: archivist-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.1.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.1.7
 - !ruby/object:Gem::Dependency
   name: retriable
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.1
 - !ruby/object:Gem::Dependency
   name: simhash
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -94,169 +83,157 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rethinkdb
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: '1.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: '1.14'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.3'
 - !ruby/object:Gem::Dependency
-  name: debugger
+  name: byebug
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.4'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.3'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.9'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.18'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.18'
 description: WordTree common library code
 email:
 - duane.johnson@gmail.com
 executables: []
-extensions: []
+extensions:
+- ext/extconf.rb
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
+- ".gitignore"
+- ".rspec"
 - Gemfile
 - Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- ext/Makefile
+- ext/extconf.rb
+- ext/wordtree.cc
 - lib/wordtree.rb
 - lib/wordtree/archdown.rb
 - lib/wordtree/book.rb
+- lib/wordtree/book_list.rb
 - lib/wordtree/db/librarian.rb
 - lib/wordtree/disk/librarian.rb
 - lib/wordtree/disk/library.rb
 - lib/wordtree/disk/library_locator.rb
-- lib/wordtree/text_utils.rb
+- lib/wordtree/ngrams.rb
+- lib/wordtree/text.rb
 - lib/wordtree/version.rb
 - spec/fixtures/cassettes/archive_org_download_book.yml
 - spec/fixtures/library/bo/ok/book/book.1grams.json
@@ -268,33 +245,32 @@ files:
 - spec/wordtree/disk/librarian_spec.rb
 - spec/wordtree/disk/library_locator_spec.rb
 - spec/wordtree/disk/library_spec.rb
-- spec/wordtree/text_utils_spec.rb
+- spec/wordtree/text_spec.rb
 - wordtree.gemspec
-homepage: ''
+homepage: https://github.com/wordtreefoundation/wordtree-ruby
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
-summary: Wordtree common library code
+specification_version: 4
+summary: WordTree common library code
 test_files:
 - spec/fixtures/cassettes/archive_org_download_book.yml
 - spec/fixtures/library/bo/ok/book/book.1grams.json
@@ -306,5 +282,4 @@ test_files:
 - spec/wordtree/disk/librarian_spec.rb
 - spec/wordtree/disk/library_locator_spec.rb
 - spec/wordtree/disk/library_spec.rb
-- spec/wordtree/text_utils_spec.rb
-has_rdoc: 
+- spec/wordtree/text_spec.rb