wordtree 0.0.2 → 0.1.0

data/README.md

@@ -30,6 +30,11 @@ Modify and save a book to your "library":
     book.year = 2014
     librarian.save(book)
 
+Clean the text:
+
+    book.clean_content
+    # returns the book without punctuation
+
 
 
 ## Contributing

data/lib/wordtree/book.rb

@@ -1,4 +1,6 @@
 require 'virtus'
+require 'simhash'
+
 require 'wordtree/text_utils'
 
 module WordTree
@@ -13,9 +15,9 @@ module WordTree
     attribute :source, String
     attribute :status, String
     # Size of the content in bytes
-    attribute :size_bytes, Integer
+    attribute :size_bytes, Integer, :default => :content_size
     # A simhash (locality-sensitive hash) of the content
-    attribute :simhash, String
+    attribute :simhash, Integer, :default => :calculate_simhash
 
     attribute :content, String
 
@@ -28,11 +30,19 @@ module WordTree
     end
 
     def metadata
-      attributes.select{ |k,v| !v.nil? && k != :content && k != :id }
+      attributes.select{ |k,v| !v.nil? && k != :content }
+    end
+
+    def content_clean(wrap=120)
+      TextUtils.clean_text(content, wrap)
+    end
+
+    def content_size
+      content ? content.size : nil
     end
 
-    def clean_content
-      TextUtils.clean_text(content)
+    def calculate_simhash
+      content ? content_clean.simhash(:split_by => /\s/) : nil
     end
   end
 end

data/lib/wordtree/db/librarian.rb

@@ -0,0 +1,68 @@
+require 'rethinkdb'
+require 'wordtree/book'
+
+module WordTree
+  module DB
+    class Librarian
+      def initialize(rethinkdb_connection)
+        @rdb = rethinkdb_connection
+        @r = RethinkDB::RQL.new
+      end
+
+      def find(book_id)
+        result = @r.table('books').get(book_id).run(@rdb)
+        result ? Book.new(result) : nil
+      end
+
+      def save(book)
+        result = @r.table('books').insert(book.metadata, :upsert => true).run(@rdb)
+        return result["replaced"] == 1 || result["inserted"] == 1 || result["unchanged"] == 1
+      end
+
+      def search(params, page=1, per_page=20)
+        conditions = match_list(params,
+                       [:id, :title, :author, :source, :status],
+                       [:year, :size_bytes])
+        cursor = @r.table('books').
+                   order_by(:index => 'year').
+                   filter(&conditions).
+                   skip((page-1)*per_page).
+                   limit(per_page).
+                   run(@rdb)
+        if !(results = cursor.to_a).empty?
+          results.map{ |result| Book.new(result) }
+        else
+          nil
+        end
+      end
+
+    protected
+
+      # Create a condition Proc suitable for RethinkDB search queries
+      def match_list(params, string_keys=[], numeric_keys=[], escape=true)
+        Proc.new do |record|
+          (
+            string_keys.map do |key|
+              if params[key]
+                term = escape ? Regexp.escape(params[key]) : params[key]
+                record[key.to_s].match("(?i)#{term}")
+              end
+            end +
+            numeric_keys.map do |key|
+              if params[key]
+                if params[key].include?(',')
+                  low, high = params[key].split(',', 2).map{ |v| v.to_i }
+                  (record[key.to_s] >= low) & (record[key.to_s] <= high)
+                else
+                  value = params[key].to_i
+                  record[key.to_s].eq(value)
+                end
+              end
+            end
+          ).compact.foldl{ |a,b| a & b }
+        end
+      end
+
+    end
+  end
+end

data/lib/wordtree/disk/librarian.rb

@@ -0,0 +1,62 @@
+require 'preamble'
+require 'wordtree/book'
+require 'wordtree/disk/library'
+require 'wordtree/archdown'
+
+module WordTree
+  module Disk
+    class Librarian
+      attr_reader :library
+
+      def initialize(library)
+        @library = library
+      end
+
+      def find(book_id)
+        begin
+          retrieved = Preamble.load(library.path_to(book_id))
+          Book.create(book_id, retrieved.metadata, retrieved.content)
+        rescue Errno::ENOENT
+          nil
+        end
+      end
+
+      def save(book)
+        library.mkdir(book.id)
+        Preamble.new(book.metadata, book.content || "").save(library.path_to(book.id))
+      end
+
+      def archive_org_get(*book_ids, &block)
+        book_ids.map do |book_id|
+          archive_org_get_with_conditions(identifier: book_id, &block)
+        end.flatten(1)
+      end
+
+      def archive_org_get_range_of_years(start_year, end_year, &block)
+        archive_org_get_with_conditions({
+          :start_year => start_year,
+          :end_year   => end_year
+        }, &block)
+      end
+
+      # Downloads a set of books to the on-disk library and
+      # returns a list of book_ids
+      def archive_org_get_with_conditions(conditions, &block)
+        archdown = Archdown.new
+        [].tap do |archive_org_ids|
+          archdown.download_all(conditions) do |metadata, content, failure|
+            if failure
+              #TODO: logging
+              $stderr.puts "Unable to download from archive.org: #{failure}"
+            else
+              book = Book.create(metadata["archive_org_id"], metadata, content)
+              save(book)
+              yield book, self if block_given?
+              archive_org_ids << book.id
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wordtree/disk/library.rb

@@ -0,0 +1,43 @@
+require 'fileutils'
+
+require 'wordtree/archdown'
+require 'wordtree/disk/library_locator'
+
+module WordTree
+  module Disk
+    class Library
+
+      FILE_TYPES = {
+        :raw => "%s.md"
+      }
+
+      # The file path to the root of the library directory, e.g. /data/library
+      attr_reader :root
+
+      def initialize(root)
+        @root = root
+      end
+
+      # returns the full path of a book's subdirectory within the library
+      # Accepts either a String or a LibraryLocator object
+      def dir_of(book_id)
+        File.expand_path(LibraryLocator.identity(book_id).relpath, root)
+      end
+
+      def path_to(book_id, type=:raw)
+        File.join(dir_of(book_id), file_type(book_id, type))
+      end
+
+      def file_type(book_id, type=:raw)
+        locator = LibraryLocator.identity(book_id)
+        FILE_TYPES[type] % locator.id
+      end
+
+      # Create all subdirs up to the location where a book is stored
+      # Accepts either a String or a LibraryLocator object
+      def mkdir(book_id)
+        FileUtils.mkdir_p(dir_of(book_id))
+      end
+    end
+  end
+end

data/lib/wordtree/disk/library_locator.rb

@@ -0,0 +1,46 @@
+module WordTree
+  module Disk
+    # A class that converts from a book ID to a location within the library, e.g.
+    #
+    # "firstbooknapole00gruagoog"
+    #
+    #   becomes
+    #
+    # "fi/og/firstbooknapole00gruagoog/"
+    #
+    #   or, in context of the full path:
+    #
+    # [/data/library/] "fi/og/firstbooknapole00gruagoog/" [firstbooknapole00gruagoog.md]
+    #
+    class LibraryLocator
+      # The book ID to locate
+      attr_reader :id
+
+      # Construct a LibraryLocator from a string (book ID)
+      def initialize(id)
+        @id = id
+      end
+
+      def first
+        @id[0..1].downcase
+      end
+
+      def last
+        @id[-2..-1].downcase
+      end
+
+      # Returns a "relative" path to be joined to the library root,
+      # e.g. if the identifier is "firstbooknapole00gruagoog", then relpath
+      # should return "fi/og/firstbooknapole00gruagoog", i.e. probably later to
+      # become something like "/data/library/fi/og/firstbooknapole00gruagoog" 
+      def relpath
+        File.join(first, last, @id)
+      end
+
+      # Constructor that is as willing to use a String as it is a LibraryLocator
+      def self.identity(id)
+        id.is_a?(LibraryLocator) ? id : new(id)
+      end
+    end
+  end
+end

data/lib/wordtree/version.rb

@@ -1,3 +1,3 @@
 module Wordtree
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

data/lib/wordtree.rb

@@ -1,6 +1,8 @@
 require "wordtree/version"
 require "wordtree/book"
 
-require "wordtree/library"
-require "wordtree/librarian"
-require "wordtree/library_locator"
+require "wordtree/db/librarian"
+
+require "wordtree/disk/library"
+require "wordtree/disk/librarian"
+require "wordtree/disk/library_locator"

data/spec/spec_helper.rb

@@ -12,6 +12,12 @@ module Fixtures
   end
 end
 
+RDB_CONFIG = {
+  :host => "localhost",
+  :port => 28015,
+  :db   => "test"
+}
+
 VCR.configure do |c|
   include Fixtures
   c.cassette_library_dir = fixture('cassettes')

data/spec/wordtree/book_spec.rb

@@ -14,6 +14,6 @@ describe WordTree::Book do
 
   it "can return cleaned content" do
     book = WordTree::Book.create("book", {}, "Wi&ld\nContent!")
-    expect(book.clean_content).to eq("wild content\n")
+    expect(book.content_clean).to eq("wild content\n")
   end
 end

data/spec/wordtree/db/librarian_spec.rb

@@ -0,0 +1,58 @@
+require_relative '../../spec_helper'
+require 'tmpdir'
+require 'preamble'
+require 'wordtree/db/librarian'
+
+describe WordTree::DB::Librarian do
+  it "instantiates" do
+    WordTree::DB::Librarian.new("connection")
+  end
+
+  context "with rethinkdb connection" do
+    let(:r) { RethinkDB::RQL.new }
+    let(:rdb) { r.connect(
+      :host => RDB_CONFIG[:host],
+      :port => RDB_CONFIG[:port],
+      :db   => RDB_CONFIG[:db]
+    ) }
+
+    let(:librarian) { WordTree::DB::Librarian.new(rdb) }
+
+    before(:each) do
+      begin
+        r.table_drop('books').run(rdb)
+      rescue RethinkDB::RqlRuntimeError
+      ensure
+        r.table_create('books').run(rdb)
+      end
+    end
+
+    describe "#find" do
+      it "returns nil if book doesn't exist" do
+        book = librarian.find("abc")
+        expect(book).to be_nil
+      end
+
+      it "finds a book" do
+        r.table('books').insert(
+          :id => "firstbooknapole00gruagoog",
+          :year => 1809).run(rdb)
+        book = librarian.find("firstbooknapole00gruagoog")
+        expect(book).to be_a(WordTree::Book)
+      end
+    end
+
+    describe "#save" do
+      it "saves a book to the db" do
+        book = WordTree::Book.create('test', {:year => 1800}, "body")
+        librarian.save(book)
+        result = r.table('books').run(rdb).to_a
+        expect(result).to eq([{
+          "id" => "test",
+          "simhash" => 1318950168412674304,
+          "size_bytes" => 4,
+          "year" => 1800}])
+      end
+    end
+  end
+end

data/spec/wordtree/disk/librarian_spec.rb

@@ -0,0 +1,60 @@
+require_relative '../../spec_helper'
+require 'tmpdir'
+require 'preamble'
+require 'wordtree/disk/librarian'
+
+describe WordTree::Disk::Librarian do
+  let(:root) { Dir.mktmpdir }
+  let(:library) { WordTree::Disk::Library.new(root) }
+  let(:librarian) { WordTree::Disk::Librarian.new(library) }
+
+  it "downloads an archive.org book" do
+    VCR.use_cassette('archive_org_download_book') do
+      librarian.archive_org_get("firstbooknapole00gruagoog")
+      book = librarian.find("firstbooknapole00gruagoog")
+      expect(book.year).to eq(1809)
+    end
+  end
+
+  context "with fixture library" do
+    # Need a read-only library with fixtures in it for some tests
+    let(:root) { fixture("library") }
+
+    describe "#find" do
+      it "returns nil if the book is not found" do
+        book = librarian.find("nobook")
+        expect(book).to be_nil
+      end
+
+      it "loads book from disk" do
+        book = librarian.find("book")
+        expect(book.id).to eq("book")
+        expect(book.year).to eq(1800)
+        expect(book.content).to eq("Book with content")
+      end
+    end
+
+    it "saves to disk (yaml, content)" do
+      tmp_root = Dir.mktmpdir
+      tmp_library = WordTree::Disk::Library.new(tmp_root)
+      tmp_librarian = WordTree::Disk::Librarian.new(tmp_library)
+
+      book = librarian.find("book")
+
+      book.source = "test"
+      book.content += "."
+
+      tmp_librarian.save(book)
+
+      updated = Preamble.load(tmp_library.path_to("book"))
+      expect(updated.metadata).to eq(
+        :id => "book",
+        :year => 1800,
+        :source => "test",
+        :simhash => 14921967289891934128,
+        :size_bytes => 17)
+      expect(updated.content).to eq("Book with content.")
+    end
+
+  end
+end

data/spec/wordtree/{library_spec.rb → disk/library_spec.rb}

@@ -1,10 +1,10 @@
 require 'spec_helper'
-require 'wordtree/library'
+require 'wordtree/disk/library'
 require 'tmpdir'
 
-describe WordTree::Library do
+describe WordTree::Disk::Library do
   let(:root) { Dir.mktmpdir('library') }
-  let(:library) { WordTree::Library.new(root) }
+  let(:library) { WordTree::Disk::Library.new(root) }
 
   it "initializes with path" do
     expect(library.root).to eq(root)

data/wordtree.gemspec

@@ -22,6 +22,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "preamble", ">= 0.0.3"
   spec.add_dependency "archivist-client", ">= 0.1.7"
   spec.add_dependency "retriable"
+  spec.add_dependency "simhash", "0.2.5"
+  spec.add_dependency "rethinkdb", "~> 1.13.0"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wordtree
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-17 00:00:00.000000000 Z
+date: 2014-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: virtus
@@ -75,6 +75,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simhash
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.5
+- !ruby/object:Gem::Dependency
+  name: rethinkdb
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.13.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -220,17 +252,19 @@ files:
 - lib/wordtree.rb
 - lib/wordtree/archdown.rb
 - lib/wordtree/book.rb
-- lib/wordtree/librarian.rb
-- lib/wordtree/library.rb
-- lib/wordtree/library_locator.rb
+- lib/wordtree/db/librarian.rb
+- lib/wordtree/disk/librarian.rb
+- lib/wordtree/disk/library.rb
+- lib/wordtree/disk/library_locator.rb
 - lib/wordtree/text_utils.rb
 - lib/wordtree/version.rb
 - spec/fixtures/cassettes/archive_org_download_book.yml
 - spec/fixtures/library/bo/ok/book/book.md
 - spec/spec_helper.rb
 - spec/wordtree/book_spec.rb
-- spec/wordtree/librarian_spec.rb
-- spec/wordtree/library_spec.rb
+- spec/wordtree/db/librarian_spec.rb
+- spec/wordtree/disk/librarian_spec.rb
+- spec/wordtree/disk/library_spec.rb
 - spec/wordtree/text_utils_spec.rb
 - wordtree.gemspec
 homepage: ''
@@ -263,7 +297,8 @@ test_files:
 - spec/fixtures/library/bo/ok/book/book.md
 - spec/spec_helper.rb
 - spec/wordtree/book_spec.rb
-- spec/wordtree/librarian_spec.rb
-- spec/wordtree/library_spec.rb
+- spec/wordtree/db/librarian_spec.rb
+- spec/wordtree/disk/librarian_spec.rb
+- spec/wordtree/disk/library_spec.rb
 - spec/wordtree/text_utils_spec.rb
 has_rdoc: 

data/lib/wordtree/librarian.rb

@@ -1,58 +0,0 @@
-require 'preamble'
-require 'wordtree/book'
-require 'wordtree/library'
-require 'wordtree/archdown'
-
-module WordTree
-  class Librarian
-    attr_reader :library
-
-    def initialize(library)
-      @library = library
-    end
-
-    def find(book_id)
-      retrieved = Preamble.load(library.path_to(book_id))
-      Book.create(book_id, retrieved.metadata, retrieved.content)
-    end
-
-    def save(book)
-      library.mkdir(book.id)
-      Preamble.new(book.metadata, book.content).save(library.path_to(book.id))
-    end
-
-    def archive_org_get(*book_ids, &block)
-      book_ids.map do |book_id|
-        archive_org_get_with_conditions(identifier: book_id, &block)
-      end.flatten(1)
-    end
-
-    def archive_org_get_range_of_years(start_year, end_year, &block)
-      archive_org_get_with_conditions({
-        :start_year => start_year,
-        :end_year   => end_year
-      }, &block)
-    end
-
-    # Downloads a set of books to the on-disk library and
-    # returns a list of book_ids
-    def archive_org_get_with_conditions(conditions, &block)
-      archdown = Archdown.new
-      [].tap do |archive_org_ids|
-        archdown.download_all(conditions) do |metadata, content, failure|
-          if failure
-            #TODO: logging
-            $stderr.puts "Unable to download from archive.org: #{failure}"
-          else
-            book = Book.create(metadata["archive_org_id"], metadata, content)
-            save(book)
-            yield book, self if block_given?
-            archive_org_ids << book.id
-          end
-        end
-      end
-    end
-
-
-  end
-end

data/lib/wordtree/library.rb

@@ -1,41 +0,0 @@
-require 'fileutils'
-
-require 'wordtree/archdown'
-require 'wordtree/library_locator'
-
-module WordTree
-  class Library
-
-    FILE_TYPES = {
-      :raw => "%s.md"
-    }
-
-    # The file path to the root of the library directory, e.g. /data/library
-    attr_reader :root
-
-    def initialize(root)
-      @root = root
-    end
-
-    # returns the full path of a book's subdirectory within the library
-    # Accepts either a String or a LibraryLocator object
-    def dir_of(book_id)
-      File.expand_path(LibraryLocator.identity(book_id).relpath, root)
-    end
-
-    def path_to(book_id, type=:raw)
-      File.join(dir_of(book_id), file_type(book_id, type))
-    end
-
-    def file_type(book_id, type=:raw)
-      locator = LibraryLocator.identity(book_id)
-      FILE_TYPES[type] % locator.id
-    end
-
-    # Create all subdirs up to the location where a book is stored
-    # Accepts either a String or a LibraryLocator object
-    def mkdir(book_id)
-      FileUtils.mkdir_p(dir_of(book_id))
-    end
-  end
-end

data/lib/wordtree/library_locator.rb

@@ -1,44 +0,0 @@
-module WordTree
-  # A class that converts from a book ID to a location within the library, e.g.
-  #
-  # "firstbooknapole00gruagoog"
-  #
-  #   becomes
-  #
-  # "fi/og/firstbooknapole00gruagoog/"
-  #
-  #   or, in context of the full path:
-  #
-  # [/data/library/] "fi/og/firstbooknapole00gruagoog/" [firstbooknapole00gruagoog.md]
-  #
-  class LibraryLocator
-    # The book ID to locate
-    attr_reader :id
-
-    # Construct a LibraryLocator from a string (book ID)
-    def initialize(id)
-      @id = id
-    end
-
-    def first
-      @id[0..1].downcase
-    end
-
-    def last
-      @id[-2..-1].downcase
-    end
-
-    # Returns a "relative" path to be joined to the library root,
-    # e.g. if the identifier is "firstbooknapole00gruagoog", then relpath
-    # should return "fi/og/firstbooknapole00gruagoog", i.e. probably later to
-    # become something like "/data/library/fi/og/firstbooknapole00gruagoog" 
-    def relpath
-      File.join(first, last, @id)
-    end
-
-    # Constructor that is as willing to use a String as it is a LibraryLocator
-    def self.identity(id)
-      id.is_a?(LibraryLocator) ? id : new(id)
-    end
-  end
-end

data/spec/wordtree/librarian_spec.rb

@@ -1,48 +0,0 @@
-require_relative '../spec_helper'
-require 'tmpdir'
-require 'preamble'
-require 'wordtree/librarian'
-
-describe WordTree::Librarian do
-  let(:root) { Dir.mktmpdir }
-  let(:library) { WordTree::Library.new(root) }
-  let(:librarian) { WordTree::Librarian.new(library) }
-
-  it "downloads an archive.org book" do
-    VCR.use_cassette('archive_org_download_book') do
-      librarian.archive_org_get("firstbooknapole00gruagoog")
-      book = librarian.find("firstbooknapole00gruagoog")
-      expect(book.year).to eq(1809)
-    end
-  end
-
-  context "with fixture library" do
-    # Need a read-only library with fixtures in it for some tests
-    let(:root) { fixture("library") }
-
-    it "loads book from disk" do
-      book = librarian.find("book")
-      expect(book.id).to eq("book")
-      expect(book.year).to eq(1800)
-      expect(book.content).to eq("Book with content")
-    end
-
-    it "saves to disk (yaml, content)" do
-      tmp_root = Dir.mktmpdir
-      tmp_library = WordTree::Library.new(tmp_root)
-      tmp_librarian = WordTree::Librarian.new(tmp_library)
-
-      book = librarian.find("book")
-
-      book.source = "test"
-      book.content += "."
-
-      tmp_librarian.save(book)
-
-      updated = Preamble.load(tmp_library.path_to("book"))
-      expect(updated.metadata).to eq({:year => 1800, :source => "test"})
-      expect(updated.content).to eq("Book with content.")
-    end
-
-  end
-end