wordtree 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in wordtree.gemspec
+gemspec
+
+# gem "archivist-client", :path => "../archivist-client"

data/Guardfile

@@ -0,0 +1,8 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec, :cmd => 'bundle exec rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Duane Johnson
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# Wordtree
+
+This is the WordTree ruby gem for text analysis.
+
+## Installation
+
+    $ gem install wordtree
+
+## Usage
+
+    require 'wordtree'
+
+    library = WordTree::Library.new("/tmp/library")
+    librarian = WordTree::Librarian.new(library)
+
+Find a book in your on-disk "library":
+
+    book = librarian.find('firstbooknapole00gruagoog')
+    book.metadata
+    book.content
+
+Modify and save a book to your "library":
+
+    book.year = 2014
+    librarian.save(book)
+
+Download a book from Archive.org to your "library":
+
+    book_id = librarian.archive_org_get('latewarbetween_00hunt')
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/wordtree/archdown.rb

@@ -0,0 +1,58 @@
+require 'archivist/client'
+require 'retriable'
+
+module WordTree
+  class Archdown
+    attr_reader :client
+
+    def initialize
+      @client = Archivist::Client::Base.new
+    end
+
+    def metadata_for(archivist_book)
+      author = archivist_book.creators ? archivist_book.creators.join('; ') : nil
+      {
+        'title'          => archivist_book.title,
+        'author'         => author,
+        'year'           => archivist_book.date.year,
+        'source'         => "http://archive.org/details/#{archivist_book.identifier}",
+        'status'         => "OCR ONLY",
+        'archive_org_id' => archivist_book.identifier,
+      }
+    end
+
+    def content_for(archivist_book)
+      [archivist_book.download, nil]
+    rescue Archivist::Model::Document::UnsupportedFormat => e
+      [nil, e.to_s]
+    rescue StandardError => e
+      [nil, e.to_s]
+    end
+
+    def download_all(search_terms, &each_book)
+      page = 1
+      loop do
+        archivist_books =
+          ::Retriable.retriable(:on => Faraday::Error::TimeoutError) do
+            @client.search(search_terms.merge(:page => page))
+          end
+  
+        break if archivist_books.empty?
+  
+        archivist_books.each do |archivist_book|
+          download(archivist_book, &each_book)
+        end
+
+        page += 1
+      end
+    end
+
+    def download(archivist_book, &block)
+      metadata = metadata_for(archivist_book)
+      content, failure = content_for(archivist_book)
+
+      yield metadata, content, failure if block_given?
+    end
+
+  end
+end

data/lib/wordtree/book.rb

@@ -0,0 +1,38 @@
+require 'virtus'
+require 'wordtree/text_utils'
+
+module WordTree
+  class Book
+    include Virtus.model
+
+    attribute :id, String, :default => :default_id
+    attribute :archive_org_id, String
+    attribute :title, String
+    attribute :author, String
+    attribute :year, Integer
+    attribute :source, String
+    attribute :status, String
+    # Size of the content in bytes
+    attribute :size_bytes, Integer
+    # A simhash (locality-sensitive hash) of the content
+    attribute :simhash, String
+
+    attribute :content, String
+
+    def self.create(id, metadata, content)
+      new(metadata.merge("id" => id, "content" => content))
+    end
+
+    def default_id
+      archive_org_id
+    end
+
+    def metadata
+      attributes.select{ |k,v| !v.nil? && k != :content && k != :id }
+    end
+
+    def clean_content
+      TextUtils.clean_text(content)
+    end
+  end
+end

data/lib/wordtree/librarian.rb

@@ -0,0 +1,58 @@
+require 'preamble'
+require 'wordtree/book'
+require 'wordtree/library'
+require 'wordtree/archdown'
+
+module WordTree
+  class Librarian
+    attr_reader :library
+
+    def initialize(library)
+      @library = library
+    end
+
+    def find(book_id)
+      retrieved = Preamble.load(library.path_to(book_id))
+      Book.create(book_id, retrieved.metadata, retrieved.content)
+    end
+
+    def save(book)
+      library.mkdir(book.id)
+      Preamble.new(book.metadata, book.content).save(library.path_to(book.id))
+    end
+
+    def archive_org_get_book(book_id, &block)
+      archive_org_get({
+        :filters => ["identifier:#{book_id}"]
+      }, &block)
+    end
+
+    def archive_org_get_range_of_years(start_year, end_year, &block)
+      archive_org_get({
+        :start_year => start_year,
+        :end_year   => end_year
+      }, &block)
+    end
+
+    # Downloads a set of books to the on-disk library and
+    # returns a list of book_ids
+    def archive_org_get(conditions, &block)
+      archdown = Archdown.new
+      [].tap do |archive_org_ids|
+        archdown.download_all(conditions) do |metadata, content, failure|
+          if failure
+            #TODO: logging
+            $stderr.puts "Unable to download from archive.org: #{failure}"
+          else
+            book = Book.create(metadata["archive_org_id"], metadata, content)
+            save(book)
+            yield book, self if block_given?
+            archive_org_ids << book.id
+          end
+        end
+      end
+    end
+
+
+  end
+end

data/lib/wordtree/library.rb

@@ -0,0 +1,41 @@
+require 'fileutils'
+
+require 'wordtree/archdown'
+require 'wordtree/library_locator'
+
+module WordTree
+  class Library
+
+    FILE_TYPES = {
+      :raw => "%s.md"
+    }
+
+    # The file path to the root of the library directory, e.g. /data/library
+    attr_reader :root
+
+    def initialize(root)
+      @root = root
+    end
+
+    # returns the full path of a book's subdirectory within the library
+    # Accepts either a String or a LibraryLocator object
+    def dir_of(book_id)
+      File.expand_path(LibraryLocator.identity(book_id).relpath, root)
+    end
+
+    def path_to(book_id, type=:raw)
+      File.join(dir_of(book_id), file_type(book_id, type))
+    end
+
+    def file_type(book_id, type=:raw)
+      locator = LibraryLocator.identity(book_id)
+      FILE_TYPES[type] % locator.id
+    end
+
+    # Create all subdirs up to the location where a book is stored
+    # Accepts either a String or a LibraryLocator object
+    def mkdir(book_id)
+      FileUtils.mkdir_p(dir_of(book_id))
+    end
+  end
+end

data/lib/wordtree/library_locator.rb

@@ -0,0 +1,44 @@
+module WordTree
+  # A class that converts from a book ID to a location within the library, e.g.
+  #
+  # "firstbooknapole00gruagoog"
+  #
+  #   becomes
+  #
+  # "fi/og/firstbooknapole00gruagoog/"
+  #
+  #   or, in context of the full path:
+  #
+  # [/data/library/] "fi/og/firstbooknapole00gruagoog/" [firstbooknapole00gruagoog.md]
+  #
+  class LibraryLocator
+    # The book ID to locate
+    attr_reader :id
+
+    # Construct a LibraryLocator from a string (book ID)
+    def initialize(id)
+      @id = id
+    end
+
+    def first
+      @id[0..1].downcase
+    end
+
+    def last
+      @id[-2..-1].downcase
+    end
+
+    # Returns a "relative" path to be joined to the library root,
+    # e.g. if the identifier is "firstbooknapole00gruagoog", then relpath
+    # should return "fi/og/firstbooknapole00gruagoog", i.e. probably later to
+    # become something like "/data/library/fi/og/firstbooknapole00gruagoog" 
+    def relpath
+      File.join(first, last, @id)
+    end
+
+    # Constructor that is as willing to use a String as it is a LibraryLocator
+    def self.identity(id)
+      id.is_a?(LibraryLocator) ? id : new(id)
+    end
+  end
+end

data/lib/wordtree/text_utils.rb

@@ -0,0 +1,73 @@
+module WordTree
+  module TextUtils
+    def self.split_near(text, split_index)
+      if split_index >= text.size
+        return [text, ""]
+      else
+        index = split_index
+        while index >= 0
+          if text[index] == ' '
+            return [text[0...index], text[(index+1)..-1]]
+          end
+          index -= 1
+        end
+        return [text[0...split_index], text[split_index..-1]]
+      end
+    end
+
+    # Remove punctuation an non-alphabetical characters from a text, and return
+    # a cleaned-up version wrapped at +wrap+ characters per line.
+    def self.clean_text(input, wrap=120)
+      join = nil
+      output = String.new
+      output_line = String.new
+
+      # Ignore non-UTF-8 characters
+      input = input.encode('UTF-8', :invalid => :replace, :undef => :replace).downcase
+
+      _0 = '0'.ord
+      _9 = '9'.ord
+      _a = 'a'.ord
+      _z = 'z'.ord
+      _A = 'A'.ord
+      _Z = 'Z'.ord
+      _dash = '-'.ord
+      _space = ' '.ord
+      _newline = "\n".ord
+
+      join_lines = false
+      just_added_space = false
+      line_length = 0
+      input.each_char do |c|
+        c = c.ord
+        # Change upper-case to lower-case
+        c -= 32 if (c >= _A && c <= _Z)
+        # Change newlines to spaces
+        c = _space if c == _newline
+
+        if c == _dash
+          # In case of a dash, set the scoop-spaces-up flag
+          join_lines = true
+        elsif join_lines && (c == _space)
+          # ignore
+        elsif (c >= _a && c <= _z) || (c == _space && !just_added_space)
+          # Add letters and spaces
+          output << c.chr
+          line_length += 1
+          just_added_space = (c == _space)
+          join_lines = false
+        end
+      end
+
+      wrapped_output = String.new
+      begin
+        output_line, remainder = split_near(output, wrap)
+        wrapped_output << output_line + "\n"
+        output = remainder
+      end while remainder.size > wrap
+      wrapped_output << remainder + "\n" unless remainder.empty?
+
+      return wrapped_output
+    end
+  end
+end

data/lib/wordtree/version.rb

@@ -0,0 +1,3 @@
+module Wordtree
+  VERSION = "0.0.1"
+end

data/lib/wordtree.rb

@@ -0,0 +1,5 @@
+require "wordtree/version"
+
+module Wordtree
+  # Your code goes here...
+end