archdown 0.3

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,107 @@
+PATH
+  remote: .
+  specs:
+    archdown (0.3)
+      archivist-client
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    archivist-client (0.1.6)
+      faraday
+      faraday_middleware
+      representable
+      virtus (~> 1.0.0)
+    axiom-types (0.1.1)
+      descendants_tracker (~> 0.0.4)
+      ice_nine (~> 0.11.0)
+      thread_safe (~> 0.3, >= 0.3.1)
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    coderay (1.1.0)
+    coercible (1.0.0)
+      descendants_tracker (~> 0.0.1)
+    columnize (0.8.9)
+    debugger (1.6.8)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.3.5)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.3.5)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.2.5)
+    equalizer (0.0.9)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.9.1)
+      faraday (>= 0.7.4, < 0.10)
+    ffi (1.9.3)
+    formatador (0.2.5)
+    guard (2.6.1)
+      formatador (>= 0.2.4)
+      listen (~> 2.7)
+      lumberjack (~> 1.0)
+      pry (>= 0.9.12)
+      thor (>= 0.18.1)
+    guard-rspec (4.2.10)
+      guard (~> 2.1)
+      rspec (>= 2.14, < 4.0)
+    ice_nine (0.11.0)
+    listen (2.7.9)
+      celluloid (>= 0.15.2)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    lumberjack (1.0.9)
+    method_source (0.8.2)
+    mini_portile (0.6.0)
+    multi_json (1.10.1)
+    multipart-post (2.0.0)
+    nokogiri (1.6.2.1)
+      mini_portile (= 0.6.0)
+    pry (0.10.0)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.3.2)
+    rb-fsevent (0.9.4)
+    rb-inotify (0.9.5)
+      ffi (>= 0.5.0)
+    representable (1.8.5)
+      multi_json
+      nokogiri
+      uber
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.2)
+    slop (3.5.0)
+    thor (0.19.1)
+    thread_safe (0.3.4)
+    timers (1.1.0)
+    uber (0.0.7)
+    virtus (1.0.2)
+      axiom-types (~> 0.1)
+      coercible (~> 1.0)
+      descendants_tracker (~> 0.0.3)
+      equalizer (~> 0.0.9)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  archdown!
+  bundler (~> 1.3)
+  debugger
+  guard
+  guard-rspec
+  rake
+  rspec

data/Guardfile

@@ -0,0 +1,8 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 WordTree Foundation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,16 @@
+archdown
+========
+
+Command-line utility to download books from archive.org using archivist-client
+
+Example Usage
+-------------
+
+```sh
+$ bundle install
+$ bundle exec ruby archdown.rb -y 1700-1830
+
+# ...
+# downloads a lot of books into ./library
+# ...
+```

data/bin/archdown

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require "archdown"
+require 'optparse'
+
+# download books (text files) from archive.org
+
+options = {
+  :library_root => './library',
+  :start_year => 1700,
+  :end_year   => 1700
+}
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: archdown [options]"
+
+  opts.on("-l", "--library PATH", "Store files in PATH") do |lib|
+    options[:library_root] = lib
+  end
+
+  opts.on("-y", "--year RANGE", "Query for books published within RANGE years, e.g. 1700-1750") do |range|
+    options[:start_year] = range.split('-').first.to_i
+    options[:end_year] = range.split('-').last.to_i
+  end
+end
+parser.parse!
+
+if ARGV.empty?
+  download = Archdown::Download.new(
+    options[:library_root],
+    :start_year => options[:start_year],
+    :end_year   => options[:end_year])
+  download.go! do |metadata, librarian|
+    puts metadata.to_yaml
+  end
+else
+  puts parser
+  puts
+  p options, ARGV
+end
+
+
+

data/lib/archdown/download.rb

@@ -0,0 +1,26 @@
+require 'archivist/client'
+
+require 'archdown/librarian'
+require 'archdown/library'
+
+module Archdown
+  class Download
+    attr_reader :library_root, :search_terms
+
+    def initialize(library_root, search_terms)
+      @library = Library.new(library_root)
+      @search_terms = search_terms
+      @client = Archivist::Client::Base.new
+    end
+
+    def go!(&each_book)
+      page = 1
+      while not (books = @client.search(@search_terms.merge(:page => page))).empty?
+        books.each do |book|
+          Librarian.new(@library, book).store_book(&each_book)
+        end
+        page += 1
+      end
+    end
+  end
+end

data/lib/archdown/librarian.rb

@@ -0,0 +1,64 @@
+require 'fileutils'
+require 'yaml'
+
+require 'archivist/client'
+
+module Archdown
+  # The Librarian takes a book and puts it in the library
+  class Librarian
+    attr_reader :library, :book
+
+    def initialize(library, book)
+      @library = library
+      @book = book
+      @failure = nil
+    end
+
+    def metadata
+      {
+        'title'  => @book.title,
+        'author' => @book.creators ? @book.creators.join(';') : nil,
+        'year'   => @book.date.year,
+        'source' => "http://archive.org/details/#{@book.identifier}",
+        'status' => "OCR ONLY",
+        'archive_org_id' => @book.identifier,
+      }.tap do |meta|
+        meta['failure'] = @failure if @failure
+      end
+    end
+
+    def book_dir
+      @library.path_for_identifier(@book.identifier)
+    end
+
+    def book_filepath
+      File.join(book_dir, @book.identifier + '.md')
+    end
+
+    def make_book_dir
+      FileUtils.mkdir_p(book_dir)
+    end
+
+    def store_book(&block)
+      make_book_dir
+
+      begin
+        text = @book.download
+      rescue Archivist::Model::Document::UnsupportedFormat => e
+        @failure = e.to_s
+      rescue StandardError => e
+        @failure = e.to_s
+      end
+
+      yield metadata, self if block_given?
+
+      content = metadata.to_yaml
+      content += "---\n"
+      content += text if text
+
+      File.open(book_filepath, "w") do |file|
+        file.write content
+      end
+    end
+  end
+end

data/lib/archdown/library.rb

@@ -0,0 +1,22 @@
+module Archdown
+  # The Library knows about its locations for books (e.g. file directory)
+  class Library
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    def first(identifier)
+      identifier[0..1].downcase
+    end
+
+    def last(identifier)
+      identifier[-2..-1].downcase
+    end
+
+    def path_for_identifier(identifier, *parts)
+      File.join(path, first(identifier), last(identifier), identifier, *parts)
+    end
+  end
+end

data/lib/archdown/version.rb

@@ -0,0 +1,3 @@
+module Archdown
+  VERSION = '0.3'
+end

data/lib/archdown.rb

@@ -0,0 +1,3 @@
+require 'archdown/library'
+require 'archdown/librarian'
+require 'archdown/download'

data/spec/archdown/library_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+require 'archdown/library'
+
+describe Archdown::Library do
+  let(:library) { Archdown::Library.new("/tmp/library") }
+
+  it "initializes" do
+    expect{ library }.to_not raise_error
+  end
+
+  it "gets first 2 chars" do
+    expect(library.first('happy')).to eq('ha')
+  end
+
+  it "gets last 2 chars" do
+    expect(library.last('happy')).to eq('py')
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,25 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+
+module Fixtures
+  def fixture(name)
+    File.join(File.dirname(__FILE__), 'fixtures', name)
+  end
+end
+
+RSpec.configure do |config|
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  config.include Fixtures
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,175 @@
+--- !ruby/object:Gem::Specification
+name: archdown
+version: !ruby/object:Gem::Version
+  version: '0.3'
+  prerelease: 
+platform: ruby
+authors:
+- Duane Johnson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: archivist-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Archive.org download library with command-line tool
+email:
+- duane.johnson@gmail.com
+executables:
+- archdown
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- Guardfile
+- LICENSE
+- README.md
+- bin/archdown
+- lib/archdown.rb
+- lib/archdown/download.rb
+- lib/archdown/librarian.rb
+- lib/archdown/library.rb
+- lib/archdown/version.rb
+- spec/archdown/library_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Archive.org download library
+test_files:
+- spec/archdown/library_spec.rb
+- spec/spec_helper.rb
+has_rdoc: