bomdb 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 209c584dfb7167f1a15fb8c56c5f3875d5e38542
-  data.tar.gz: 4916437d8203e3d2358936f747e4fe05b0a91b24
+  metadata.gz: e9e5b23505580c0871d836fffd14b2355a636562
+  data.tar.gz: 7c31fa7627d57d7f6a179d21ad8ccab40dcc108d
 SHA512:
-  metadata.gz: fd1ddabc09baac5422810257ec93ed3133bbc90d6bc952602f97da6dcdb0f62f169423711d04ce39cb970cc88cf9358a6105f7d103c23c7f85ecd8673d9d6b15
-  data.tar.gz: 38e2f20ece2af48daa61360b671f3b9d3a2b760b0757f20eb9bab6e22efa2a3185e3ba25aec4b02b9153abb08e6f19d0ee8cba785fa346ccd4c9218f3be0e838
+  metadata.gz: 69b236b5a23555654d6c2b311c7acd9f0c7767ff902957a50a0ed91db8c4c231b28e058e191f698ca840afbed8da02738cb3ffe9cc12d37cff009b71552f415b
+  data.tar.gz: f3bff6acf0729645bbd90677a229db21db19fe59ea2adec262bd086318b16adf3e917ce503244b6f0cf6effb95d9a6c8b1b6c45ab91c8b1b9355ffaac4483a1d

data/.gitignore

@@ -1,13 +1,3 @@
 /.bundle/
-/.yardoc
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-*.bundle
-*.so
-*.o
-*.a
-mkmf.log
+/reference/
+.DS_Store

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    bomdb (0.0.1)
+    bomdb (0.1.0)
+      colorize (~> 0.7)
       constellation (~> 0.1)
       sequel (~> 4.21)
       sqlite3 (~> 1.3)
@@ -12,6 +13,7 @@ GEM
   specs:
     byebug (4.0.5)
       columnize (= 0.9.0)
+    colorize (0.7.5)
     columnize (0.9.0)
     constellation (0.1.1)
       multi_json

data/README.md

@@ -4,6 +4,7 @@ This is a command-line tool (packaged as a Ruby gem) that provides multiple edit
 
 ## Usage
 
+### SHOW a formatted edition of the Book of Mormon
 Let's show the 1992 edition of the Book of Mormon:
 
 ```bash
@@ -26,7 +27,7 @@ Moroni 10:34 And now I bid unto all farewell. I soon go to rest in the paradise
 Suppose we want to remove the book, chapter, and verse headings from the output:
 
 ```bash
-$ bomdb show 1829 --no-book --no-chapter --no-verse
+$ bomdb show 1829 --no-verse
 
 I Nephi having been born of goodly parents, therefore I was taught somewhat in all the learning of my father...
 # ... etc ...
@@ -40,6 +41,29 @@ $ bomdb show 1829 --exclude Bible-OT
 # ... shows 6080 verses instead of the usual 6604
 ```
 
+### ALIGN a new Book of Mormon text file
+
+Suppose you have a new Book of Mormon text file that has been scanned from OCR or otherwise entered as a text file. It would take a lot of work to manually align all 6604 verses with the "standard" book, chapter, and verse numbers. Instead, BomDB helps automate this process:
+
+Given this text:
+
+```
+I, Nephi, having been born of goodly parents, therefore I was taught somewhat in all the learning of my father; and having seen many afflictions in the course of my days--nevertheless, having been highly favored of the Lord in all my days; yea, having had a great knowledge of the goodness and the mysteries of God, therefore I make a record of my proceedings in my days; yea, I make a record in the language of my father, which consists of the learning of the Jews and the language of the Egyptians. And I know that the record which I make is true; and I make it with mine own hand; and I make it according to my knowledge.
+```
+
+You can automatically align and annotate it:
+
+```bash
+$ bomdb align my_typed_bom.txt
+
+# preamble text skipped...
+1 Nephi 1:2 yea, I make a record in the language of my father, which consists of the learning of the Jews and the language of the Egyptians. 
+1 Nephi 1:3 And I know that the record which I make  is true; and I make it with mine own hand; and I make it according to my knowledge. 
+# ... shows 6601 more verses
+```
+
+Note that `align` requires the [dwdiff](http://linux.die.net/man/1/dwdiff) command on your system.
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/bomdb.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = "http://bomdb.wordtree.org"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0")
+  spec.files         = `git ls-files -z`.split("\x0").reject{ |f| f =~ %r|data/.*\.json| }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sqlite3',       '~> 1.3'
   spec.add_dependency 'thor',          '~> 0.19'
   spec.add_dependency 'constellation', '~> 0.1'
+  spec.add_dependency 'colorize',      '~> 0.7'
 
   # spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake',    '~> 10.0'

data/lib/bomdb.rb

@@ -15,13 +15,27 @@ module BomDB
   end
 end
 
+require 'byebug'
+
 require 'bomdb/version'
 require 'bomdb/config'
 require 'bomdb/schema'
 require 'bomdb/query'
 
+require 'bomdb/diff/dwdiff'
+require 'bomdb/diff/aligner'
+
 require 'bomdb/import/base'
 require 'bomdb/import/result'
 require 'bomdb/import/books'
 require 'bomdb/import/verses'
-require 'bomdb/import/biblical_refs'
+require 'bomdb/import/editions'
+require 'bomdb/import/contents'
+require 'bomdb/import/refs'
+
+require 'bomdb/export/base'
+require 'bomdb/export/result'
+require 'bomdb/export/books'
+require 'bomdb/export/verses'
+require 'bomdb/export/editions'
+require 'bomdb/export/contents'

data/lib/bomdb/cli/application.rb

@@ -1,128 +1,274 @@
 require 'thor'
 require 'bomdb'
+require 'colorize'
 
 module BomDB
   module Cli
     class Application < Thor
-      desc "import TYPE", "import data of TYPE into database, e.g. books"
-      option :reset, :type => :boolean, :default => false
-      def import(type, file=nil)
-        case type.downcase
-        when 'books'          
-          result = import_books(file, options[:reset])
-          show_result_and_maybe_exit(result)
-        when 'verses'
-          result = import_verses(file, options[:reset])
-          show_result_and_maybe_exit(result)
-        when 'biblical-refs'
-          result = import_biblical_refs(true)
-          show_result_and_maybe_exit(result)
+
+
+
+      desc "import FILE", "import data from FILE into database, e.g. books.json"
+      option :type,   :type => :string,  :default => nil
+      option :format, :type => :string,  :default => 'json'
+      def import(file)
+        type   = (options[:type]   || type_from_file(file)).downcase
+        format = (options[:format] || format_from_file(file)).downcase
+
+        importer =
+        case type
+        when 'books'    then BomDB::Import::Books.new(BomDB.db)
+        when 'verses'   then BomDB::Import::Verses.new(BomDB.db)
+        when 'editions' then BomDB::Import::Editions.new(BomDB.db)
+        when 'contents' then BomDB::Import::Contents.new(BomDB.db)
+        when 'refs'     then BomDB::Import::Refs.new(BomDB.db)
         else
           puts "Unknown import type #{type}"
           exit -1
         end
+
+        result = importer.import(read(file), format: format)
+        show_result_and_maybe_exit(result)
       end
 
-      desc "build", "[delete and re-]build the database"
-      option :delete, :type => :boolean, :default => false, :aliases => [:y]
-      def build
-        dbp = BomDB.config.db_path
-        if File.exist?(dbp) and !options[:delete]
-          puts "Database file '#{dbp}' exists. Delete? (y/N) "
+
+
+      desc "export TYPE", "export data from the database, e.g. books"
+      option :format, :type => :string,  :default => 'json'
+      option :editions, :type => :string, :default => :all
+      def export(type)
+        format = options[:format].downcase
+
+        exporter =
+        case type
+        when 'books'    then BomDB::Export::Books.new(BomDB.db)
+        when 'verses'   then BomDB::Export::Verses.new(BomDB.db)
+        when 'editions' then BomDB::Export::Editions.new(BomDB.db)
+        when 'contents' then BomDB::Export::Contents.new(BomDB.db, edition_prefixes: options[:editions])
+        # when 'refs'     then BomDB::Export::Refs.new(BomDB.db)
+        else
+          puts "Unknown import type #{type}"
+          exit -1
+        end
+
+        result = exporter.export(format: format)
+        if result.success?
+          puts result.to_s
+        else
+          show_result_and_maybe_exit(result)
+        end
+      end
+
+
+
+      desc "create", "create a new Book of Mormon database"
+      option :file, :type => :string, :default => BomDB.config.db_path,
+             :description => "the filepath of the database file on disk"
+      option :bare, :type => :boolean, :default => false,
+             :description => "create a bare database without any data"
+      option :delete, :type => :boolean, :default => false, :aliases => [:y],
+             :description => "don't ask for confirmation before overwriting an existing database"
+      def create
+        db_path = options[:file]
+
+        if File.exist?(db_path) and !options[:delete]
+          puts "Database file '#{db_path}' exists. Delete? (y/N) "
           if $stdin.gets.chomp.downcase != "y"
             puts "Exiting..."
             exit -1
           end
+          verbing = 'Overwriting'
+        else
+          verbing = 'Creating'
         end
 
-        # Delete the database
-        FileUtils.rm(dbp)
+        puts "#{verbing} database '#{db_path}' ..."
+        schema = BomDB::Schema.create(db_path)
+        puts "Created the following tables:"
+        schema.db.tables.each{ |t| puts "\t#{t}" }
 
-        puts "Importing books..."
-        show_result_and_maybe_exit(import_books(nil, true))
+        unless options[:bare]
+          puts "Importing books..."
+          import('books.json')
 
-        puts "Importing verses..."
-        show_result_and_maybe_exit(import_verses(nil, true))
+          puts "Importing verses..."
+          import('verses.json')
 
-        puts "Importing biblical refs..."
-        show_result_and_maybe_exit(import_biblical_refs(true))
+          puts "Importing editions..."
+          import('editions.json')
 
+          puts "Importing contents..."
+          import('contents.json')
+
+          puts "Importing refs..."
+          import('refs.json')
+        end
+
+        puts "Done."
       end
 
+
+
       desc "show EDITION RANGE", "show an edition of the Book of Mormon, or a RANGE of verses"
-      option :book,    :type => :boolean, :default => true, :aliases => [:b]
-      option :chapter, :type => :boolean, :default => true, :aliases => [:c]
-      option :verse,   :type => :boolean, :default => true, :aliases => [:v]
-      option :exclude, :type => :string, :aliases => [:x]
-      option :sep,     :type => :string,  :default => ' '
+      option :verse,   :type => :boolean, :default => true,
+             :description => "show book, chapter, verse annotations"
+      option :exclude, :type => :string, :aliases => [:x],
+             :description => "exclude verses that are references, e.g. Bible-OT references"
+      option :sep,     :type => :string,  :default => " ",
+             :description => "separator between annotations and content, defaults to ' '"
+      option :linesep, :type => :string,  :default => "\n",
+             :description => "separator between verses. Defaults to newline ('\\n')."
+      option :color,   :type => :boolean, :default => true,
+             :description => "show chapter and verse in color"
+      option :"for-alignment", :type => :boolean, :default => false,
+             :description => "show output in 'alignment' mode. Useful for debugging 'align' subcommand issues."
       def show(edition = '1829', range = nil)
-        query = BomDB::Query.new(
+        body_format = nil
+        if options[:"for-alignment"]
+          linesep = " "
+          verse_format = verse_format_for_alignment(options[:color])
+        else
+          linesep = options[:linesep]
+          if options[:verse]
+            if options[:color]
+              verse_format = lambda{ |b,c,v| b.colorize(:yellow) + ' ' + 
+                                             c.to_s.colorize(:green) + ':' + 
+                                             v.to_s.colorize(:light_green) }
+            else
+              verse_format = nil
+            end
+          else
+            verse_format = lambda{ |b,c,v| '' }
+          end
+        end
+        BomDB::Query.new(
           edition: edition,
           exclude: options[:exclude]
           # range: range
-        )
-        query.print(
-          book: options[:book],
-          chapter: options[:chapter],
-          verse: options[:verse],
-          sep: options[:sep]
+        ).print(
+          verse_format: verse_format,
+          body_format: body_format,
+          sep:     options[:sep],
+          linesep: linesep
         )
       end
 
+
+
       desc "editions", "list available editions of the Book of Mormon"
-      # option :available, :type => :boolean, :default => true
+      option :all, :type => :boolean, :default => false
       def editions
-        eds = BomDB.db[:versions].map do |r|
-          [r[:version_year], r[:version_name]].join(' -- ')
-        end
-        puts eds.join('\n')
+        eds = BomDB.db[:editions].
+          left_outer_join(:contents, :edition_id => :edition_id).
+          select_group(:edition_name).
+          select_append{ Sequel.as(count(:verse_id), :count) }.
+          order(:edition_name).
+          map { |r| "#{r[:count]} verses\t#{r[:edition_name]}" }
+        puts eds.join("\n")
       end
 
-      desc "reference-types", "list reference types"
-      def reference_types
-        rts = BomDB.db[:refs].distinct.select(:ref_name).map do |r|
-          r[:ref_name]
+
+
+      desc "references TYPE", "list reference types or references of TYPE"
+      def references(type=nil)
+        if type.nil?
+          rts = BomDB.db[:refs].
+            select_group(:ref_name).
+            select_append{ Sequel.as(count(ref_id), :count) }.
+            map { |r| "#{r[:ref_name]} (#{r[:count]} refs)" }
+          puts rts.join("\n")
+        else
+          rts = BomDB.db[:refs].where(:ref_name => type).
+            map{ |r| "#{r[:ref_book]} #{r[:ref_chapter]}:#{r[:ref_verse]}" }
+          puts rts.join("\n")
         end
-        puts rts.join('\n')
       end
 
-      private
 
-      def datafile(file)
-        File.join(BomDB.config.data_dir, file)
+
+      desc "align FILE EDITION", "give verse annotations from EDITION to a new Book of Mormon text FILE that lacks verse annotations"
+      option :dwdiff, :type => :string, :default => "/usr/local/bin/dwdiff",
+             :description => "the filepath of the dwdiff binary (shell command)"
+      option :'edition-only', :type => :boolean, :default => false,
+             :description => "show the alignment-formatted edition output only (useful for debugging)"
+      option :'diff-only', :type => :boolean, :default => false,
+             :description => "show the dwdiff output only (useful for debugging)"
+      def align(file, edition = '1829')
+        io = StringIO.new
+
+        BomDB::Query.new(edition: edition).print(
+          verse_format: verse_format_for_alignment,
+          linesep: ' ',
+          io: io
+        )
+        if options[:'edition-only']
+          puts io.string
+          exit
+        end
+
+        dwdiff = Diff::Dwdiff.new(options[:dwdiff])
+        diff = dwdiff.diff(io.string, File.read(file))
+
+        if options[:'diff-only']
+          puts diff
+          exit
+        end
+
+        puts Diff::Aligner.parse(diff)
       end
 
-      def import_books(file, reset = true)
-        data = File.read(file || datafile("books.json"))
 
-        import = BomDB::Import::Books.new(BomDB.db)
-        import.reset if reset
 
-        import.json(data)
+      private
+
+      def datafile(file)
+        
       end
 
-      def import_verses(file, reset = true)
-        data = File.read(file || datafile("verses.json"))
+      def read(file)
+        File.read(relative_or_data_file(file))
+      end
 
-        import = BomDB::Import::Verses.new(BomDB.db)
-        import.reset if reset
+      def relative_or_data_file(file)
+        if File.exist?(file)
+          file
+        else
+          if File.basename(file) == file
+            # Try our gem's data directory as a fallback for this file
+            File.join(BomDB.config.data_dir, file)
+          else
+            $stderr.puts "File not found: #{file}"
+            exit -1
+          end
+        end
+      end
 
-        import.json(data)
+      def format_from_file(file)
+        case File.extname(file).downcase
+        when ".txt" then "text"
+        when ".json" then "json"
+        else
+          $stderr.puts "Unable to determine format from file: #{file}"
+          exit -1
+        end
       end
 
-      def import_biblical_refs(reset = true)
-        import = BomDB::Import::BiblicalRefs.new(BomDB.db)
-        import.reset if reset
+      def type_from_file(file)
+        type = File.basename(file).gsub(/\.(txt|json)$/, '').downcase
+      end
 
-        import.import
+      def verse_format_for_alignment
+        verse_format = lambda do |book, chapter, verse|
+          "[|#{book} #{chapter}:#{verse}|]"
+        end
       end
 
       def show_result_and_maybe_exit(result)
-        if !result.success?
-          puts result.message
-          # if result.error.is_a?(Sequel::UniqueConstraintViolation)
-          puts "Try again with '--reset'? (NOTE: data may be deleted)"
-          # end
+        if result.success?
+          $stderr.puts "Success"
+        else
+          $stderr.puts result.message
+          # $stderr.puts "Try again with '--reset'? (NOTE: data may be deleted)"
           exit -1
         end
       end