bomdb 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed29a8b091ac7a85d4094e529a263c7b31f3cbed
-  data.tar.gz: df521bfa0632df2c26abc1e428d99c053238f6ae
+  metadata.gz: f7fbd83f638d94be3f15c4cf8442e2cdeccf8b1e
+  data.tar.gz: 2625af19b7be4478cd1abf1a39900ff41d21e10c
 SHA512:
-  metadata.gz: 78ff3e653b5a63293b576926f2083d3c7f440fbeb21c0adc6141d235568a00f210b8cbb7dffaf184cf3964d0b8c308d3c6dcb43ec06b5d563a66f96383566497
-  data.tar.gz: e0eec88eeecf615ac5a505162936cba47722a255b71e8ba5cafba7dc45c503b711a5fedc5b4fb362ed9c4c1fc4511e0e27bddbfe4e14842c091cf69a5a1e0e63
+  metadata.gz: 3c3d9acb48871cf5676c901c3d45b233a4d7e208da3d873b5b63930a2dbbe3d8456472b3045597f61493f8181b468f7d9a82defe4354d4252857ad2ded14ddcd
+  data.tar.gz: 58c3914913e18e8aa645997b13599d33e4c5876ce9fbf0e0928fd17e8603baf20ef87dd50f20822922fc2761fe2e6db5907fe58f0d8350012ffff667c62d8e48

data/Gemfile.lock

@@ -1,18 +1,25 @@
 PATH
   remote: .
   specs:
-    bomdb (0.4.0)
+    bomdb (0.5.0)
       colorize (~> 0.7)
       constellation (~> 0.1)
       levenshtein-ffi (~> 1.1)
+      mericope (~> 0.1.1)
       sequel (~> 4.21)
       sqlite3 (~> 1.3)
-      text_clean
+      text_clean (~> 0)
       thor (~> 0.19)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (4.2.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
     byebug (4.0.5)
       columnize (= 0.9.0)
     colorize (0.7.5)
@@ -20,18 +27,23 @@ GEM
     constellation (0.1.1)
       multi_json
     diff-lcs (1.2.5)
-    ffi (1.9.3)
+    ffi (1.9.8)
+    i18n (0.7.0)
+    json (1.8.2)
     levenshtein-ffi (1.1.0)
       ffi (~> 1.9)
+    mericope (0.1.1)
+      activesupport (~> 4.0)
+    minitest (5.6.0)
     multi_json (1.11.0)
     rake (10.4.2)
     rspec (3.2.0)
       rspec-core (~> 3.2.0)
       rspec-expectations (~> 3.2.0)
       rspec-mocks (~> 3.2.0)
-    rspec-core (3.2.2)
+    rspec-core (3.2.3)
       rspec-support (~> 3.2.0)
-    rspec-expectations (3.2.0)
+    rspec-expectations (3.2.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.2.0)
     rspec-mocks (3.2.1)
@@ -42,6 +54,9 @@ GEM
     sqlite3 (1.3.10)
     text_clean (0.2.2)
     thor (0.19.1)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
 
 PLATFORMS
   ruby

data/README.md

@@ -117,7 +117,14 @@ Suppose you have a new Book of Mormon text file that has been scanned from OCR o
 Given this text:
 
 ```
-I, Nephi, having been born of goodly parents, therefore I was taught somewhat in all the learning of my father; and having seen many afflictions in the course of my days--nevertheless, having been highly favored of the Lord in all my days; yea, having had a great knowledge of the goodness and the mysteries of God, therefore I make a record of my proceedings in my days; yea, I make a record in the language of my father, which consists of the learning of the Jews and the language of the Egyptians. And I know that the record which I make is true; and I make it with mine own hand; and I make it according to my knowledge.
+I, Nephi, having been born of goodly parents, therefore I was taught somewhat
+in all the learning of my father; and having seen many afflictions in the course
+of my days--nevertheless, having been highly favored of the Lord in all my days;
+yea, having had a great knowledge of the goodness and the mysteries of God,
+therefore I make a record of my proceedings in my days; yea, I make a record in
+the language of my father, which consists of the learning of the Jews and the
+language of the Egyptians. And I know that the record which I make is true; and
+I make it with mine own hand; and I make it according to my knowledge.
 ```
 
 You can automatically align and annotate it:
@@ -125,14 +132,67 @@ You can automatically align and annotate it:
 ```bash
 $ bomdb align my_typed_bom.txt
 
-# preamble text skipped...
+1 Nephi 1:1 I, Nephi, having been born of goodly parents, therefore I was taught somewhat in all the learning of my father; and having seen many afflictions in the course of my days--nevertheless, having been highly favored of the Lord in all my days; yea, having had a great knowledge of the goodness and the mysteries of God, therefore I make a record of my proceedings in my days;
 1 Nephi 1:2 yea, I make a record in the language of my father, which consists of the learning of the Jews and the language of the Egyptians. 
 1 Nephi 1:3 And I know that the record which I make  is true; and I make it with mine own hand; and I make it according to my knowledge. 
-# ... shows 6601 more verses
 ```
 
 Note that `align` requires the [dwdiff](http://linux.die.net/man/1/dwdiff) command on your system.
 
+### Custom Queries
+
+Here's a simple way to analyze the Book of Mormon to see the wherefore/therefore divergence leading to the [Mosiah Priority](http://signaturebookslibrary.org/new-approaches-to-the-book-of-mormon-10/) (Mosiah was written before 1 Nephi) hypothesis:
+
+```ruby
+$ bundle exec irb -rbomdb
+irb(main):001:0> q = BomDB::Query.new(exclude: 'Bible-OT')
+=> #<BomDB::Query:0x007f90ad1cb408 @edition=1829, @exclude="Bible-OT", @headings=false>
+irb(main):002:0> q.books.map{ |book,content| [book, content.scan(/wherefore/i).size, content.scan(/therefore/i).size] }
+=> [
+     ["1 Nephi", 99, 13], ["2 Nephi", 126, 6], ["Jacob", 53, 1],
+     ["Enos", 6, 0], ["Jarom", 3, 0], ["Omni", 6, 0], ["Words of Mormon", 5, 0],
+     ["Mosiah", 0, 122], ["Alma", 3, 288], ["Helaman", 0, 63],
+     ["3 Nephi", 3, 96], ["4 Nephi", 0, 5], ["Mormon", 0, 22],
+     ["Ether", 63, 26], ["Moroni", 38, 0]
+   ]
+```
+
+Or perhaps a little more visually:
+```ruby
+require 'bomdb'
+
+q = BomDB::Query.new(exclude: "Bible-OT")
+
+q.books.each do |book,content|
+  words = content.scan(/ +/).size
+  puts book.ljust(18) +
+    'W' * (content.scan(/wherefore/i).size.to_f / words * 2000) +
+    'T' * (content.scan(/therefore/i).size.to_f / words * 2000)
+end
+
+# 1 Nephi           WWWWWWWWT
+# 2 Nephi           WWWWWWWWWWWWWW
+# Jacob             WWWWWWWWWWW
+# Enos              WWWWWWWWWW
+# Jarom             WWWWWWWW
+# Omni              WWWWWWWW
+# Words of Mormon   WWWWWWWWWWW
+# Mosiah            TTTTTTTT
+# Alma              TTTTTT
+# Helaman           TTTTTT
+# 3 Nephi           TTTTTTT
+# 4 Nephi           TTTTT
+# Mormon            TTTT
+# Ether             WWWWWWWTTT
+# Moroni            WWWWWWWWWWWW
+```
+
+Other possible enumerables on a Query include:
+
+- books: enumerate on each book of the Book of Mormon
+- chapters: enumerate on each book and chapter of the Book of Mormon
+- wordgroups(N): enumerate on consecutive N words, e.g. each 1000 words
+
 ## Installation
 
 Ruby 2.1 is required. You should also have a normal build environment set up, e.g. command line tools on the mac, or GCC on Linux.

data/bomdb.gemspec

@@ -24,8 +24,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'thor',            '~> 0.19'
   spec.add_dependency 'constellation',   '~> 0.1'
   spec.add_dependency 'colorize',        '~> 0.7'
-  spec.add_dependency 'text_clean'
+  spec.add_dependency 'text_clean',      '~> 0'
   spec.add_dependency 'levenshtein-ffi', '~> 1.1'
+  spec.add_dependency 'mericope',        '~> 0.1.1'
 
   # spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake',    '~> 10.0'

data/lib/bomdb.rb

@@ -1,4 +1,5 @@
 require 'sequel'
+require 'mericope'
 
 dbfile = ARGV.first || "book_of_mormon.db"
 

data/lib/bomdb/export/verses.rb

@@ -5,13 +5,17 @@ module BomDB
     class Verses < Export::Base
       def export_json
         verses = []
-        @db[:verses].join(:books, :book_id => :book_id).
+        @db[:verses].
+          join(:books, :book_id => :book_id).
           where(:verse_heading => nil).
           order(:book_sort, :verse_chapter).
-          select_group(:book_name, :verse_chapter).
-          select_append{ Sequel.as(max(:verse_number), :count) }. 
         each do |v|
-          verses << { book: v[:book_name], chapter: v[:verse_chapter], verses: v[:count] }
+          verses << {
+            range_id: v[:verse_range_id],
+            book: v[:book_name],
+            chapter: v[:verse_chapter],
+            verse: v[:verse_number]
+          }
         end
         Export::Result.new(success: true, body: JSON.pretty_generate(verses))
       end

data/lib/bomdb/import/verses.rb

@@ -9,30 +9,38 @@ module BomDB
       # Expected data format is:
       # [
       #   {
-      #     "book": String,
-      #     "chapter": Int,
-      #     "verses": Int
+      #     "range_id": Int,  # the mericope range ID
+      #     "book": String,   # the name of the book, e.g. 1 Nephi
+      #     "chapter": Int,   # the chapter number
+      #     "verse": Int      # the verse number
       #   },
       #   ...
       # ]
       def import_json(data, **args)
+        book, chapter = nil, nil
+        verse_model = Models::Verse.new(@db)
         data.each do |r|
-          # Create a heading per chapter
-          Models::Verse.new(@db).find_or_create(
-            chapter:   r['chapter'],
-            verse:     nil,
-            book_name: r['book'],
-            heading:   true
-          )
-
-          # Create as many verses as is called for per chapter
-          (1..r['verses']).each do |verse_number|
+          if r['chapter'] != chapter || r['book'] != book
+            # Create a heading per chapter
             Models::Verse.new(@db).find_or_create(
               chapter:   r['chapter'],
-              verse:     verse_number,
-              book_name: r['book']
+              verse:     nil,
+              book_name: r['book'],
+              heading:   true
             )
+            chapter = r['chapter']
           end
+
+          if r['book'] != book
+            puts "Importing chapters & verses for '#{r['book']}'"
+            book = r['book']
+          end
+          verse_model.find_or_create(
+            chapter:   r['chapter'],
+            verse:     r['verse'],
+            book_name: r['book'],
+            range_id:  r['range_id']
+          )
         end
         Import::Result.new(success: true)
       rescue Sequel::UniqueConstraintViolation => e

data/lib/bomdb/models/verse.rb

@@ -16,19 +16,28 @@ module BomDB
       end
 
       # Create a verse and return its verse_id
-      def create(chapter:, verse:, book_name: nil, book_id: nil, heading: false)
+      def create(chapter:, verse:, book_name: nil, book_id: nil, heading: false, range_id: nil)
         @db[:verses].insert(
           book_id:       book_by_name_or_id(book_name, book_id),
           verse_chapter: chapter,
           verse_number:  verse,
-          verse_heading: heading ? 0 : nil
+          verse_heading: heading ? 0 : nil,
+          verse_range_id: range_id
         )
       end
 
       # Returns a verse_id after finding or creating the verse
-      def find_or_create(**args)
-        verse = find(**args)
-        (verse && verse[:verse_id]) || create(**args)
+      def find_or_create(chapter:, verse:, book_name: nil, book_id: nil, heading: false, range_id: nil)
+        v = find(
+          chapter: chapter, verse: verse,
+          book_name: book_name, book_id: book_id,
+          heading: heading
+        )
+        (v && v[:verse_id]) || create(
+          chapter: chapter, verse: verse,
+          book_name: book_name, book_id: book_id,
+          heading: heading, range_id: range_id
+        )
       end
 
       protected

data/lib/bomdb/query.rb

@@ -2,7 +2,7 @@ require 'bomdb/models/edition'
 
 module BomDB
   class Query
-    def initialize(edition:, exclude: nil, headings: false)
+    def initialize(edition: 1829, exclude: nil, headings: false)
       @edition = edition
       @exclude = exclude
       @headings = headings
@@ -35,6 +35,38 @@ module BomDB
       query.each(&block)
     end
 
+    def chapters
+      groups = query.all.group_by do |x|
+        [x[:book_name], x[:verse_chapter]]
+      end
+      Enumerator.new(groups.size) do |y|
+        groups.each do |heading, rows|
+          content = rows.map{ |r| r[:content_body] }.join(" ")
+          y.yield(heading, content)
+        end
+      end
+    end
+
+    def books
+      groups = query.all.group_by{ |x| x[:book_name] }
+      Enumerator.new(groups.size) do |y|
+        groups.each do |heading, rows|
+          content = rows.map{ |r| r[:content_body] }.join(" ")
+          y.yield(heading, content)
+        end
+      end
+    end
+
+    def wordgroups(wordcount)
+      content = query.all.map{ |r| r[:content_body] }.join(" ")
+      groups = content.scan(/[^ ]+/).each_slice(wordcount).map{ |w| w.join(" ") }
+      Enumerator.new(groups.size) do |y|
+        groups.each do |g|
+          y.yield(g)
+        end
+      end
+    end
+
     def print(verse_format: nil, body_format: nil, sep: ' ', linesep: "\n", io: $stdout)
       shown = false
       verse_format ||= lambda{ |book, chapter, verse| "#{book}#{sep}#{chapter}:#{verse}" }

data/lib/bomdb/schema.rb

@@ -37,6 +37,8 @@ module BomDB
         integer :verse_number,  :null => true
         integer :verse_heading, :null => true
 
+        integer :verse_range_id, :unique => true, :null => true
+
         index [:book_id, :verse_chapter, :verse_number, :verse_heading], :unique => true
       end if include?(tables, :verses)
 

data/lib/bomdb/version.rb

@@ -1,3 +1,3 @@
 module BomDB
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bomdb
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Duane Johnson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-14 00:00:00.000000000 Z
+date: 2015-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -84,14 +84,14 @@ dependencies:
   name: text_clean
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: mericope
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement