bomdb 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 033e279990a8a889f27f0cfd0569d3e76e9345aa
-  data.tar.gz: f8a5d832f7398bd0caeb5669785dd337519baa41
+  metadata.gz: 10684c60ce9c451d4c17ab58a0f102cf418d4b18
+  data.tar.gz: 9d383abefdc1e561179b681c0d7edfaa95bc4ea1
 SHA512:
-  metadata.gz: 3ae1472a90909e6c194662317bd0d284a9cad4fdf30c5148ed7200f3623d9cdc0085292cc47751fc17f1a2c6099f443ee42f083617f464c469d3017c1e34be1e
-  data.tar.gz: f832dd8e4625497a14985f9a9e2f684a9ca5025c201d869ac48093ac75cbb96b86eb9f1e21eb532e0e5ba6328c41fa6c70632bad14f1f9afb1cfde7f251fdf29
+  metadata.gz: 96ad2c76b012fcf6919ec86878e27eb5e98fd572c63b71731183bd8b1c07435e3f2bbacb13b99e897203a2d3cff67803fab8f973de98aaac197bb9e5d66890f9
+  data.tar.gz: 2e7bbdac26409d4a1eb316332598abc16df4493d32d7e241e54a59075293743c71b8fba0b3a4597018e808b8b2264e73e54e0092d04bdff6965de11e60b61b78

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    bomdb (0.2.1)
+    bomdb (0.2.3)
       colorize (~> 0.7)
       constellation (~> 0.1)
+      levenshtein-ffi (~> 1.1)
       sequel (~> 4.21)
       sqlite3 (~> 1.3)
       text_clean
@@ -19,6 +20,9 @@ GEM
     constellation (0.1.1)
       multi_json
     diff-lcs (1.2.5)
+    ffi (1.9.3)
+    levenshtein-ffi (1.1.0)
+      ffi (~> 1.9)
     multi_json (1.11.0)
     rake (10.4.2)
     rspec (3.2.0)

data/README.md

@@ -104,7 +104,7 @@ Bible-OT (594 refs)
 ```
 
 ```bash
-$bomdb references Bible-NT
+$ bomdb references Bible-NT
 1 Corinthians 15:32
 Luke 12:19
 ```
@@ -135,7 +135,13 @@ Note that `align` requires the [dwdiff](http://linux.die.net/man/1/dwdiff) comma
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Ruby 2.1 is required. You should also have a normal build environment set up, e.g. command line tools on the mac, or GCC on Linux.
+
+To install BomDB for use on the command line, use `gem install`:
+
+    $ gem install bomdb
+
+To include bomdb in another Ruby app, add this line to your application's `Gemfile`:
 
 ```ruby
 source 'https://rubygems.org'
@@ -147,10 +153,6 @@ And then execute:
 
     $ bundle
 
-Or install it yourself as:
-
-    $ gem install bomdb
-
 ## Configuration
 
 Some settings can be configured in a .bomdb file in your home directory:

data/bomdb.gemspec

@@ -19,12 +19,13 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.bindir        = "bin"
 
-  spec.add_dependency 'sequel',        '~> 4.21'
-  spec.add_dependency 'sqlite3',       '~> 1.3'
-  spec.add_dependency 'thor',          '~> 0.19'
-  spec.add_dependency 'constellation', '~> 0.1'
-  spec.add_dependency 'colorize',      '~> 0.7'
+  spec.add_dependency 'sequel',          '~> 4.21'
+  spec.add_dependency 'sqlite3',         '~> 1.3'
+  spec.add_dependency 'thor',            '~> 0.19'
+  spec.add_dependency 'constellation',   '~> 0.1'
+  spec.add_dependency 'colorize',        '~> 0.7'
   spec.add_dependency 'text_clean'
+  spec.add_dependency 'levenshtein-ffi', '~> 1.1'
 
   # spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake',    '~> 10.0'

data/lib/bomdb.rb

@@ -15,8 +15,6 @@ module BomDB
   end
 end
 
-require 'byebug'
-
 require 'bomdb/version'
 require 'bomdb/config'
 require 'bomdb/schema'

data/lib/bomdb/cli/application.rb

@@ -29,7 +29,12 @@ module BomDB
           exit -1
         end
 
-        result = importer.import(read(file), format: format)
+        begin
+          result = importer.import(read(file), format: format)
+        rescue JSON::ParserError
+          puts "Couldn't parse as JSON. Use '--format=text'?"
+          exit -1
+        end
         show_result_and_maybe_exit(result)
       end
 
@@ -246,24 +251,21 @@ module BomDB
         end
 
         dwdiff = Diff::Dwdiff.new(options[:dwdiff])
-        diff = dwdiff.diff(io.string, File.read(file))
+        align_str = File.read(file).gsub(/\s\s+/, ' ').gsub(':', '~')
+        diff = dwdiff.diff(io.string, align_str)
 
         if options[:'diff-only']
           puts diff
           exit
         end
 
-        puts Diff::Aligner.parse(diff)
+        puts Diff::Aligner.parse(diff).gsub('~', ':')
       end
 
 
 
       private
 
-      def datafile(file)
-        
-      end
-
       def read(file)
         File.read(relative_or_data_file(file))
       end

data/lib/bomdb/diff/aligner.rb

@@ -1,39 +1,53 @@
 require 'strscan'
+require 'levenshtein'
 
 module BomDB
   module Diff
     class Aligner
       DIFF_RE = /\{(\+|\-)(.+?)\1\}/
       INSERT_RE = /\{\+(.+?)\+\}/
+      WS_INSERT_RE = /\s?\{\+(.+?)\+\}/
       VERSE_RE = /\[\|([^\]]+)\|\]/
 
-      def self.parse_verse_heading(scanner, deletion, verse_match)
+      def self.parse_verse_heading(verse_match, deletion, insertion = nil)
         # the text of the verse, e.g. "1 Nephi 1:1"
         verse = verse_match[1]
-
-        # the range of the verse capture, e.g. [2, 17] from ". [|1 Nephi 1:1|]Yea"
-        verse_capture_slice = Range.new(*verse_match.offset(0), true)
-
-        # the deletion without the verse, e.g. ". Yea"
-        deletion_without_verse = deletion.clone
-        deletion_without_verse.slice!(verse_capture_slice)
+        before = after = ''
 
         # if there's an insertion immediately following...
-        if scanner.scan(INSERT_RE)
-          insertion = scanner.matched.match(INSERT_RE)[1]
-          insert_pos = verse_match.offset(0).first
+        if insertion
+          # we can assume split will succeed, because the verse was matched
+          del_before, del_after = deletion.split(verse_match[0], 2)
 
-          # if the match, without the verse heading, is the same size as its
-          # substitution, then concat the pre_match, add the verse heading, and
-          # concat the post_match
-          if insertion.size > insert_pos
-            insertion[0...insert_pos] + "\n" + verse + insertion[(insert_pos-1)..-1]
+          del_before.strip!
+          del_after.strip!
+
+          if del_before.empty? && del_after.empty?
+            # do nothing
+          elsif del_before.empty?
+            # the entire insertion goes after the verse heading
+            after = insertion.chomp
+          elsif del_after.empty?
+            # the entire insertion goes before the verse heading
+            before = insertion.chomp
           else
-            insertion[0...insert_pos] + "\n" + verse
+            # we have to use some heuristics to figure out where to split
+            # the insertion.
+
+            candidates = (0..(insertion.size-1)).map do |i|
+              d1 = Levenshtein.distance(del_before, insertion[0..i])
+              d2 = Levenshtein.distance(del_after, insertion[(i + 1)..-1])
+              d3 = insertion[i] == ' ' ? 1 : 0
+              [ d1 + d2 + d3, insertion[0..i].chomp, insertion[(i + 1)..-1].chomp ]
+            end.sort_by{ |a| a.first }
+            if candidates.empty?
+              raise "Unable to find candidate split for #{del_before.inspect}, #{del_after.inspect} on #{insertion.inspect}"
+            end
+
+            score, before, after = candidates.first
           end
-        else
-          "\n" + verse
         end
+        [before, verse, after]
       end
 
       def self.parse(diff_text)
@@ -50,10 +64,22 @@ module BomDB
             diff_match = DIFF_RE.match(scanner.matched)
             case diff_match[1]
             when '-' then # this is a deletion
-              inner = diff_match[2]
+
+              delete_inner = diff_match[2] # e.g. ", [|1 Nephi 1:1|] I"
+              # see if there's a verse heading in delete_inner
+              verse_match = VERSE_RE.match(delete_inner)
+
               # the only deletions we care about are those with verse headings inside them
-              if verse_match = VERSE_RE.match(inner)
-                output << parse_verse_heading(scanner, inner, verse_match)
+              if verse_match
+                if scanner.scan(WS_INSERT_RE)
+                  ws_insert_match = WS_INSERT_RE.match(scanner.matched)
+                  insert_inner = ws_insert_match[1]
+                else
+                  insert_inner = nil
+                end
+                before, verse, after = parse_verse_heading(verse_match, delete_inner, insert_inner)
+                output << before + "\n" + verse
+                output << " " + after
                 last_pos = scanner.pos
               end
             when '+' then # this is an insertion
@@ -65,7 +91,7 @@ module BomDB
           end
         end
 
-        return output
+        return output.gsub(/  +/, ' ').gsub(/ +$/, '')
       end
     end
   end

data/lib/bomdb/import/contents.rb

@@ -7,6 +7,7 @@ module BomDB
       tables :books, :verses, :editions, :contents
       DEFAULT_VERSE_CONTENT_RE = /^\s*(.+)\s+(\d+):(\d+)\s+(.*)$/
       DEFAULT_VERSE_REF_RE     = /^\s*(.+)\s+(\d+):(\d+)$/
+      MAX_DUPS = 5
 
       def import_text(data)
         if opts[:edition_prefix].nil?
@@ -23,9 +24,11 @@ module BomDB
             error: "Edition matching prefix '#{opts[:edition_prefix]}' not found"
           )
         end
+        edition_id = edition[:edition_id]
 
         verse_re = opts[:verse_re] || DEFAULT_VERSE_CONTENT_RE
 
+        times_tried = 0
         data.each_line do |line|
           if line =~ verse_re
             book_name, chapter, verse, content = $1, $2, $3, $4
@@ -39,11 +42,24 @@ module BomDB
               book_id: book[:book_id]
             )
 
-            @db[:contents].insert(
-              edition_id:   edition[:edition_id],
-              verse_id:     verse_id,
-              content_body: content
-            )
+            begin
+              @db[:contents].insert(
+                edition_id:   edition_id,
+                verse_id:     verse_id,
+                content_body: content
+              )
+            rescue Sequel::UniqueConstraintViolation => e
+              msg = "edition_id: #{edition_id}, verse: '#{book_name} #{chapter}:#{verse}', content: #{content.inspect}"
+              $stderr.puts "Warning: duplicate #{msg}"
+              times_tried += 1
+              if times_tried > MAX_DUPS
+                return Import::Result.new(success: false,
+                  error: "Too many duplicate rows. Stopped at #{msg}"
+                )
+              else
+                next
+              end
+            end
           end
         end
         Import::Result.new(success: true)

data/lib/bomdb/version.rb

@@ -1,3 +1,3 @@
 module BomDB
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bomdb
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Duane Johnson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-12 00:00:00.000000000 Z
+date: 2015-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: levenshtein-ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement