bomdb 0.0.1 → 0.1.0

data/lib/bomdb/diff/aligner.rb

@@ -0,0 +1,72 @@
+require 'strscan'
+
+module BomDB
+  module Diff
+    class Aligner
+      DIFF_RE = /\{(\+|\-)(.+?)\1\}/
+      INSERT_RE = /\{\+(.+?)\+\}/
+      VERSE_RE = /\[\|([^\]]+)\|\]/
+
+      def self.parse_verse_heading(scanner, deletion, verse_match)
+        # the text of the verse, e.g. "1 Nephi 1:1"
+        verse = verse_match[1]
+
+        # the range of the verse capture, e.g. [2, 17] from ". [|1 Nephi 1:1|]Yea"
+        verse_capture_slice = Range.new(*verse_match.offset(0), true)
+
+        # the deletion without the verse, e.g. ". Yea"
+        deletion_without_verse = deletion.clone
+        deletion_without_verse.slice!(verse_capture_slice)
+
+        # if there's an insertion immediately following...
+        if scanner.scan(INSERT_RE)
+          insertion = scanner.matched.match(INSERT_RE)[1]
+          insert_pos = verse_match.offset(0).first
+
+          # if the match, without the verse heading, is the same size as its
+          # substitution, then concat the pre_match, add the verse heading, and
+          # concat the post_match
+          if insertion.size > insert_pos
+            insertion[0...insert_pos] + "\n" + verse + insertion[(insert_pos-1)..-1]
+          else
+            insertion[0...insert_pos] + "\n" + verse
+          end
+        else
+          "\n" + verse
+        end
+      end
+
+      def self.parse(diff_text)
+        scanner = StringScanner.new(diff_text)
+
+        output = ""
+
+        last_pos = 0
+        while !scanner.eos?
+          if scanner.scan_until(DIFF_RE)
+            output << scanner.pre_match[last_pos..-1]
+            last_pos = scanner.pos
+            
+            diff_match = DIFF_RE.match(scanner.matched)
+            case diff_match[1]
+            when '-' then # this is a deletion
+              inner = diff_match[2]
+              # the only deletions we care about are those with verse headings inside them
+              if verse_match = VERSE_RE.match(inner)
+                output << parse_verse_heading(scanner, inner, verse_match)
+                last_pos = scanner.pos
+              end
+            when '+' then # this is an insertion
+              output << diff_match[2]
+            end
+          else
+            output << scanner.rest
+            break
+          end
+        end
+
+        return output
+      end
+    end
+  end
+end

data/lib/bomdb/diff/dwdiff.rb

@@ -0,0 +1,28 @@
+require 'tmpdir'
+
+module BomDB
+  module Diff
+    # This wraps the command-line tool, dwdiff
+    # See http://linux.die.net/man/1/dwdiff
+    class Dwdiff
+      def initialize(bin = '/usr/local/bin/dwdiff')
+        @bin = bin
+      end
+
+      def diff(str1, str2)
+        Dir.mktmpdir("bomdb") do |dir|
+          file1 = File.join(dir, "file1.txt")
+          file2 = File.join(dir, "file2.txt")
+          File.open(file1, "w"){ |f1| f1.write(str1) }
+          File.open(file2, "w"){ |f2| f2.write(str2) }
+          # -w : start-delete marker, {-
+          # -x : end-delete marker,   -}
+          # -y : start-insert marker, {+
+          # -z : end-insert marker,   +}
+          # -P : use punctuation characters as delimiters
+          `#{@bin} -w'{-' -x'-}' -y'{+' -z'+}' -P #{file1} #{file2}`
+        end
+      end
+    end
+  end
+end

data/lib/bomdb/export/base.rb

@@ -0,0 +1,24 @@
+module BomDB
+  module Export
+    class Base
+      attr_reader :db, :opts
+
+      def initialize(db, **opts)
+        @db = db
+        @opts = opts
+      end
+
+      def export(format: 'json', **options)
+        case format
+        when 'json' then export_json
+        when 'text' then export_text
+        else
+          return Import::Result.new(
+            success: false,
+            error: "Unknown format: #{format}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/bomdb/export/books.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+module BomDB
+  module Export
+    class Books < Export::Base
+      def export_json
+        books = []
+        @db[:books].each do |b|
+          books << JSON::generate([ b[:book_name], b[:book_group], b[:book_sort] ], array_nl: ' ')
+        end
+        Export::Result.new(success: true, body: "[\n  " + books.join(",\n  ") + "\n]\n")
+      end
+    end
+  end
+end

data/lib/bomdb/export/contents.rb

@@ -0,0 +1,100 @@
+require 'json'
+require 'bomdb/models/edition'
+
+module BomDB
+  module Export
+    class Contents < Export::Base
+      def export_json
+        editions_by_id = selected_editions()
+
+        contents = {}
+        content_query(editions_by_id.keys).each do |r|
+          edition = editions_by_id[ r[:edition_id] ]
+          book    = (contents[ r[:book_name] ] ||= {})
+          chapter = (book[ r[:verse_chapter] ] ||= {})
+          verse   = (chapter[ full_verse_ref(r) ] ||= {})
+          verse[ edition[:edition_year] ] = r[:content_body]
+        end
+
+        frame = {
+          editions: editions_legend(editions_by_id),
+          contents: contents
+        }
+
+        Export::Result.new(success: true, body: JSON.pretty_generate(frame))
+      end
+
+      def export_text
+        editions_by_id = selected_editions()
+
+        output = ""
+        editions_by_id.each_pair do |id, edition|
+          title = edition[:edition_name]
+          output << title + "\n"
+          output << ('=' * title.size) + "\n"
+
+          content_query([id]).each do |r|
+            output << full_verse_ref(r) + "\t" + r[:content_body] + "\n"
+          end
+
+          output << "\n"
+        end
+
+        Export::Result.new(success: true, body: output)
+      end
+
+      protected
+
+      def editions_legend(editions_by_id)
+        {}.tap do |editions|
+          editions_by_id.each_pair do |id, row|
+            year, name = row[:edition_year], row[:edition_name]
+            editions[ year ] = { year: year, name: name }
+          end
+        end
+      end
+
+      def selected_editions
+        {}.tap do |editions_by_id|
+          if opts[:edition_prefixes] == :all
+            # "all" means all editions that actually have content
+            edition_query.each do |e|
+              editions_by_id[ e[:edition_id] ] = e
+            end
+          else
+            # export editions that are mentioned by name-prefix
+            ed_model = Models::Edition.new(@db)
+            opts[:edition_prefixes].each do |epat|
+              e = ed_model.find(epat)
+              editions_by_id[ e[:edition_id] ] = e
+            end
+          end
+        end
+      end
+
+      def edition_query
+        @db[:editions].
+          left_outer_join(:contents, :edition_id => :edition_id).
+          select_group(:editions__edition_id, :edition_year, :edition_name).
+          select_append{ Sequel.as(count(:verse_id), :count) }.
+          having{ count > 0 }.
+          order(:edition_name)
+      end
+
+      def content_query(edition_ids)
+        @db[:verses].
+          join(:books, :book_id => :book_id).
+          join(:editions).
+          join(:contents, :edition_id => :edition_id, :verse_id => :verses__verse_id).
+          order(:book_sort, :verse_heading, :verse_chapter, :verse_number).
+          select(:editions__edition_id, :book_name, :verse_chapter, :verse_number, :content_body).
+          where(:editions__edition_id => edition_ids).
+          where(:verse_heading => nil)
+      end
+
+      def full_verse_ref(row)
+        "#{row[:book_name]} #{row[:verse_chapter]}:#{row[:verse_number]}"
+      end
+    end
+  end
+end

data/lib/bomdb/export/editions.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+module BomDB
+  module Export
+    class Editions < Export::Base
+      def export_json
+        editions = []
+        @db[:editions].order(:edition_year, :edition_name).each do |e|
+          editions << JSON::generate([e[:edition_year], e[:edition_name]], array_nl: ' ')
+        end
+        Export::Result.new(success: true, body: "[\n  " + editions.join(",\n  ") + "\n]\n")
+      end
+    end
+  end
+end

data/lib/bomdb/export/result.rb

@@ -0,0 +1,29 @@
+module BomDB
+  module Export
+    class Result
+      attr_reader :success, :error, :body
+
+      def initialize(success:, body: nil, error: nil)
+        @success = success
+        @error = error
+        @body = body
+      end
+
+      def success?
+        @success
+      end
+
+      def to_s
+        @body
+      end
+
+      def message
+        if @success
+          "Succeeded"
+        else
+          @error.to_s
+        end
+      end
+    end
+  end
+end

data/lib/bomdb/export/verses.rb

@@ -0,0 +1,20 @@
+require 'json'
+
+module BomDB
+  module Export
+    class Verses < Export::Base
+      def export_json
+        verses = []
+        @db[:verses].join(:books, :book_id => :book_id).
+          where(:verse_heading => nil).
+          order(:book_sort, :verse_chapter).
+          select_group(:book_name, :verse_chapter).
+          select_append{ Sequel.as(max(:verse_number), :count) }. 
+        each do |v|
+          verses << { book: v[:book_name], chapter: v[:verse_chapter], verses: v[:count] }
+        end
+        Export::Result.new(success: true, body: JSON.pretty_generate(verses))
+      end
+    end
+  end
+end

data/lib/bomdb/import/base.rb

@@ -3,8 +3,38 @@ require 'json'
 module BomDB
   module Import
     class Base
-      def initialize(db)
+      attr_reader :db, :opts
+
+      def initialize(db, **opts)
         @db = db
+        @opts = opts
+      end
+
+      def self.tables(*tables)
+        @tables = tables
+      end
+
+      def tables
+        self.class.instance_variable_get("@tables")
+      end
+
+      def import(data, format: 'json')
+        if !schema.has_tables?(tables)
+          return Import::Result.new(
+            success: false,
+            error: "Database table(s) not present: [#{tables.join(', ')}]"
+          )
+        end
+
+        case format
+        when 'json' then import_json(ensure_parsed_json(data))
+        when 'text' then import_text(data)
+        else
+          return Import::Result.new(
+            success: false,
+            error: "Unknown format: #{format}"
+          )
+        end
       end
 
       def ensure_parsed_json(data)

data/lib/bomdb/import/books.rb

@@ -3,24 +3,15 @@ require 'json'
 module BomDB
   module Import
     class Books < Import::Base
-
-      def reset
-        schema.reset(:books)
-      end
+      tables :books
 
       # Expected data format is:
       # [
       #   [book_name:String, book_group:String, book_sort:Integer],
       #   ...
       # ]
-      def json(data)
-        if !schema.has_tables?(:books)
-          return Import::Result.new(
-            success: false,
-            error: "Database table 'books' not present."
-          )
-        end
-        ensure_parsed_json(data).each do |name, group, sort|
+      def import_json(data)
+        data.each do |name, group, sort|
           @db[:books].insert(
             book_name: name,
             book_group: group,

data/lib/bomdb/import/contents.rb

@@ -0,0 +1,137 @@
+require 'bomdb/models/verse'
+require 'bomdb/models/edition'
+
+module BomDB
+  module Import
+    class Contents < Import::Base
+      tables :books, :verses, :editions, :contents
+      DEFAULT_VERSE_CONTENT_RE = /^\s*(.+)(\d+):(\d+)\s*(.*)$/
+      DEFAULT_VERSE_REF_RE = /^([^:]+)\s+(\d+):(\d+)$/
+
+      def import_text(data)
+        if opts[:edition_id].nil?
+          raise ArgumentError, "Edition is required for text import of contents"
+        end
+
+        verse_re = opts[:verse_re] || DEFAULT_VERSE_CONTENT_RE
+
+        data.each_line do |line|
+          if line =~ verse_re
+            book_name, chapter, verse, content = $1, $2, $3, $4
+
+            book = find_book(book_name)
+            return book if book.is_a?(Import::Result)
+
+            verse_id = Models::Verse.new(@db).find_or_create(
+              chapter: chapter,
+              verse: verse,
+              book_id: book[:book_id]
+            )
+
+            @db[:contents].insert(
+              edition_id:   opts[:edition_id],
+              verse_id:     verse_id,
+              content_body: content
+            )
+          end
+        end
+      end
+
+      def import_json(data)
+        # this cross-ref is for looking up file-edition-id => database-edition-id
+        editions_xref = {}
+
+        ed_model = Models::Edition.new(@db)
+        verse_model = Models::Verse.new(@db)
+
+        data['editions'].each_pair do |id, e|
+          editions_xref[ id ] = ed_model.find_or_create(e["year"].to_i, e["name"])
+        end
+
+        data['contents'].each_pair do |book_name, chapters|
+          chapters.each_pair do |chapter, verses|
+            verses.each_pair do |verse_full_ref, editions|
+              match = DEFAULT_VERSE_REF_RE.match(verse_full_ref)
+              if match
+                verse_number = match[3].to_i
+                verse = verse_model.find(
+                  book_name: book_name,
+                  chapter: chapter,
+                  verse: verse_number
+                )
+                if verse.nil?
+                  return Import::Result.new(success: false,
+                    error: "Unable to find verse: book: " +
+                           "'#{book_name}', chapter: '#{chapter}', " +
+                           "verse: '#{verse_number}'")
+                end
+                editions.each_pair do |file_edition_id, content_body|
+                  @db[:contents].insert(
+                    edition_id: editions_xref[ file_edition_id ],
+                    verse_id: verse[:verse_id],
+                    content_body: content_body
+                  )
+                end
+              else
+                $stderr.puts "Unable to parse verse ref from '#{verse_full_ref}', skipping"
+              end
+            end
+          end
+        end
+        Import::Result.new(success: true)
+      end
+
+      def import_json_old(data)
+        data.each_pair do |book_name, year_editions|
+          year_editions.each do |year_edition|
+            year_edition.each_pair do |year, d|
+              m = d["meta"]
+
+              book = find_book(book_name)
+              return book if book.is_a?(Import::Result)
+
+              verse_id = Models::Verse.new(@db).find_or_create(
+                chapter: m['chapter'],
+                verse: m['verse'],
+                book_id: book[:book_id],
+                heading: m['heading']
+              )
+
+              ed_id = opts[:edition_id] || find_or_create_edition(year)
+
+              @db[:contents].insert(
+                edition_id:   ed_id,
+                verse_id:     verse_id,
+                content_body: d["content"]
+              )
+            end
+          end
+        end
+        Import::Result.new(success: true)
+      rescue Sequel::UniqueConstraintViolation => e
+        Import::Result.new(success: false, error: e)
+      end
+
+      protected
+
+      def find_book(book_name)
+        book = @db[:books].where(:book_name => book_name).first
+        if book.nil?
+          Import::Result.new(success: false, error: "Unable to find book '#{book_name}'")
+        else
+          book
+        end
+      end
+
+      def find_or_create_edition(year, name = nil)
+        name ||= year.to_s
+        edition = @db[:editions].where(:edition_year => year).first
+        edition_id =  (edition && edition[:edition_id]) || @db[:editions].insert(
+          edition_name: name,
+          edition_year: year
+        )
+        return edition_id
+      end
+    end
+  end
+end