bomdb 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -12
- data/Gemfile.lock +3 -1
- data/README.md +25 -1
- data/bomdb.gemspec +2 -1
- data/data/book_of_mormon.db +0 -0
- data/lib/bomdb.rb +15 -1
- data/lib/bomdb/cli/application.rb +217 -71
- data/lib/bomdb/diff/aligner.rb +72 -0
- data/lib/bomdb/diff/dwdiff.rb +28 -0
- data/lib/bomdb/export/base.rb +24 -0
- data/lib/bomdb/export/books.rb +15 -0
- data/lib/bomdb/export/contents.rb +100 -0
- data/lib/bomdb/export/editions.rb +15 -0
- data/lib/bomdb/export/result.rb +29 -0
- data/lib/bomdb/export/verses.rb +20 -0
- data/lib/bomdb/import/base.rb +31 -1
- data/lib/bomdb/import/books.rb +3 -12
- data/lib/bomdb/import/contents.rb +137 -0
- data/lib/bomdb/import/editions.rb +26 -0
- data/lib/bomdb/import/refs.rb +65 -0
- data/lib/bomdb/import/verses.rb +31 -50
- data/lib/bomdb/models/edition.rb +27 -0
- data/lib/bomdb/models/verse.rb +47 -0
- data/lib/bomdb/query.rb +29 -18
- data/lib/bomdb/schema.rb +28 -28
- data/lib/bomdb/version.rb +1 -1
- data/spec/bomdb/query_spec.rb +27 -0
- data/spec/bomdb/schema_spec.rb +28 -0
- data/spec/spec_helper.rb +11 -0
- metadata +33 -5
- data/data/books.json +0 -17
- data/data/verses.json +0 -145590
- data/lib/bomdb/import/biblical_refs.rb +0 -359
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module BomDB
|
4
|
+
module Import
|
5
|
+
class Editions < Import::Base
|
6
|
+
tables :editions
|
7
|
+
|
8
|
+
# Expected data format is:
|
9
|
+
# [
|
10
|
+
# [edition_year:Integer, edition_name:String],
|
11
|
+
# ...
|
12
|
+
# ]
|
13
|
+
def import_json(data, **args)
|
14
|
+
data.each do |year, name|
|
15
|
+
@db[:editions].insert(
|
16
|
+
edition_year: year,
|
17
|
+
edition_name: name
|
18
|
+
)
|
19
|
+
end
|
20
|
+
Import::Result.new(success: true)
|
21
|
+
rescue Sequel::UniqueConstraintViolation => e
|
22
|
+
Import::Result.new(success: false, error: e)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module BomDB
|
4
|
+
module Import
|
5
|
+
class Refs < Import::Base
|
6
|
+
tables :refs
|
7
|
+
|
8
|
+
# Expected data format is:
|
9
|
+
# {
|
10
|
+
# (ref_name:String): [
|
11
|
+
# {
|
12
|
+
# "book":String,"chapter":Int,"verse":Int,
|
13
|
+
# "ref_book":String,"ref_chapter":Int,"ref_verse":Int,
|
14
|
+
# "is_parallel":Bool,"is_quotation":Bool
|
15
|
+
# }
|
16
|
+
# ...
|
17
|
+
# ]
|
18
|
+
# }
|
19
|
+
def import_json(data, **args)
|
20
|
+
data.each_pair do |ref_name, rows|
|
21
|
+
rows.each do |r|
|
22
|
+
verse = @db[:verses].
|
23
|
+
join(:books, :book_id => :book_id).
|
24
|
+
where(
|
25
|
+
book_name: r['book'],
|
26
|
+
verse_chapter: r['chapter'],
|
27
|
+
verse_number: r['verse'],
|
28
|
+
verse_heading: nil).
|
29
|
+
first
|
30
|
+
if verse.nil?
|
31
|
+
return Import::Result.new(
|
32
|
+
success: false,
|
33
|
+
error: "Unable to find verse #{r['book']} #{r['chapter']}:#{r['verse']}"
|
34
|
+
)
|
35
|
+
end
|
36
|
+
|
37
|
+
@db[:refs].insert(
|
38
|
+
verse_id: verse[:verse_id],
|
39
|
+
ref_name: ref_name,
|
40
|
+
ref_book: r['ref_book'],
|
41
|
+
ref_chapter: r['ref_chapter'],
|
42
|
+
ref_verse: r['ref_verse'],
|
43
|
+
ref_is_parallel: r['is_parallel'],
|
44
|
+
ref_is_quotation: r['is_quotation']
|
45
|
+
)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
Import::Result.new(success: true)
|
49
|
+
rescue Sequel::UniqueConstraintViolation => e
|
50
|
+
Import::Result.new(success: false, error: e)
|
51
|
+
end
|
52
|
+
|
53
|
+
# def list
|
54
|
+
# @db[
|
55
|
+
# "SELECT book_name, verse_chapter, verse_number, ref_name, ref_book, ref_chapter, ref_verse " +
|
56
|
+
# "FROM `verses` v " +
|
57
|
+
# "JOIN `books` b ON b.book_id = v.book_id " +
|
58
|
+
# "JOIN `refs` r ON r.verse_id = v.verse_id " +
|
59
|
+
# "ORDER BY ref_book, ref_chapter, ref_verse"
|
60
|
+
# ].map(&:inspect).join("\n")
|
61
|
+
# end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/lib/bomdb/import/verses.rb
CHANGED
@@ -1,62 +1,43 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'bomdb/models/verse'
|
3
|
+
|
1
4
|
module BomDB
|
2
5
|
module Import
|
3
6
|
class Verses < Import::Base
|
4
|
-
|
5
|
-
|
6
|
-
def reset
|
7
|
-
schema.reset(*(TABLES - [:books]))
|
8
|
-
end
|
9
|
-
|
10
|
-
def json(data)
|
11
|
-
if schema.has_tables?(*TABLES)
|
12
|
-
ensure_parsed_json(data).each_pair do |book_name, year_versions|
|
13
|
-
year_versions.each do |year_version|
|
14
|
-
year_version.each_pair do |year, d|
|
15
|
-
m = d["meta"]
|
16
|
-
|
17
|
-
book = @db[:books].where(:book_name => book_name).first
|
18
|
-
if book.nil?
|
19
|
-
return Import::Result.new(success: false, error: "Unable to find book '#{book_name}'")
|
20
|
-
end
|
7
|
+
tables :books, :verses
|
21
8
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
9
|
+
# Expected data format is:
|
10
|
+
# [
|
11
|
+
# {
|
12
|
+
# "book": String,
|
13
|
+
# "chapter": Int,
|
14
|
+
# "verses": Int
|
15
|
+
# },
|
16
|
+
# ...
|
17
|
+
# ]
|
18
|
+
def import_json(data, **args)
|
19
|
+
data.each do |r|
|
20
|
+
# Create a heading per chapter
|
21
|
+
Models::Verse.new(@db).find_or_create(
|
22
|
+
chapter: r['chapter'],
|
23
|
+
verse: nil,
|
24
|
+
book_name: r['book'],
|
25
|
+
heading: true
|
26
|
+
)
|
40
27
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
28
|
+
# Create as many verses as is called for per chapter
|
29
|
+
(1..r['verses']).each do |verse_number|
|
30
|
+
Models::Verse.new(@db).find_or_create(
|
31
|
+
chapter: r['chapter'],
|
32
|
+
verse: verse_number,
|
33
|
+
book_name: r['book']
|
34
|
+
)
|
48
35
|
end
|
49
|
-
Import::Result.new(success: true)
|
50
|
-
else
|
51
|
-
Import::Result.new(
|
52
|
-
success: false,
|
53
|
-
error: "Database tables #{TABLES.join(', ')} not ready. Try again with --reset."
|
54
|
-
)
|
55
36
|
end
|
37
|
+
Import::Result.new(success: true)
|
56
38
|
rescue Sequel::UniqueConstraintViolation => e
|
57
39
|
Import::Result.new(success: false, error: e)
|
58
40
|
end
|
59
|
-
|
60
41
|
end
|
61
42
|
end
|
62
|
-
end
|
43
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module BomDB
|
2
|
+
module Models
|
3
|
+
class Edition
|
4
|
+
def initialize(db)
|
5
|
+
@db = db
|
6
|
+
end
|
7
|
+
|
8
|
+
# Find an edition and return a hash, or nil if not found
|
9
|
+
def find(edition_name_prefix)
|
10
|
+
@db[:editions].
|
11
|
+
where(Sequel.like(:edition_name, "#{edition_name_prefix}%")).
|
12
|
+
or(:edition_year => edition_name_prefix).
|
13
|
+
first
|
14
|
+
end
|
15
|
+
|
16
|
+
# Returns an edition_id, either found in the db, or created as necessary
|
17
|
+
def find_or_create(year, name)
|
18
|
+
found = @db[:editions].where(edition_year: year, edition_name: name).first
|
19
|
+
return found[:edition_id] if found
|
20
|
+
@db[:editions].insert(
|
21
|
+
edition_year: year,
|
22
|
+
edition_name: name
|
23
|
+
)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module BomDB
|
2
|
+
module Models
|
3
|
+
class Verse
|
4
|
+
def initialize(db)
|
5
|
+
@db = db
|
6
|
+
end
|
7
|
+
|
8
|
+
# Find a verse and return a hash, or nil if not found
|
9
|
+
def find(chapter:, verse:, book_name: nil, book_id: nil, heading: false)
|
10
|
+
@db[:verses].where(
|
11
|
+
book_id: book_by_name_or_id(book_name, book_id),
|
12
|
+
verse_chapter: chapter,
|
13
|
+
verse_number: verse,
|
14
|
+
verse_heading: heading ? 0 : nil
|
15
|
+
).first
|
16
|
+
end
|
17
|
+
|
18
|
+
# Create a verse and return its verse_id
|
19
|
+
def create(chapter:, verse:, book_name: nil, book_id: nil, heading: false)
|
20
|
+
@db[:verses].insert(
|
21
|
+
book_id: book_by_name_or_id(book_name, book_id),
|
22
|
+
verse_chapter: chapter,
|
23
|
+
verse_number: verse,
|
24
|
+
verse_heading: heading ? 0 : nil
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
# Returns a verse_id after finding or creating the verse
|
29
|
+
def find_or_create(**args)
|
30
|
+
verse = find(**args)
|
31
|
+
(verse && verse[:verse_id]) || create(**args)
|
32
|
+
end
|
33
|
+
|
34
|
+
protected
|
35
|
+
|
36
|
+
def book_by_name_or_id(book_name, book_id)
|
37
|
+
if book_id.nil? and book_name.nil?
|
38
|
+
raise ArgumentError, "book_name or book_id is required"
|
39
|
+
elsif book_id.nil?
|
40
|
+
@db[:books].where(book_name: book_name).first[:book_id]
|
41
|
+
else
|
42
|
+
book_id
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/lib/bomdb/query.rb
CHANGED
@@ -1,40 +1,51 @@
|
|
1
|
+
require 'bomdb/models/edition'
|
2
|
+
|
1
3
|
module BomDB
|
2
4
|
class Query
|
3
|
-
def initialize(edition:, exclude: nil)
|
5
|
+
def initialize(edition:, exclude: nil, headings: false)
|
4
6
|
@edition = edition
|
5
7
|
@exclude = exclude
|
8
|
+
@headings = headings
|
6
9
|
end
|
7
10
|
|
8
|
-
def query
|
11
|
+
def query
|
9
12
|
db = BomDB.db
|
13
|
+
edition_model = Models::Edition.new(db)
|
14
|
+
edition = edition_model.find(@edition)
|
15
|
+
if edition.nil?
|
16
|
+
raise "Unable to find edition: #{@edition}"
|
17
|
+
end
|
10
18
|
q = db[:verses].
|
11
19
|
join(:books, :book_id => :book_id).
|
12
|
-
join(:
|
13
|
-
join(:contents, :
|
20
|
+
join(:editions).
|
21
|
+
join(:contents, :edition_id => :edition_id, :verse_id => :verses__verse_id).
|
14
22
|
order(:book_sort, :verse_heading, :verse_chapter, :verse_number).
|
15
23
|
select(:book_name, :verse_chapter, :verse_number, :content_body)
|
16
|
-
q.where!(:
|
17
|
-
q.where!(:verse_heading => nil) unless headings
|
24
|
+
q.where!(:editions__edition_id => edition[:edition_id]) if @edition
|
25
|
+
q.where!(:verse_heading => nil) unless @headings
|
18
26
|
q.exclude!(:verses__verse_id => db[:refs].select(:verse_id).where(:ref_name => @exclude)) if @exclude
|
19
|
-
# require 'byebug'; byebug
|
20
|
-
# p q
|
21
27
|
q
|
22
28
|
end
|
23
29
|
|
24
|
-
def
|
30
|
+
def each(&block)
|
31
|
+
query.each(&block)
|
32
|
+
end
|
33
|
+
|
34
|
+
def print(verse_format: nil, body_format: nil, sep: ' ', linesep: "\n", io: $stdout)
|
25
35
|
shown = false
|
36
|
+
verse_format ||= lambda{ |book, chapter, verse| "#{book}#{sep}#{chapter}:#{verse}" }
|
37
|
+
body_format ||= lambda{ |body| body + linesep }
|
26
38
|
query.each do |row|
|
27
39
|
shown = true
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
].compact.join(sep)
|
40
|
+
io.print verse_format[
|
41
|
+
row[:book_name],
|
42
|
+
row[:verse_chapter],
|
43
|
+
row[:verse_number]
|
44
|
+
]
|
45
|
+
io.print sep
|
46
|
+
io.print body_format[ row[:content_body] ]
|
36
47
|
end
|
37
|
-
puts "Nothing found" unless shown
|
48
|
+
io.puts "Nothing found" unless shown
|
38
49
|
end
|
39
50
|
end
|
40
51
|
end
|
data/lib/bomdb/schema.rb
CHANGED
@@ -1,62 +1,61 @@
|
|
1
1
|
require 'set'
|
2
|
+
require 'sequel'
|
3
|
+
require 'fileutils'
|
2
4
|
|
3
5
|
module BomDB
|
4
6
|
class Schema
|
7
|
+
attr_reader :db
|
8
|
+
|
5
9
|
def initialize(db)
|
6
10
|
@db = db
|
7
11
|
end
|
8
12
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
def reset(*table_names)
|
14
|
-
drop_tables(table_names)
|
15
|
-
create_tables(table_names)
|
13
|
+
# create a new database
|
14
|
+
def self.create(db_path, tables = :all)
|
15
|
+
FileUtils.rm(db_path) if File.exist?(db_path)
|
16
|
+
new(Sequel.sqlite(db_path)).create_tables(tables)
|
16
17
|
end
|
17
18
|
|
18
|
-
def
|
19
|
-
(
|
20
|
-
@db.drop_table(name) if @db.tables.include?(name)
|
21
|
-
end
|
19
|
+
def has_tables?(tables)
|
20
|
+
Set.new(@db.tables) >= Set.new(tables)
|
22
21
|
end
|
23
22
|
|
24
|
-
def create_tables(
|
23
|
+
def create_tables(tables = :all)
|
25
24
|
@db.create_table(:books) do
|
26
25
|
primary_key :book_id
|
27
26
|
|
28
27
|
string :book_name, :unique => true
|
29
28
|
string :book_group
|
30
29
|
integer :book_sort
|
31
|
-
end if include?(
|
30
|
+
end if include?(tables, :books)
|
32
31
|
|
33
32
|
@db.create_table(:verses) do
|
34
33
|
primary_key :verse_id
|
35
34
|
foreign_key :book_id, :books
|
36
35
|
|
37
36
|
integer :verse_chapter, :null => true
|
38
|
-
integer :verse_number,
|
37
|
+
integer :verse_number, :null => true
|
39
38
|
integer :verse_heading, :null => true
|
40
39
|
|
41
40
|
index [:book_id, :verse_chapter, :verse_number, :verse_heading], :unique => true
|
42
|
-
end if include?(
|
41
|
+
end if include?(tables, :verses)
|
43
42
|
|
44
|
-
@db.create_table(:
|
45
|
-
primary_key :
|
43
|
+
@db.create_table(:editions) do
|
44
|
+
primary_key :edition_id
|
46
45
|
|
47
|
-
|
48
|
-
|
49
|
-
end if include?(
|
46
|
+
integer :edition_year
|
47
|
+
string :edition_name, :unique => true
|
48
|
+
end if include?(tables, :editions)
|
50
49
|
|
51
50
|
@db.create_table(:contents) do
|
52
51
|
primary_key :content_id
|
53
|
-
foreign_key :
|
52
|
+
foreign_key :edition_id, :editions
|
54
53
|
foreign_key :verse_id, :verses
|
55
54
|
|
56
55
|
string :content_body
|
57
56
|
|
58
|
-
index [:
|
59
|
-
end if include?(
|
57
|
+
index [:edition_id, :verse_id], :unique => true
|
58
|
+
end if include?(tables, :contents)
|
60
59
|
|
61
60
|
@db.create_table(:refs) do
|
62
61
|
primary_key :ref_id
|
@@ -71,7 +70,7 @@ module BomDB
|
|
71
70
|
integer :ref_page_end
|
72
71
|
boolean :ref_is_parallel
|
73
72
|
boolean :ref_is_quotation
|
74
|
-
end if include?(
|
73
|
+
end if include?(tables, :refs)
|
75
74
|
|
76
75
|
@db.create_table(:notes) do
|
77
76
|
primary_key :note_id
|
@@ -79,14 +78,15 @@ module BomDB
|
|
79
78
|
|
80
79
|
string :note_highlight
|
81
80
|
string :note_body
|
82
|
-
end if include?(
|
81
|
+
end if include?(tables, :notes)
|
82
|
+
|
83
|
+
self
|
83
84
|
end
|
84
85
|
|
85
86
|
protected
|
86
87
|
|
87
|
-
def include?(
|
88
|
-
|
89
|
-
table_names.map{ |n| n.to_sym }.include?(name.to_sym)
|
88
|
+
def include?(tables, name)
|
89
|
+
tables == :all || tables.map{ |n| n.to_sym }.include?(name.to_sym)
|
90
90
|
end
|
91
91
|
end
|
92
92
|
end
|