iev 0.3.0 → 0.3.3

data/lib/iev/data_conversions.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# (c) Copyright 2020 Ribose Inc.
+#
+
+module IEV
+  module DataConversions
+    refine String do
+      def decode_html!
+        replace(decode_html)
+        nil
+      end
+
+      def decode_html
+        HTMLEntities.new(:expanded).decode(self)
+      end
+
+      # Normalize various encoding anomalies like `\uFEFF` in strings
+      def sanitize!
+        unicode_normalize!
+        gsub!("\uFEFF", "")
+        gsub!("\u2011", "-")
+        gsub!("\u00a0", " ")
+        gsub!(/[\u2000-\u2006]/, " ")
+        strip!
+        nil
+      end
+
+      # @see sanitize!
+      def sanitize
+        dup.tap(&:sanitize!)
+      end
+
+      def to_three_char_code
+        IEV::Iso639Code.three_char_code(self).first
+      end
+    end
+  end
+end

data/lib/iev/db.rb

@@ -1,7 +1,7 @@
 # require 'pstore'
 require_relative "db_cache"
 
-module Iev
+module IEV
   # Cache class.
   class Db
     # @param global_cache [String] filename of global DB
@@ -43,12 +43,12 @@ module Iev
 
     # @return [Hash]
     def new_bib_entry(code, lang)
-      Iev.get(code, lang)
+      IEV.get(code, lang)
     end
 
     # @param dir [String] DB dir
     # @param global [TrueClass, FalseClass]
-    # @return [Iev::DbCache, nil]
+    # @return [IEV::DbCache, nil]
     def open_cache_biblio(dir, global: true)
       return nil if dir.nil?
 

data/lib/iev/db_cache.rb

@@ -1,6 +1,6 @@
 require "fileutils"
 
-module Iev
+module IEV
   class DbCache
     # @return [String]
     attr_reader :dir
@@ -73,7 +73,7 @@ module Iev
     end
 
     # Set version of the DB to the gem version.
-    # @return [Iev::DbCache]
+    # @return [IEV::DbCache]
     def set_version
       File.write "#{@dir}/version", VERSION, encoding: "utf-8"
       self

data/lib/iev/db_writer.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+# (c) Copyright 2020 Ribose Inc.
+#
+
+module IEV
+  class DbWriter
+    include CLI::UI
+    using DataConversions
+
+    attr_reader :db
+
+    def initialize(db)
+      @db = db
+    end
+
+    def import_spreadsheet(file)
+      Profiler.measure("xlsx-import") do
+        workbook = open_workbook(file)
+        row_enumerator = workbook.sheets.first.simple_rows.each
+
+        title_row = row_enumerator.next
+        symbolized_title_row = title_row.compact.transform_values(&:to_sym)
+
+        create_table(symbolized_title_row.values)
+
+        loop do
+          row = row_enumerator.next
+          next if row.empty?
+          data = prepare_data(row, symbolized_title_row)
+          display_progress(data)
+          insert_data(data)
+        end
+      end
+    end
+
+    private
+
+    def open_workbook(file)
+      info "Opening spreadsheet..."
+      Creek::Book.new(file)
+    end
+
+    # Creates a database table which is going to be filled with data extracted
+    # from the spreadsheet.
+    #
+    # Note that columns are defined as +VARCHAR(255)+, but they can store
+    # strings of any length without truncating, see:
+    # https://www.sqlite.org/faq.html#q9
+    def create_table(column_names)
+      db.create_table!(:concepts) do
+        column_names.each { |cn| column cn, String }
+        primary_key column_names[0..1], name: :iev_pk
+        index column_names[0]
+        index column_names[1]
+      end
+    end
+
+    # Replaces A, B, C… keys with real column names and sanitizes cell
+    # content.
+    def prepare_data(row, title_row)
+      data = row.dup
+      data.transform_keys! { |k| title_row[k] }
+      data.transform_values! { |v| v&.sanitize }
+      data
+    end
+
+    def display_progress(data)
+      ievref = data[:IEVREF]
+      lang = data[:LANGUAGE].to_three_char_code
+      set_ui_tag "#{ievref} (#{lang})"
+      progress "Importing term #{ievref} (#{lang})..."
+    end
+
+    def insert_data(data)
+      db[:concepts].insert(data)
+    rescue Sequel::UniqueConstraintViolation
+      warn "Duplicated (TERMID, LANGUAGE) pair, skipping"
+    end
+  end
+end