sp2db 0.0.3

data/lib/sp2db/base_table.rb

@@ -0,0 +1,238 @@
+module Sp2db
+  class BaseTable
+
+    attr_accessor :name,
+                  :sheet_name,
+                  :worksheet,
+                  :find_columns,
+                  :spreadsheet_id,
+                  :client
+
+    def initialize opts={}
+
+      if opts[:name].blank? && opts[:sheet_name].blank?
+        raise "Must specify at least one of name or sheet name"
+      end
+
+      opts.each do |k, v|
+        self.send "#{k}=", v
+      end
+
+      self.sheet_name ||= opts[:sheet_name] = config[:sheet_name] || worksheet.try(:title)
+    end
+
+    def active_record?
+      false
+    end
+
+    # Table name
+    def name
+      @name ||= sheet_name.try(:to_sym) || raise("Name cannot be nil")
+    end
+
+    def spreadsheet_id
+      @spreadsheet_id ||= config[:spreadsheet_id] || Sp2db.config.spreadsheet_id
+    end
+
+    def name=n
+      @name = n&.to_sym
+    end
+
+    def find_columns
+      @find_columns ||= config[:find_columns] || Sp2db.config.default_find_columns
+    end
+
+    def required_columns
+      @required_columns ||= config[:required_columns] || []
+    end
+
+    def client
+      @client = Sp2db.client
+    end
+
+    def spreadsheet
+      client.spreadsheet spreadsheet_id
+    end
+
+    def sheet_name
+      @sheet_name ||= (config[:sheet_name] || name)&.to_sym
+    end
+
+    def worksheet
+      @worksheet = spreadsheet.worksheet_by_name(self.sheet_name.to_s)
+    end
+
+    def sp_data
+      retries = 2
+      raw_data = CSV.parse worksheet.export_as_string
+      data = process_data raw_data, source: :sp
+      data
+    end
+
+    def csv_data
+      raw_data = CSV.parse File.open(csv_file)
+      data = process_data raw_data, source: :csv
+      data
+    end
+
+    def header_row
+      # @header_row ||= config[:header_row] || 0
+      0
+    end
+
+    def csv_folder
+      folder = "#{Sp2db.config.export_location}/csv"
+      FileUtils.mkdir_p folder
+      folder
+    end
+
+    def csv_file
+      "#{csv_folder}/#{name}.csv"
+    end
+
+    def sp_to_csv opts={}
+      write_csv to_csv(sp_data)
+    end
+
+    def write_csv data
+      File.open csv_file, "wb" do |f|
+        f.write data
+      end
+      csv_file
+    end
+
+    # Array of hash data to csv format
+    def to_csv data
+      attributes = data.first&.keys || []
+
+      CSV.generate(headers: true) do |csv|
+        csv << attributes
+
+        data.each do |row|
+          csv << attributes.map do |att|
+            row[att]
+          end
+        end
+      end
+    end
+
+    # Global config
+    def config
+      {}.with_indifferent_access
+    end
+
+    def process_data raw_data, opts={}
+      raw_data = data_transform raw_data, opts unless opts[:skip_data_transform]
+      raw_data = raw_filter raw_data, opts unless opts[:skip_data_filter]
+      data = call_process_data raw_data, opts
+      data
+    end
+
+
+    # Tranform data to standard csv format
+    def data_transform raw_data, opts={}
+      if config[:data_transform].present?
+        config[:data_transform].call *args, &block
+      else
+        raw_data
+      end
+    end
+
+    protected
+    # Remove header which starts with "#"
+    def valid_header? h
+      h.present? && !h.match("^#.*")
+    end
+
+    # Header with "!" at the beginning or ending is required
+    def require_header? h
+      h.present? && (h.match("^!.*") || h.match(".*?!$"))
+    end
+
+    # Convert number string to number
+    def standardize_cell_val v
+      v = ((float = Float(v)) && (float % 1.0 == 0) ? float.to_i : float) rescue v
+      v = v.force_encoding("UTF-8") if v.is_a?(String)
+      v
+    end
+
+    def call_process_data raw_data, opts={}
+      data = raw_data
+      if (data_proc = config[:process_data]).present?
+        data = data_proc.call raw_data
+      end
+      data
+    end
+
+    # Remove uncessary columns and invalid rows from csv format data
+    def raw_filter raw_data, opts={}
+      raw_header = raw_data[header_row].map.with_index do |h, idx|
+        is_valid = valid_header?(h)
+        {
+          idx: idx,
+          is_remove: !is_valid,
+          is_required: require_header?(h),
+          name: is_valid && h.gsub(/\s*/, '').gsub(/!/, '').downcase
+        }
+      end
+
+      rows = raw_data[(header_row + 1)..-1].map.with_index do |raw, rdx|
+        row = {}.with_indifferent_access
+        raw_header.each do |h|
+          val = raw[h[:idx]]
+          next if h[:is_remove]
+          if h[:is_required] && val.blank?
+            row = {}
+            break
+          end
+
+          row[h[:name]] = standardize_cell_val val
+        end
+
+        next if row.values.all?(&:blank?)
+
+        row[:id] = rdx + 1 if find_columns.include?(:id) && row[:id].blank?
+        row
+      end.compact
+         .reject(&:blank?)
+      rows = rows.select do |row|
+        if required_columns.present?
+          required_columns.all? {|col| row[col].present? }
+        else
+          true
+        end
+      end
+
+      rows
+    end
+
+    class << self
+
+      def all_tables
+        ModelTable.all_tables + NonModelTable.all_tables
+      end
+
+
+      def table_by_names *names
+        all_tables = self.all_tables
+        if names.blank?
+          all_tables
+        else
+          names.map do |n|
+            all_tables.find {|tb| tb.name == n.to_sym} || raise("Not found: #{n}")
+          end
+        end
+      end
+
+      def sp_to_csv *table_names
+        table_by_names(*table_names).map(&__method__)
+      end
+
+      def model_table_class
+        ModelTable
+      end
+
+      delegate :sp_to_db, :csv_to_db, to: :model_table_class
+    end
+  end
+end

data/lib/sp2db/client.rb

@@ -0,0 +1,56 @@
+module Sp2db
+  class Client
+
+    include Logging
+
+    attr_accessor :credential, :session
+
+    def initialize
+
+    end
+
+    def config
+      Sp2db.config
+    end
+
+    def credential
+      @credential ||= config.credential
+    end
+
+    def session
+      logger.debug "Init session"
+      unless credential = self.credential
+        return @session = saved_session
+      end
+
+      key = OpenSSL::PKey::RSA.new(credential['private_key'])
+      auth = Signet::OAuth2::Client.new(
+        token_credential_uri: credential['token_uri'],
+        audience: credential['token_uri'],
+        scope: %w(
+          https://www.googleapis.com/auth/drive
+          https://spreadsheets.google.com/feeds/
+        ),
+        issuer: credential['client_email'],
+        signing_key: key
+      )
+
+      auth.fetch_access_token!
+      @session = GoogleDrive.login_with_oauth(auth.access_token)
+    end
+
+
+    def saved_session
+      logger.debug "Use saved session"
+      GoogleDrive.saved_session Sp2db.config.personal_credential,
+                                nil,
+                                client_id: config.client_id,
+                                client_secret: config.client_secret
+    end
+
+    def spreadsheet sid
+      Spreadsheet.new session.spreadsheet_by_key(sid)
+    end
+
+  end
+end

data/lib/sp2db/config.rb

@@ -0,0 +1,82 @@
+module Sp2db
+  class Config
+
+    attr_accessor \
+                  :credential,
+                  :personal_credential,
+                  :client_id,
+                  :client_secret,
+                  :spreadsheet_id,
+                  :export_location,
+                  :default_file_extention,
+                  :import_strategy,
+                  :download_before_import,
+                  :default_extensions,
+                  :exception_handler,
+                  :non_model_tables,
+                  :default_find_columns
+
+
+    DEFAULT = {
+      personal_credential: "credentials/google_credentials.json",
+      import_strategy: :truncate_all,
+      export_location: "db/spreadsheets",
+      default_file_extention: :csv,
+      exception_handler: OpenStruct.new({
+        row_import_error: :raise,
+        table_import_error: :raise,
+      }),
+      non_model_tables: {}.with_indifferent_access,
+      download_before_import: false,
+      default_extensions: :csv,
+      default_find_columns: [:id],
+    }
+
+    SUPPORTED_EXTENSIONS = [:csv]
+
+    def initialize
+      set_default
+    end
+
+    def import_strategy=s
+      s = s.to_sym
+      ImportStrategy.valid! s
+      @import_strategy = s
+    end
+
+    def export_folder
+      FileUtils.mkdir_p export_location
+      export_location
+    end
+
+    # File name or json string or hash
+    def credential=cr
+      if File.exist?(cr) && File.file?(cr)
+        cr = File.read cr
+      end
+
+      @credential = case cr
+        when Hash, ActiveSupport::HashWithIndifferentAccess
+          cr
+        when String
+          JSON.parse cr
+        else
+          raise "Invalid data type"
+      end
+    end
+
+    def default_find_columns= cols
+      @default_find_columns = cols.map &:to_sym
+    end
+
+    private
+
+    Google::Apis.logger.level
+
+    def set_default
+      DEFAULT.each do |k, v|
+        self.send("#{k}=", v)
+      end
+    end
+  end
+end

data/lib/sp2db/exception_handler.rb

@@ -0,0 +1,26 @@
+module Sp2db
+  module ExceptionHandler
+    extend self
+
+    def row_import_error e
+      handle __method__, e
+    end
+
+    def table_import_error e
+      handle __method__, e
+    end
+
+    def handle action, e
+      case action = Sp2db.config.exception_handler[action]
+      when :skip
+        true
+      when :raise
+        raise e
+      when Proc
+        action.call(e)
+      else
+        raise e
+      end
+    end
+  end
+end

data/lib/sp2db/import_concern.rb

@@ -0,0 +1,84 @@
+module Sp2db
+  module ImportConcern
+    extend ActiveSupport::Concern
+
+    included do
+      Sp2db::ModelTable.add_models self
+    end
+
+    module ClassMethods
+
+      def sp2db_options *args, &block
+        if args.first.is_a?(Hash)
+          args.first.each do |k, v|
+            send "sp2db_#{k}", v
+          end
+        else
+          meth = args.shift
+          send "sp2db_#{meth}", *args, &block
+        end
+      end
+
+      def sp2db_config
+        @sp2db_config ||= {}.with_indifferent_access
+      end
+
+      [:find_columns, :required_columns].each do |opt|
+        define_method "sp2db_#{opt}" do |*cols|
+          cols = cols&.flatten
+          sp2db_config[opt] = cols.map(&:to_sym) if cols.present?
+          sp2db_config[opt]
+        end
+      end
+
+      def sp2db_priority pr=nil
+        sp2db_config[:priority] = pr if pr.present?
+        sp2db_config[:priority]
+      end
+
+      def sp2db_import_strategy s=nil
+        if s.present?
+          s = s.to_sym
+          ImportStrategy.valid! s
+          sp2db_config[:import_strategy] = s
+        end
+
+        sp2db_config[:import_strategy]
+      end
+
+      def sp2db_sheet_name s=nil
+        sp2db_config[:sheet_name] = s.to_sym if s.present?
+        sp2db_config[:sheet_name]
+      end
+
+      def sp2db_header_row s=nil
+        sp2db_config[:header_row] = s if s.present?
+        sp2db_config[:header_row]
+      end
+
+      def sp2db_spreadsheet_id s=nil
+        sp2db_config[:spreadsheet_id] = s if s.present?
+        sp2db_config[:spreadsheet_id]
+      end
+
+      [
+        :data_transform,
+        :process_data,
+        :before_import_row,
+        :after_import_row,
+        :after_import_table,
+      ].each do |option|
+        define_method "sp2db_#{option}" do |method=nil, &block|
+          sp2db_config[option] = if method.present?
+            method.is_a?(Proc) ? method : method.to_sym
+          elsif block.present?
+            block
+          else
+            sp2db_config[option]
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/sp2db/import_strategy.rb

@@ -0,0 +1,157 @@
+module Sp2db
+  module ImportStrategy
+
+    extend self
+
+    # Add strategy
+    def add label, strategy=nil, &block
+      strategy ||= Class.new(Base)
+      strategy.class_eval(&block) if block_given?
+      strategies[label.to_sym] = strategy
+    end
+
+    # @!strategies [ro] strategies
+    def strategies
+      @strategies ||= {}.with_indifferent_access
+    end
+
+    def strategy_by_name name
+      strategies[name.to_s] || raise("Invalid import strategy: #{name}")
+    end
+
+    def labels
+      strategies.keys.map(&:to_sym)
+    end
+
+    def valid! s
+      raise "Unsuported strategies" unless labels.include?(s.to_sym)
+      true
+    end
+
+    class Base
+
+      include Logging
+
+      attr_accessor :table, :rows, :result
+
+      delegate :model,
+               :find_columns,
+                to: :table
+
+      def initialize table, rows
+        self.table = table
+        self.rows = rows
+      end
+
+      def result
+        @result ||= {
+          records: [],
+          errors: [],
+        }.with_indifferent_access
+      end
+
+      def errors
+        result[:errors]
+      end
+
+      def records
+        result[:records]
+      end
+
+      def before_import
+        logger.debug "Run before import table: #{self.table.name}"
+      end
+
+      def find_db_row row
+        if find_columns.present?
+          cond = {}
+          find_columns.each do |col|
+            cond[col] = row[col]
+          end
+          model.find_by cond
+        else
+          nil # nil to skip
+        end
+      end
+
+      def set_record_value record, row
+        row.each do |k, v|
+          record.send("#{k}=", v)
+        end
+        record
+      end
+
+      def import_row row
+        record = find_db_row(row) || model.new(row)
+        record = set_record_value record, row
+        return unless record.present?
+        record.save! if record.new_record? || record.changed?
+        record
+      end
+
+      def after_import
+        logger.debug "Run after import table: #{self.table.name}"
+      end
+
+      def import
+        logger.debug "Start import table: #{self.table.name}"
+        ActiveRecord::Base.transaction(requires_new: true) do
+          before_import
+          rows.each do |row|
+            row = row.clone
+            begin
+              table.before_import_row row
+              record = import_row row
+              records << record
+              table.after_import_row record
+            rescue ActiveRecord::ActiveRecordError => e
+              logger.error e.try(:message)
+              errors << {
+                message: e.try(:message),
+                exception: e,
+                row: row,
+                table: table.name,
+              }
+              next unless ExceptionHandler.row_import_error e
+            end
+          end
+          after_import
+          table.after_import_table result
+          logger.debug "Import finished: #{self.table.name}"
+          return result
+        end
+
+      end
+
+    end
+  end
+
+  ImportStrategy.add :truncate_all do
+    def before_import
+      logger.info "Truncte all data: #{self.table.name}"
+      model.all.delete_all
+    end
+
+    def find_db_row row
+      nil
+    end
+  end
+
+  ImportStrategy.add :overwrite do
+  end
+
+  ImportStrategy.add :fill_empty do
+    def set_record_value record, row
+      row.each do |k, v|
+        record.send("#{k}=", v) if record.send(k).blank?
+      end
+      record
+    end
+  end
+
+  ImportStrategy.add :skip do
+    def set_record_value record, row
+      record
+    end
+  end
+end

data/lib/sp2db/logging.rb

@@ -0,0 +1,29 @@
+module Sp2db
+  module Logging
+
+    extend self
+
+    # @!attribute [rw] logger
+    # @return [Logger] The logger.
+    def logger
+      @logger ||= get_logger
+    end
+
+    # @return [Logger]
+    def logger= l
+      @logger = l
+    end
+
+    # @return [Logger]
+    def get_logger
+      if  defined?(::Rails) && ::Rails.respond_to?(:logger) && !::Rails.logger.nil?
+        ::Rails.logger
+      else
+        logger = Logger.new($stdout)
+        logger.level = Logger::WARN
+        logger
+      end
+    end
+
+  end
+end