flydata 0.2.17 → 0.2.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a3b273dc439a9c6c58398ed75794f67ab7fcd785
-  data.tar.gz: 20a9d323a93ebb65865e5dbee86750d3c71b8dbe
+  metadata.gz: e60423bcd2e535f420b6cffae38bb02d4c4183b8
+  data.tar.gz: eff1e3250db8d8e036c977f7c5c0fb50232d4fe2
 SHA512:
-  metadata.gz: 1596629dd6c036e157db63f20ffef8fb84f9201f1677b201892e8b59b2f81c5178535a58a3b2f6a4e35fca908ac87a6b5cc3d8af10ecdbd8408734532a9f4a57
-  data.tar.gz: c113a7bdce48f2d683769b50c134efdc6925e6b40897ce14e90ac3c51cf8e54d6c018d1c9ef967ced2e7f957bb78584d7e0fdf10eac182ab3853757d3c6ec369
+  metadata.gz: e30e9303ae3bfa61ccd55eda8837c3606b6502e3523cd4d476068f4ac34b7814ed4fd4858e3af0327705d6f26d2a17d3ec205277acfd92844c919f15ecce15be
+  data.tar.gz: d26ad0a84abdeaac414f75c357a5a10f5b6c918659f7be55cd17603940c8d9bf76bbe67535f2ed75ce891723c01554a2c6d2bbcc167282cc0380b6ace81fd748

data/VERSION

@@ -1 +1 @@
-0.2.17
+0.2.18

data/flydata-core/.gitignore

@@ -0,0 +1,16 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/flydata-core/.rspec

@@ -0,0 +1 @@
+--color

data/flydata-core/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 2.0.0
+  - 2.1.0
+  - 2.1.1

data/flydata-core/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+# for dependencies
+group :development, :test do
+   gem 'timecop'
+   gem 'ruby-prof'
+   gem 'rubocop'
+   gem 'rspec', '~> 3.1'
+   gem 'simplecov', :require => false
+end
+

data/flydata-core/Gemfile.lock

@@ -0,0 +1,51 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.0.0)
+    astrolabe (1.3.0)
+      parser (>= 2.2.0.pre.3, < 3.0)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    multi_json (1.10.1)
+    parser (2.2.0.pre.8)
+      ast (>= 1.1, < 3.0)
+      slop (~> 3.4, >= 3.4.5)
+    powerpack (0.0.9)
+    rainbow (2.0.0)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+    rubocop (0.27.1)
+      astrolabe (~> 1.3)
+      parser (>= 2.2.0.pre.7, < 3.0)
+      powerpack (~> 0.0.6)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.4)
+    ruby-prof (0.15.2)
+    ruby-progressbar (1.7.0)
+    simplecov (0.9.1)
+      docile (~> 1.1.0)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    slop (3.6.0)
+    timecop (0.7.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rspec (~> 3.1)
+  rubocop
+  ruby-prof
+  simplecov
+  timecop

data/flydata-core/lib/flydata-core/core_ext/module/include.rb

@@ -0,0 +1,5 @@
+class Module
+  def include_unless_included(m)
+    include m unless self.include?(m)
+  end
+end

data/flydata-core/lib/flydata-core/core_ext/module.rb

@@ -0,0 +1 @@
+require_relative 'module/include'

data/flydata-core/lib/flydata-core/core_ext/object/prepend.rb

@@ -0,0 +1,17 @@
+module PrependDependenciesCheckable
+  def check_prepend_dependencies(target, dependencies)
+    order = Array(dependencies[:before]) + [target] + Array(dependencies[:after])
+    self.class.ancestors.reverse.each { |cls| order.shift if cls == order.first }
+    order.empty?
+  end
+
+  def check_prepend_dependencies!(target, dependencies)
+    unless check_prepend_dependencies(target, dependencies)
+      raise "Invalid prepend dependencies. target:#{target} ancestors:#{self.class.ancestors} dependencies:#{dependencies}"
+    end
+  end
+end
+
+class Object
+  include PrependDependenciesCheckable
+end

data/flydata-core/lib/flydata-core/core_ext/object.rb

@@ -0,0 +1 @@
+require_relative 'object/prepend'

data/flydata-core/lib/flydata-core/core_ext.rb

@@ -0,0 +1,3 @@
+Dir["#{File.dirname(__FILE__)}/core_ext/*.rb"].each do |path|
+  require path
+end

data/flydata-core/lib/flydata-core/errors.rb

@@ -0,0 +1,334 @@
+require 'flydata-core/thread_context'
+
+module FlydataCore
+
+
+# Error 12xx are derived from AWS.
+# Error 100xx are FlyData errros.
+module ErrorCode
+  VALUE_OUT_OF_RANGE_ERROR = 1204
+  INVALID_DATE_FORMAT_ERROR = 1205
+  INVALID_TIMESTAMP_FORMAT_ERROR = 1206
+  INVALID_NUMBER_ERROR = 1207
+  INVALID_FLOAT_ERROR = 1208
+  INVALID_DECIMAL_ERROR = 1209
+  INVALID_BOOLEAN_ERROR = 1210
+  NOT_NULL_VIOLATION_ERROR = 1213
+  INVALID_VARCHAR_ERROR = 1214
+  INVALID_CHAR_ERROR = 1215
+  INVALID_UTF8_CHARACTER_ERROR = 1220
+  INVALID_DATA_FORMAT_ERROR = 1201
+  MORE_COLUMNS_ERROR = 1202
+  LESS_COLUMNS_ERROR = 1203
+  NO_DATA_ERROR = 1211
+  RESERVED_TABLE_NAME_ERROR = 10001
+  RESERVED_COLUMN_NAME_ERROR = 10002
+  TABLE_MISSING_ERROR = 10003
+  AUTHENTICATION_ERROR = 10004
+  RETRY_LIMIT_EXCEED_ERROR = 10005
+  INFORMATION_SCHEMA_ERROR = 10006
+  TABLE_REVISION_ERROR = 10007
+  S3_ACCESS_ERROR = 10008
+  REDSHIFT_ACCESS_ERROR = 10009
+  BREAKING_ALTER_TABLE_ERROR = 10010
+  INTERNAL_ERROR = 1200
+end
+
+# Error Level
+# 0 Emergency: system is unusable
+# 1 Alert: action must be taken immediately
+# 2 Critical: critical conditions
+# 3 Error: error conditions
+# 4 Warning: warning conditions
+# 5 Notice: normal but significant condition
+# 6 Informational: informational messages
+# 7 Debug: debug-level messages
+module ErrorLevel
+  ERROR = 3  # failed to load a record
+  WARN = 4   # loaded a record but a record is modified
+end
+
+# RetryableError
+
+class RetryableError < StandardError
+  def initialize(original_exception, retry_alert_limit = nil)
+    @original_exception = original_exception
+    @retry_alert_limit = retry_alert_limit
+  end
+  attr_reader :original_exception
+  attr_reader :retry_alert_limit
+end
+
+# DataDeliveryError
+# data_node_id, data_entry_id, chunk_identifier, table_name, err_code, err_reason, err_level
+
+class DataDeliveryError < StandardError
+  def initialize(error_reason, error_content = {})
+    error_content = if error_reason.kind_of?(String)
+                      { err_reason: error_reason}.merge(error_content)
+                    elsif error_reason.kind_of?(Hash)
+                      error_reason.merge(error_content)
+                    else
+                      raise ArgumentError.new(
+                        "Invalid argument. First parameter must be string or hash object.")
+                    end
+    @error_content = default_params.
+      merge(context_params).
+      merge(error_content).
+      merge(overwrite_params)
+    filter_content(@error_content)
+    @error_content[:err_code] = err_code
+  end
+  attr_reader :error_content
+  def default_params
+    if h = FlydataCore::ThreadContext.parameters
+      {data_node_id: h[:node_id],
+       data_entry_id: h[:data_entry_id],
+       table_name: h[:table_name],
+       chunk_identifier: h[:chunk_identifier],
+       err_level: ErrorLevel::ERROR}
+    else
+      {}
+    end
+  end
+  def context_params
+    DataDeliveryErrorThreadContext.context_params || {}
+  end
+  def overwrite_params
+    {}
+  end
+  def err_code
+    raise NoMethodError, "default_parameters is not implemented"
+  end
+  def warn?
+    err_level == ErrorLevel::WARN
+  end
+  def err_level
+    @error_content[:err_level]
+  end
+  def to_s
+    if r = @error_content[:err_reason]
+      r
+    else
+      super
+    end
+  end
+
+  MAX_CONTENT_SIZE = 1024 # 1kb
+
+  def filter_content(error_content = @error_content)
+    error_content.each do |k, value|
+      if value.kind_of?(String) && value.length > MAX_CONTENT_SIZE
+        error_content[k] = value[0..MAX_CONTENT_SIZE-1]
+      end
+    end
+  end
+end
+
+# + RecordDeliveryError
+# data_node_id, data_entry_id, chunk_identifier, table_name, err_code, err_reason, err_level, record_no, raw_record
+
+class RecordDeliveryError < DataDeliveryError
+  def to_s
+    "#{super} raw_record=(#{@error_content[:raw_record]})"
+  end
+end
+class InvalidDataFormatError < RecordDeliveryError
+  def err_code; ErrorCode::INVALID_DATA_FORMAT_ERROR; end
+end
+class MoreColumnsError < RecordDeliveryError
+  def err_code; ErrorCode::MORE_COLUMNS_ERROR; end
+end
+class LessColumnsError < RecordDeliveryError
+  def err_code; ErrorCode::LESS_COLUMNS_ERROR; end
+end
+class NoDataError < RecordDeliveryError
+  def err_code; ErrorCode::NO_DATA_ERROR; end
+end
+class ReservedTableNameError < RecordDeliveryError
+  def err_code; ErrorCode::RESERVED_TABLE_NAME_ERROR; end
+end
+class ReservedColumnNameError < RecordDeliveryError
+  def err_code; ErrorCode::RESERVED_COLUMN_NAME_ERROR; end
+end
+class TableMissingError < RecordDeliveryError
+  def err_code; ErrorCode::TABLE_MISSING_ERROR; end
+end
+class RecordDeliveryInternalError < RecordDeliveryError
+  def err_code; ErrorCode::INTERNAL_ERROR; end
+end
+
+# information schema, especially flydata_ctl_columns table, is broken
+class InformationSchemaError < RecordDeliveryError
+  def err_code; ErrorCode::INFORMATION_SCHEMA_ERROR; end
+end
+
+# No table information found in the database
+class MissingTableInformationError < InformationSchemaError
+  def err_code; ErrorCode::INFORMATION_SCHEMA_ERROR; end
+end
+
+# Table's revision does not match the record's revision.
+class TableRevisionError < RecordDeliveryError
+  def err_code; ErrorCode::TABLE_REVISION_ERROR; end
+end
+
+# Unsupported ALTER TABLE which breaks the tables sync consistency.
+class BreakingAlterTableError < RecordDeliveryError
+  def err_code; ErrorCode::BREAKING_ALTER_TABLE_ERROR; end
+end
+
+#   + BadValueError
+# data_node_id, data_entry_id, chunk_identifier, table_name, err_code, err_reason, err_level, record_no, raw_record, colname, type, raw_value, modified_value
+
+class BadValueError < RecordDeliveryError
+  def modified_value
+    @error_content[:modified_value]
+  end
+
+  def has_modified_value?
+    @error_content.has_key?(:modified_value)
+  end
+
+  def to_s
+    "#{super} column=(#{@error_content[:col_name]}) value=(#{@error_content[:raw_value]})"
+  end
+end
+class ValueOutOfRangeError < BadValueError
+  def err_code; ErrorCode::VALUE_OUT_OF_RANGE_ERROR; end
+end
+class InvalidDateFormatError < BadValueError
+  def err_code; ErrorCode::INVALID_DATE_FORMAT_ERROR; end
+end
+class InvalidTimestampFormatError < BadValueError
+  def err_code; ErrorCode::INVALID_TIMESTAMP_FORMAT_ERROR; end
+end
+class InvalidNumberError < BadValueError
+  def err_code; ErrorCode::INVALID_NUMBER_ERROR; end
+end
+class InvalidFloatError < BadValueError
+  def err_code; ErrorCode::INVALID_FLOAT_ERROR; end
+end
+class InvalidDecimalError < BadValueError
+  def err_code; ErrorCode::INVALID_DECIMAL_ERROR; end
+end
+class InvalidBooleanError < BadValueError
+  def err_code; ErrorCode::INVALID_BOOLEAN_ERROR; end
+end
+class NotNullViolationError < BadValueError
+  def err_code; ErrorCode::NOT_NULL_VIOLATION_ERROR; end
+end
+class InvalidVarcharError < BadValueError
+  def err_code; ErrorCode::INVALID_VARCHAR_ERROR; end
+end
+class InvalidCharError < BadValueError
+  def err_code; ErrorCode::INVALID_CHAR_ERROR; end
+end
+class InvalidUtf8CharacterError < BadValueError
+  def err_code; ErrorCode::INVALID_UTF8_CHARACTER_ERROR; end
+end
+
+
+# + ChunkDeliveryError
+# data_node_id, data_entry_id, chunk_identifier, table_name, err_code, err_reason, err_level
+
+class ChunkDeliveryError < DataDeliveryError
+end
+
+class AuthenticationError < ChunkDeliveryError
+  # end_point_url
+  def err_code; ErrorCode::AUTHENTICATION_ERROR; end
+end
+
+class RetryLimitExceedError < ChunkDeliveryError
+  def err_code; ErrorCode::RETRY_LIMIT_EXCEED_ERROR; end
+end
+
+class ChunkDeliveryInternalError < ChunkDeliveryError
+  def err_code; ErrorCode::INTERNAL_ERROR; end
+end
+
+class UserResourceError < ChunkDeliveryError
+  def default_params
+    super.merge(timestamp: Time.now.utc)
+  end
+  def overwrite_params
+    super.merge(table_name: nil)
+  end
+end
+
+class S3AccessError < UserResourceError
+  def err_code; ErrorCode::S3_ACCESS_ERROR; end
+end
+
+class RedshiftAccessError < UserResourceError
+  def err_code; ErrorCode::REDSHIFT_ACCESS_ERROR; end
+end
+
+
+## Depricated errors
+
+# The record is valid but not supported.
+class UnsupportedRecordFormat < StandardError; end
+
+# Table Def Error
+class TableDefError < StandardError
+  attr_reader :err_hash
+  def initialize(err_hash)
+    @err_hash = err_hash
+  end
+end
+
+## Error container
+
+class DataDeliveryErrorThreadContext
+  THREAD_LOCAL_KEY_ERROR_LIST = :flydata_data_delivery_error_list
+  THREAD_LOCAL_KEY_RECORD_LIMIT = "#{THREAD_LOCAL_KEY_ERROR_LIST}_limit".to_sym
+  DEFAULT_RECORD_LIMIT = 10
+  THREAD_LOCAL_KEY_CONTEXT_PARAMS = "#{THREAD_LOCAL_KEY_ERROR_LIST}_context_params".to_sym
+
+  def self.initialize(record_limit = DEFAULT_RECORD_LIMIT)
+    reset
+    Thread.current[THREAD_LOCAL_KEY_RECORD_LIMIT] = record_limit
+  end
+
+  def self.add_error(error)
+    return if error_list.include?(error)
+    if error_list.count < record_limit
+      error_list << error
+    end
+    #TODO: write warning log if record limit
+  end
+
+  def self.reset
+    Thread.current[THREAD_LOCAL_KEY_ERROR_LIST] = nil
+    Thread.current[THREAD_LOCAL_KEY_RECORD_LIMIT] = nil
+    Thread.current[THREAD_LOCAL_KEY_CONTEXT_PARAMS] = nil
+  end
+
+  def self.record_limit
+    Thread.current[THREAD_LOCAL_KEY_RECORD_LIMIT] || DEFAULT_RECORD_LIMIT
+  end
+
+  def self.error_list
+    unless list = Thread.current[THREAD_LOCAL_KEY_ERROR_LIST]
+      list = Thread.current[THREAD_LOCAL_KEY_ERROR_LIST] = []
+    end
+    list
+  end
+
+  def self.set_context_params(params)
+    Thread.current[THREAD_LOCAL_KEY_CONTEXT_PARAMS] = params
+  end
+
+  def self.add_context_params(params)
+    cur_param = Thread.current[THREAD_LOCAL_KEY_CONTEXT_PARAMS] || {}
+    Thread.current[THREAD_LOCAL_KEY_CONTEXT_PARAMS] = cur_param.merge(params)
+  end
+
+  def self.context_params
+    Thread.current[THREAD_LOCAL_KEY_CONTEXT_PARAMS]
+  end
+end
+
+end

data/flydata-core/lib/flydata-core/logger.rb

@@ -0,0 +1,205 @@
+require 'forwardable'
+
+require 'flydata-core/errors'
+require 'flydata-core/thread_context'
+
+module FlydataCore
+  class LogContext
+    THREAD_LOCAL_KEY_LOG_CONTEXT = 'flydata_log_context'
+    THREAD_LOCAL_KEY_LOGGER = 'flydata_logger'
+
+    class << self
+      extend Forwardable
+      def_delegators :log_context, :[], :[]=, :each, :merge, :merge!, :delete
+
+      def log_context
+        if h = Thread.current[THREAD_LOCAL_KEY_LOG_CONTEXT]
+          h
+        else
+          Thread.current[THREAD_LOCAL_KEY_LOG_CONTEXT] = {}
+        end
+      end
+
+      def reset_log_context
+        Thread.current[THREAD_LOCAL_KEY_LOG_CONTEXT] = nil
+      end
+
+      def logger
+        Thread.current[THREAD_LOCAL_KEY_LOGGER]
+      end
+
+      def set_logger(new_logger)
+        Thread.current[THREAD_LOCAL_KEY_LOGGER] = new_logger
+      end
+
+      def reset
+        reset_log_context
+        Thread.current[THREAD_LOCAL_KEY_LOGGER] = nil
+      end
+    end
+  end
+
+  module Logger
+    attr_accessor :logger
+
+    def initialize_log_context(new_logger, log_items)
+      reset_log_context
+      set_log_context_logger(new_logger)
+      set_log_context_items(log_items)
+    end
+
+    def log_context_items
+      LogContext.log_context
+    end
+
+    def log_context_item(k)
+      LogContext[k]
+    end
+
+    def set_log_context_item(k, v)
+      LogContext[k] = v
+    end
+
+    def delete_log_context_item(*keys)
+      keys.each do |k|
+        LogContext.delete(k)
+      end
+    end
+
+    def set_log_context_items(items)
+      reset_log_context_items
+      add_log_context_items(items)
+    end
+
+    def add_log_context_items(items)
+      LogContext.merge!(items)
+    end
+
+    def reset_log_context_items
+      LogContext.reset_log_context
+    end
+
+    def log_context_logger
+      LogContext.logger
+    end
+
+    def set_log_context_logger(new_logger)
+      LogContext.set_logger(new_logger)
+    end
+
+    def reset_log_context
+      LogContext.reset
+    end
+
+    # for override
+    def custom_log_items
+      {}
+    end
+
+    def filter_log_params(log_params)
+      log_params
+    end
+
+    # log methods
+    def log_common(level, message, log_params = {}, options = {})
+      # check keys
+      if e = log_params[:error]
+        message += " error_class=#{e.class.to_s} error=\"#{e.to_s.strip.gsub(/\n/, ' ')}\""
+        log_params.delete(:error)
+      end
+
+      # get prefix text
+      prefix = if log_params[:prefix]
+                 log_params[:prefix]
+               elsif log_context_items[:prefix]
+                 log_context_items[:prefix]
+               elsif custom_log_items[:prefix]
+                 custom_log_items[:prefix]
+               else
+                 ''
+               end
+      prefix = "#{prefix} " unless prefix.to_s == ''
+
+      # build suffix text
+      suffix = [build_log_text(filter_log_params(log_params)),
+                build_log_text,
+                build_log_text(custom_log_items)].compact.join(' ')
+      suffix = " #{suffix}" unless suffix.empty?
+
+      msg = "#{prefix}#{message}#{suffix}"
+
+      # add backtrace of error
+      if e and options[:backtrace]
+        backtrace = e.backtrace ? e.backtrace.join("\n") : 'backtrace is empty.'
+        msg += " backtrace:\n#{backtrace}"
+      end
+
+      # get logger
+      lg = options[:logger] || logger || log_context_logger || $log
+      lg.send(level, msg)
+    end
+
+    def log_debug(message, log_params = {}, options = {})
+      log_common(:debug, message, log_params, options)
+    end
+
+    def log_info(message, log_params = {}, options = {})
+      log_common(:info, message, log_params, options)
+    end
+
+    def log_warn(message, log_params = {}, options = {})
+      log_common(:warn, message, log_params, options)
+    end
+
+    def log_error(message, log_params = {}, options = {})
+      log_common(:error, message, log_params, options)
+    end
+
+    def log_error_with_backtrace(message, log_params = {}, options = {})
+      log_common(:error, message, log_params, options.merge(backtrace: true))
+    end
+
+    def build_log_text(items = log_context_items)
+      str_list = []
+      items.each do |k, v|
+        next if k == :prefix
+        case v
+        when Array
+          str_list << "#{k}:[#{v.join(',')}]"
+        when Hash
+          str_list << build_log_text(v)
+        else
+          str_list << if k.to_s.start_with?('__')
+                        v.to_s.empty? ? nil : v.to_s
+                      elsif %w|' " ( [|.include?(v.to_s[0]) or (v.kind_of?(Numeric))
+                        "#{k}:#{v}"
+                      else
+                        "#{k}:\"#{v}\""
+                      end
+        end
+      end
+      ret = str_list.compact.join(' ')
+      ret.empty? ? nil : ret
+    end
+
+    # Log an error for the given exception.  It does not log a RetryableError if
+    # retry_count is less than retry_alert_limit.
+    # It returns the original exception of a RetryableError it has one.
+    # This method requires method 'logger' which returns a Ruby Logger instance or
+    # a compatible one.
+    def log_retryable_error(exception, message = "unknown error occured.", retry_count = -1,
+                  retry_alert_limit = 13)
+      is_retryable_error = false
+      if (exception.kind_of?(FlydataCore::RetryableError))
+        is_retryable_error = true
+        retry_alert_limit = exception.retry_alert_limit unless exception.retry_alert_limit.nil?
+        exception = exception.original_exception unless exception.original_exception.nil?
+      end
+
+      unless is_retryable_error && retry_count < retry_alert_limit
+        log_error_with_backtrace(message, retry_count: retry_count, error: exception)
+      end
+      exception
+    end
+  end
+end