flydata 0.7.5 → 0.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44371bbacace581ba7bfc886a977202c84028ff9
-  data.tar.gz: bbac27c82da13ad6fce04dbbdc420feebc73a17c
+  metadata.gz: 0a74c7dd378333e32076bf9d8dfb6f445f2a6b07
+  data.tar.gz: 4fbc074d26fee614f5315b674159543db96da68b
 SHA512:
-  metadata.gz: 9b52b6990ae5706dfdb79a1a81a10666d09de149562dac3d2f9defb23383625c8c8b429d006c8d9a84978f1362cd74c81f05d0cd5c35b042164af32c06462059
-  data.tar.gz: bfbe4e1c24723bd23fa8650427fa3daed18d7c619ace48e3bcb1e050822a495c80e007d46989758fe6327f20e6a9b4a64f4712e4ab2b15bc44430db8b5fb6043
+  metadata.gz: 5e88618472bc0641320931b46ab357ba02c0f17d678b1b66e5d1ef467e2ca4b0518414d8c2037f6177c310b18c2dbb7825b090649ad6615ec621e687a9355118
+  data.tar.gz: ad9d6fd284b20b55f36ed25e4c86d54fc323e856d691179e7611a87c424f94c892b33dc1ea09d8e35b7b6e49b861e573ce03611272020d69913982ea9c15473e

data/VERSION

@@ -1 +1 @@
-0.7.5
+0.7.6

data/flydata-core/lib/flydata-core/table_def/base.rb

@@ -9,24 +9,28 @@ class Base
     params ? self.new(*params, options) : nil
   end
 
-  def initialize(table_def, table_name, columns, column_def, default_charset,
+  def initialize(table_def, table_name, columns, column_def, unique_keys, default_charset,
                  default_source_charset, comment, options = {})
     @table_def = table_def
     @table_name = table_name
     @columns = columns
     @column_def = column_def
+    @unique_keys = unique_keys
     @default_charset = default_charset
     @default_source_charset = default_source_charset
     @comment = comment
 
+    # example of opts
+    # { :skip_primary_key_check => true,
+    #   :pk_override => { "table_3" => ["id", "group_id"], "table_5" => ["product_id"] }
+    # }
     opts = [:skip_primary_key_check, :pk_override].inject({}) do |h, key|
       v = options[key]
       v ||= options[key.to_s]
       h[key] = v if v
       h
     end
-
-    @pk_override = self.class.override_pk(@table_name, @columns, opts)
+    @pk_override, @pk_status = get_pk_override_with_status(@table_name, @columns, @unique_keys, opts)
     self.class.check_pk(@table_name, @columns, opts)
   end
 
@@ -54,12 +58,59 @@ class Base
              collect{|col| col[:column]}
   end
 
+  def get_pk_override_as_hash
+    @pk_override ? { @table_name => @pk_override } : nil
+  end
+
+  def converting_uk_to_pk?
+    @pk_status == :pk_unique
+  end
+
   private
 
   def self._create(source_data, options)
     raise "subclass must implement"
   end
 
+  # Get pk_override with pk_status
+  # [Arguments]
+  #   - table_name (String):
+  #   - columns (Hash):
+  #         columns that includes all column info from source DB
+  #   - unique_keys (Hash):
+  #         valid unique keys on the table which can use as a primary key
+  #   - opts    (Hash):
+  #         including pk_override values. See a comment above.
+  # [Return values]
+  # pk_status (Symbol)
+  #   :pk_original  using original pk from source DB
+  #   :pk_overridden  using user_pk_override from web server
+  #   :pk_unique    converting unique key (uk) to pk. Need to update the table attributes (UK_AS_PK_OVERRIDE) on web server
+  #   :pk_none      none of above
+  # pk_override (Hash): 
+  #   pk_override or nil
+  def get_pk_override_with_status(table_name, columns, unique_keys, opts)
+
+    pk_override = self.class.override_pk(table_name, columns, opts)
+
+    if pk_override
+      pk_status = :pk_overridden
+    elsif self.class.pk_exist?(@columns)
+      # real primary key exist. Not using pk_override
+      pk_status = :pk_original
+    else
+      # use an unique key as a pk_override if any.
+      new_pk_override = self.class.uk_to_pk_override(table_name, columns, unique_keys)
+      if new_pk_override
+        pk_override = self.class.override_pk(table_name, columns, new_pk_override)
+        pk_status = :pk_unique
+      else
+        pk_status = :pk_none
+      end
+    end
+    [pk_override, pk_status]
+  end
+
   def self.override_pk(table_name, columns, options)
     pk_override = options[:pk_override]
     return nil unless pk_override.kind_of?(Hash) && pk_override[table_name]
@@ -83,11 +134,29 @@ class Base
   end
 
   def self.check_pk(table_name, columns, options)
-    options
-    unless columns.any? {|column| column[:primary_key]} or options[:skip_primary_key_check]
+    unless pk_exist?(columns) || options[:skip_primary_key_check]
       raise TableDefError, {error: "no primary key defined", table: table_name}
     end
   end
+
+  def self.pk_exist?(columns)
+    columns.any? {|column| column[:primary_key]}
+  end
+
+  def self.uk_exist?(columns, unique_keys)
+    unique_keys.any?
+  end
+
+  def self.uk_to_pk_override(table_name, columns, unique_keys)
+    return nil unless uk_exist?(columns, unique_keys)
+
+    # use the first unique key as a substitue of primary key
+    new_pk_override = {}
+    new_pk_override[:pk_override] = { table_name => unique_keys.first }
+    $log.info %Q|Using unique key as primary key: {"#{table_name}"=>#{new_pk_override[:pk_override][table_name].inspect}}|
+
+    new_pk_override
+  end
 end
 
 end

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -90,6 +90,7 @@ class MysqlTableDef < Base
     table_name = nil
     columns = []
     column_def = {}
+    unique_keys = []
     default_charset = nil
     default_source_charset = nil
     comment = nil
@@ -133,7 +134,8 @@ class MysqlTableDef < Base
         elsif stripped_line.start_with?("CONSTRAINT")
           # constraint definition.  No acction required.
         elsif stripped_line.start_with?("UNIQUE KEY")
-          parse_key(line, columns, :unique)
+          unique_key, include_null_column = parse_key(line, columns, :unique)
+          unique_keys << unique_key unless include_null_column
         else
           # "Unknown table definition. Skip. (#{line})"
         end
@@ -142,7 +144,7 @@ class MysqlTableDef < Base
         break
       end
     end
-    position == :after_create_table ? [table_def, table_name, columns, column_def, default_charset, default_source_charset, comment] : nil
+    position == :after_create_table ? [table_def, table_name, columns, column_def, unique_keys, default_charset, default_source_charset, comment] : nil
   end
 
   # Replaces a MySQL string with an empty string ('')
@@ -307,15 +309,18 @@ class MysqlTableDef < Base
     line = /\((?:`.*?`(?:\(.*?\))?(?:,\s*)?)+\)/.match(line)[0]
     keys = line.scan(/`(.*?)`/).collect{|item| item[0]}
 
+    include_null_column = false
     keys.each do |key|
       column = columns.detect {|column|
         column[:column] === key
       }
       raise "Key #{key} must exist in the definition " if column.nil?
       column[type] = true
+      include_null_column = true unless column[:not_null]
     end
-  end
 
+    [keys, include_null_column]
+  end
 end
 
 end

data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb

@@ -123,7 +123,7 @@ class PostgresqlTableDef < Base
   end
 
   def self._create(information_schema_columns, options)
-    table_def = information_schema_columns.inspect
+    table_def = information_schema_columns.collect {|iscol| iscol.first}.inspect
     table_name = nil
     columns = []
     column_def = {}
@@ -132,8 +132,12 @@ class PostgresqlTableDef < Base
     default_charset = 'UTF_8'
     default_charset_postgresql = 'UTF8'
     comment = nil
+    unique_keys_hash = Hash.new {|h, k| h[k] = []}
 
-    information_schema_columns.each do |iscol|
+    information_schema_columns.each do |iscol_arr|
+      # An iscol_arr represents a column. Column information in all elements in iscol_arr is the same.
+      # Only difference between elements is index information.
+      iscol = iscol_arr.first
       column = parse_one_column_def(iscol)
       if table_name
         unless table_name == column[:table]
@@ -145,8 +149,17 @@ class PostgresqlTableDef < Base
       columns << column
       coldef = iscol.inspect
       column_def[column[:column]] = coldef
+
+      iscol_arr.each do |iscol|
+        # gather information for unique keys that this column belongs to.
+        if iscol['is_primary'] == 'f' && iscol['is_unique'] == 't' && iscol['is_valid'] == 't' && iscol['is_partial'] == 'f'
+          unique_keys_hash[iscol['index_name'].to_sym] << iscol['column_name']
+        end
+      end
     end
-    [table_def, table_name, columns, column_def, default_charset, default_charset_postgresql, comment]
+    unique_keys = unique_keys_hash.values
+
+    [table_def, table_name, columns, column_def, unique_keys, default_charset, default_charset_postgresql, comment]
   end
 
   def self.parse_one_column_def(information_schema_column)

data/flydata-core/spec/table_def/base_spec.rb

@@ -5,13 +5,14 @@ module TableDef
 
 describe Base do
   let(:subject_object) { described_class.new(table_def, table_name, columns,
-                           column_def, default_charset,
+                           column_def, unique_keys, default_charset,
                            default_source_charset, comment) }
 
   let(:table_def) { double('table_def') }
   let(:table_name) { double('table_name') }
   let(:columns) { double('columns') }
   let(:column_def) { double('column_def') }
+  let(:unique_keys) { double('unique_keys') }
   let(:default_charset) { double('default_charset') }
   let(:default_source_charset) { double('default_source_charset') }
   let(:comment) { double('comment') }
@@ -27,7 +28,6 @@ describe Base do
         {column: "user_id", primary_key: pk2_value},
       ]
     end
-    
 
     context 'with unordered primary keys' do
       let(:pk1_value) { 't' }

data/lib/flydata/api/data_entry.rb

@@ -164,11 +164,23 @@ module Flydata
       end
 
       # Update validity of tables
-      #  table_validity_hash: {"updated_tables":{ "bad_table": "error reason", "good_table": nil }}
-      #    table "bad_table" will be marked invalid with reason "error reason"
-      #    table "good table" will be marked valid, that is, clear its error reason if it's set.
-      def update_table_validity(data_entry_id, table_validity_hash)
-        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/update_table_validity", {:headers => {:content_type => :json}}, table_validity_hash.to_json)
+      # -- Params ( Agent 0.7.4 and older )
+      # params = {
+      #   updated_tables: { "bad_table"=>"error reason", "good_table"=>nil}
+      # }
+      # -- Params ( Agent 0.7.5 and later )
+      # table_update_hash = {
+      #   updated_tables: {
+      #     "<table-name>" => {
+      #       "invalid_table_reason" => "<error reason>",
+      #       "uk_as_pk" => ['xx', ...],
+      #     }
+      #   }
+      # }
+      # - If value is nil, the setting will be deleted.
+      # - If table's attribute hash doesn't have an attribute key, the setting for the attribute for the table will not be changed
+      def update_table_validity(data_entry_id, table_update_hash)
+        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/update_table_validity", {:headers => {:content_type => :json}}, table_update_hash.to_json)
       end
 
       # Tells the server that an initial sync has completed

data/lib/flydata/command/sync.rb

@@ -15,6 +15,7 @@ require 'flydata/output/forwarder'
 require 'flydata/parser'
 require 'flydata/preference/data_entry_preference'
 require 'flydata/sync_file_manager'
+require 'flydata/table_attribute'
 require 'flydata-core/table_def'
 require 'flydata/table_ddl'
 require 'flydata/event/api_event_sender'
@@ -1181,8 +1182,9 @@ EOM
           raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file"
         end
 
-        flydata_tabledefs, error_list = context.generate_flydata_tabledef(tables,
+        flydata_tabledefs, error_list, uk_as_pk_override = context.generate_flydata_tabledef(tables,
                         skip_primary_key_check: opts.skip_primary_key_check?)
+
         create_flydata_ctl_table = true
         append_only = tables & @append_only_tables
         flydata_tabledefs.each do |flydata_tabledef|
@@ -1191,7 +1193,8 @@ EOM
             flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?, skip_drop_table: skip_drop_table, skip_primary_key_check: opts.skip_primary_key_check?)
           create_flydata_ctl_table = false
         end
-        table_validity_hash = {}
+
+        table_validity_hash = Hash.new {|h,k| h[k] = {}}
         tables_without_error = tables
         unless error_list.empty?
           log_error_stderr("\n\nERROR: FlyData Sync will not sync the following table(s) due to an error.")
@@ -1200,7 +1203,7 @@ EOM
             group_error[error].each do |hash|
               if table = hash[:table]
                 log_error_stderr("  - #{table} (#{error})")
-                table_validity_hash[table] = error
+                table_validity_hash[table][TableAttribute::INVALID_TABLE_REASON] = error
               end
             end
           end
@@ -1215,9 +1218,13 @@ The other tables are ready to sync.  To start sync, run the generated script on
 EOS
           end
         end
+        tables_without_error.each {|table| table_validity_hash[table][TableAttribute::INVALID_TABLE_REASON] = nil }
+
+        # update table validity and uk_as_pk info on web server
+        uk_as_pk_override.each_pair {|table, val| table_validity_hash[table][TableAttribute::UK_AS_PK_OVERRIDE] = val}
+        (tables - uk_as_pk_override.keys).each {|table| table_validity_hash[table][TableAttribute::UK_AS_PK_OVERRIDE] = nil }
 
-        tables_without_error.each {|table| table_validity_hash[table] = nil }
-        flydata.data_entry.update_table_validity(de['id'], {updated_tables: table_validity_hash}) unless table_validity_hash.empty?
+        flydata.data_entry.update_table_validity(de['id'], updated_tables: table_validity_hash) unless table_validity_hash.empty?
 
         sync_fm = create_sync_file_manager(de)
         sync_fm.save_generated_ddl(tables_without_error, TableDdl::VERSION)

data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb

@@ -3,6 +3,7 @@ require 'flydata/source'
 require 'flydata-core/fluent/config_helper'
 require 'flydata/fluent-plugins/flydata_plugin_ext/flush_support'
 require 'flydata/fluent-plugins/flydata_plugin_ext/transaction_support'
+require 'flydata/table_attribute'
 
 module Fluent
 
@@ -17,6 +18,7 @@ module Fluent
         config_param :tables, :string
         config_param :tables_append_only, :string
         config_param :pk_override, :hash, default: {}
+        config_param :table_attributes, :array, default: []
         config_param :tag, :string
         config_param :position_file, :string, default: 'position.binlog.pos'
       end
@@ -61,6 +63,26 @@ module Fluent
         h
       end
 
+      # Re-construct pk_override & table_attributes
+      #   @pk_override should include both USER_PK_OVERRIDE and UK_AS_PK_OVERRIDE.
+      #   If mismatch happens here, @pk_override value should come from a local conf file.
+      #TODO support UK_AS_PK_OVERRIDE for tables existing only in local conf, not registered on the web.
+
+      pk_override_from_table_attrs = Flydata::TableAttribute::generate_pk_override(@table_attributes)
+
+      if @pk_override != pk_override_from_table_attrs
+        # @pk_override in a local conf file only includes USER_PK_OVERRIDE. Update table_attributes.
+        Flydata::TableAttribute::delete_attribute(@table_attributes, Flydata::TableAttribute::USER_PK_OVERRIDE)
+        Flydata::TableAttribute::save_attribute(@table_attributes, @pk_override, Flydata::TableAttribute::USER_PK_OVERRIDE)
+
+        @pk_override = Flydata::TableAttribute::generate_pk_override(@table_attributes)
+        $log.info "Update primary key override. Using #{@pk_override}"
+
+        prefs = @data_entry['mysql_data_entry_preference'] || @data_entry['postgresql_data_entry_preference']
+        prefs['table_attributes'] = @table_attributes
+        prefs['pk_override'] = @pk_override
+      end
+
       $log.info("Source position - resume_pos:'#{@source_position_file.read rescue IOError}' " +
                 "sent_pos:'#{@sent_position_file.read rescue nil}'")
     end

data/lib/flydata/source/sync.rb

@@ -1,6 +1,7 @@
 require 'flydata/source'
 require 'flydata/source/component'
 require 'flydata/source/errors'
+require 'flydata/table_attribute'
 
 module Flydata
 module Source
@@ -68,6 +69,35 @@ class Sync < Component
     prefs['new_tables'] =
       prefs['new_tables'].kind_of?(String) ?
         prefs['new_tables'].split(/(?:\s*,\s*|\s+)/).uniq : []
+
+    setup_pk_override_prefs(prefs)
+  end
+
+  def setup_pk_override_prefs(prefs)
+    prefs['pk_override'] = {} if prefs['pk_override'].nil?
+    prefs['user_pk_override'] = TableAttribute::extract_attribute(prefs['table_attributes'],
+                                    TableAttribute::USER_PK_OVERRIDE)
+    prefs['uk_as_pk_override'] = TableAttribute::extract_attribute(prefs['table_attributes'],
+                                     TableAttribute::UK_AS_PK_OVERRIDE)
+
+    # Re-construct pk_override & table_attributes
+    #   prefs['pk_override'] should include both USER_PK_OVERRIDE and UK_AS_PK_OVERRIDE.
+    #   If mismatch happens here, prefs['pk_override'] value should come from a local conf file.
+    #TODO support UK_AS_PK_OVERRIDE for tables existing only in local conf, not registered on the web.
+
+    pk_override_from_table_attrs = TableAttribute::generate_pk_override(prefs['table_attributes'])
+
+    if prefs['pk_override'] != pk_override_from_table_attrs
+      # prefs['pk_override'] in a local conf file only includes USER_PK_OVERRIDE.
+      prefs['user_pk_override'] = prefs['pk_override']
+
+      # Update table_attributes.
+      TableAttribute::delete_attribute(prefs['table_attributes'], TableAttribute::USER_PK_OVERRIDE)
+      TableAttribute::save_attribute(prefs['table_attributes'], prefs['user_pk_override'], TableAttribute::USER_PK_OVERRIDE)
+
+      prefs['pk_override'] = TableAttribute::generate_pk_override(prefs['table_attributes'])
+      $log.info "Primary key override is updated. Using pk_override: #{prefs['pk_override']}"
+    end
   end
 end
 

data/lib/flydata/source/sync_generate_table_ddl.rb

@@ -28,21 +28,30 @@ class SyncGenerateTableDdl < Component
 
   def generate_flydata_tabledef(tables, options)
     prefs = data_entry_prefs
+    reset_uk_as_pk_override(tables, prefs)
     options = options.merge(prefs)
     flydata_tabledefs = []
     error_list = []
+    uk_as_pk_override = {}
+
     missing_tables = each_source_tabledef(tables, options) do |source_tabledef, error|
       if error
         error_list << error.err_hash
         next
       end
+
+      if source_tabledef.converting_uk_to_pk?
+        new_pk_override = source_tabledef.get_pk_override_as_hash
+        uk_as_pk_override.merge!(new_pk_override)
+      end
       flydata_tabledefs << source_tabledef.to_flydata_tabledef
     end
+
     if missing_tables
       missing_tables.each {|missing_table| error_list << { error: "table does not exist in the #{data_source_type_display_name}", table: missing_table } }
     end
 
-    [flydata_tabledefs, error_list]
+    [flydata_tabledefs, error_list, uk_as_pk_override]
   end
 
   private
@@ -77,6 +86,21 @@ class SyncGenerateTableDdl < Component
   def each_source_tabledef(tables, options, &block)
     raise UnsupportedSourceError,  "subclass must implement"
   end
+
+  def reset_uk_as_pk_override(target_tables, prefs)
+    # Reset uk_as_pk_override for target tables.
+    prefs['uk_as_pk_override'].delete_if do |table_name, val|
+      target_tables.include?(table_name)
+    end
+    update_pk_override_pref(prefs)
+  end
+
+  def update_pk_override_pref(prefs)
+    # pk_override = user_pk_override + uk_as_pk_override
+    # (`user_pk_override` takes precedence)
+    prefs['pk_override'] = prefs['uk_as_pk_override'].merge( prefs['user_pk_override'] )
+    $log.info "primary key override: #{prefs['pk_override']}"
+  end
 end
 
 end

data/lib/flydata/source_mysql/command/mysqldump.rb

@@ -9,10 +9,33 @@ module Flydata
 
       def generate_command(dbconf, args)
         dbconf.delete('tables')
-        dbconf[:custom_option_end] = args.join(' ')
+        dbconf[:custom_option_end] = build_custom_option_end(args)
         dbconf[:command] = 'mysqldump'
         FlydataCore::Mysql::CommandGenerator::generate_mysql_cmd(dbconf)
       end
+
+      # Workaround to build --where options correctly
+      #
+      # [Background]
+      # Arguments including single/double quotations don't work because those quotations
+      # are removed from the system.
+      #
+      # User's command:
+      #   flydata mysqldump table_a --where "id = 1001"
+      # ARGV:
+      #   ["table_a", "--where", "id", "=", "1001"]
+      #
+      def build_custom_option_end(args)
+        ret = []
+        while v = args.shift
+          ret << v
+          if v == '--where'
+            ret << %Q|"#{args.join(' ')}"|
+            break
+          end
+        end
+        ret.join(' ')
+      end
     end
   end
 end

data/lib/flydata/source_mysql/data_entry.rb

@@ -10,6 +10,7 @@ class DataEntry < Source::DataEntry
       tables: {},
       tables_append_only: {},
       pk_override: {},
+      table_attributes: {},
       host: {},
       port: {},
       username: {},

data/lib/flydata/source_mysql/parser/dump_parser.rb

@@ -475,11 +475,16 @@ EOS
           def parse(line)
             start_ruby_prof
             bench_start_time = Time.now
-            _parse(line)
-          ensure
-            stop_ruby_prof
-            if ENV['FLYDATA_BENCHMARK']
-              puts " -> time:#{Time.now.to_f - bench_start_time.to_f} size:#{target_line.size}"
+            begin
+              _parse(line)
+            rescue DumpParseError => e
+              $log.error "Encountered an error while parsing this chunk:\n  #{e.message}. backtrace: #{e.backtrace}"
+              raise
+            ensure
+              stop_ruby_prof
+              if ENV['FLYDATA_BENCHMARK']
+                puts " -> time:#{Time.now.to_f - bench_start_time.to_f} size:#{target_line.size}"
+              end
             end
           end
 

data/lib/flydata/source_postgresql/data_entry.rb

@@ -10,6 +10,7 @@ class DataEntry < Source::DataEntry
       tables: {},
       tables_append_only: {},
       pk_override: {},
+      table_attributes: {},
       host: {},
       port: {},
       username: {},

data/lib/flydata/source_postgresql/table_meta.rb

@@ -50,7 +50,11 @@ SELECT
       FROM (SELECT length(substring('999999'::money::char varying, '\.([0-9]+)$')) AS scale) s)
    ELSE c.numeric_scale END AS numeric_scale,
   c.datetime_precision,
-  i.indisprimary AS is_primary
+  i.indexrelid::regclass as index_name,
+  i.indisprimary as is_primary,
+  i.indisunique as is_unique,
+  i.indisvalid as is_valid,
+  CASE WHEN i.indpred!='NULL' THEN true ELSE false END AS is_partial
 FROM
   pg_index i
   JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
@@ -66,7 +70,7 @@ WHERE
   AND c.table_schema IN (%{schema})
   AND c.table_name IN (%{tables})
 ORDER BY
-  c.table_name, c.ordinal_position, i.indisprimary asc;
+  c.table_name, c.ordinal_position, i.indisprimary desc, i.indisunique desc, i.indisvalid asc;
 EOT
 
     GET_CURRENT_SNAPSHOT_SQL = "SELECT txid_current_snapshot();"
@@ -119,8 +123,8 @@ EOT
         column_name = col['column_name'].to_sym
         table_name = col['table_name'].to_sym
         t_meta = ret[table_name]
-        t_meta[:raw_columns] ||= {}
-        t_meta[:raw_columns][column_name] = col
+        t_meta[:raw_columns] = Hash.new {|h,k| h[k] = []} unless t_meta[:raw_columns]
+        t_meta[:raw_columns][column_name] << col
       end
 
       ret.each do |table_name, t_meta|

data/lib/flydata/sync_file_manager.rb

@@ -514,7 +514,7 @@ module Flydata
     OLD_DUMP_DIR_PROPERTY = 'mysqldump_dir'
     DUMP_DIR_PROPERTY = 'dump_dir'
     def dump_dir
-      pref = @data_entry['mysql_data_entry_preference']
+      pref = @data_entry['mysql_data_entry_preference'] || @data_entry['postgresql_data_entry_preference']
       dump_dir = nil
       dump_dir_property = DUMP_DIR_PROPERTY
 

data/lib/flydata/table_attribute.rb

@@ -0,0 +1,58 @@
+module Flydata
+module TableAttribute
+  USER_PK_OVERRIDE = 'pk_override'
+  UK_AS_PK_OVERRIDE = 'uk_as_pk'
+  INVALID_TABLE_REASON = 'invalid_table_reason'
+
+  def self.generate_pk_override(table_attributes)
+      user_pk_override = extract_attribute(table_attributes, USER_PK_OVERRIDE)
+      uk_as_pk_override = extract_attribute(table_attributes, UK_AS_PK_OVERRIDE)
+
+      uk_as_pk_override.merge(user_pk_override)
+  end
+
+  def self.delete_attribute(table_attributes, key)
+    table_attributes.each {|tbl_attr| tbl_attr.delete(key.to_s)}
+    table_attributes
+  end
+
+  # Extract {<table_name> => <value_for_a_given_key>} pairs from table attributes
+  # arguments:
+  # - table_attributes (hash):
+  #     { "<table-name1>" => {
+  #         "invalid_table_reason" => "<error reason>",
+  #         "pk_override" => ['id1', ...],
+  #         "uk_as_pk" => ['uid', ...],...
+  #       }, 
+  #       "<table-name2> => {...},
+  #     }
+  #
+  # - key (symbol or String): hash key in table_attributes
+  #
+  # return value (hash):
+  #   { "table_1" => <value_1>, "table_2" => <value_2>,...}
+  def self.extract_attribute(table_attributes, key)
+    tbl_value_hash = {}
+    table_attributes.each do |tbl_attr|
+
+      table_name = tbl_attr['table_name']
+      next unless tbl_attr.has_key?(key.to_s)
+      tbl_value_hash[table_name] = tbl_attr[key.to_s]
+    end
+    tbl_value_hash
+  end
+
+  def self.save_attribute(table_attributes, values_hash, key)
+    return unless values_hash
+    values_hash.each_pair do |tbl_name, val|
+      tbl_attr = table_attributes.find { |ta| ta['table_name'] == tbl_name }
+      unless tbl_attr
+        # Add a new table
+        tbl_attr = { 'table_name' => tbl_name }
+        table_attributes << tbl_attr
+      end
+      tbl_attr[key.to_s] = val
+    end
+  end
+end
+end

data/spec/flydata/command/sync_spec.rb

@@ -175,7 +175,13 @@ module Flydata
         subject { subject_object.send(:data_entry) }
 
         let(:de) { { 'mysql_data_entry_preference' => mp } }
-        let(:mp) { { 'tables' => 'Users,Addresses' } }
+        let(:mp) { { 'tables' => 'Users,Addresses',
+                     'table_attributes' => [
+                       {"table_name"=>"Users", "status"=>"init_sync_pending"},
+                       {"table_name"=>"Addresses", "status"=>"init_sync_pending"}
+                     ],
+                     'pk_override' => {"Users"=>["id"]}
+                  } }
 
         let(:sfm) { double('sfm') }
         let(:ssl_ca_content) { double('ssl_ca_content') }
@@ -200,7 +206,14 @@ module Flydata
               end
             end
             context 'with tables_append_only' do
-              before { mp['tables_append_only'] = 'Invoices,Sessions,Addresses' }
+              before do
+                mp['tables_append_only'] = 'Invoices,Sessions,Addresses'
+                mp['table_attributes'] += [
+                    {"table_name"=>"Invoices", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
+                    {"table_name"=>"Sessions", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
+                    {"table_name"=>"Addresses", "omit_events"=>["delete"], "status"=>"init_sync_pending"}
+                ]
+              end
               it "creates an array of tables from 'tables' and 'tables_append_only' combined" do
                 subject
                 expect(mp['tables']).to eq %w(Users Addresses Invoices Sessions)

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -15,7 +15,9 @@ module Fluent
   tag #{TEST_TAG}
   database #{TEST_DB}
   tables test_table,test_table_1,test_table_2
-  tables_append_only test_table_3
+  tables_append_only test_table_3,error_table_4
+  pk_override {"test_table_1":["id1","id2"],"test_table_3":["uid"]}
+  table_attributes [{"table_name":"test_table","status":"init_sync_completed"},{"table_name":"test_table_1","status":"init_sync_completed","pk_override":["id1","id2"]},{"table_name":"test_table_2","status":"init_sync_pending"},{"table_name":"test_table_3","omit_events":["delete"],"status":"init_sync_pending","uk_as_pk":["uid"]},{"table_name":"error_table_4","omit_events":["delete"],"status":"init_sync_pending","invalid_table_reason":"table does not exist in the MySQL database"}]
   position_file #{TEST_POSITION_FILE}
   host localhost
   port 3306
@@ -237,6 +239,13 @@ EOT
 
     describe '#configure' do
       let(:config) { default_config }
+      let(:table_attributes_array) {
+        [{"table_name"=>"test_table", "status"=>"init_sync_completed"},
+        {"table_name"=>"test_table_1", "status"=>"init_sync_completed", "pk_override"=>["id1", "id2"]},
+        {"table_name"=>"test_table_2", "status"=>"init_sync_pending"},
+        {"table_name"=>"test_table_3", "omit_events"=>["delete"], "status"=>"init_sync_pending", "uk_as_pk"=>["uid"]},
+        {"table_name"=>"error_table_4", "omit_events"=>["delete"], "status"=>"init_sync_pending", "invalid_table_reason"=>"table does not exist in the MySQL database"}]
+      }
       before do
         Test.configure_plugin(plugin, config)
       end
@@ -248,8 +257,9 @@ EOT
           'mysql_data_entry_preference' => {
             'database' => 'test_db',
             'tables' => 'test_table,test_table_1,test_table_2',
-            'tables_append_only' => 'test_table_3',
-            'pk_override' => {},
+            'tables_append_only' => 'test_table_3,error_table_4',
+            'pk_override' => {"test_table_1" => ["id1", "id2"], "test_table_3" => ["uid"]},
+            'table_attributes' => table_attributes_array,
             'host' => 'localhost',
             'port' => 3306,
             'username' => 'test_admin',
@@ -265,7 +275,7 @@ EOT
       end
 
       it 'sets data tables_append_only as array' do
-        expect(plugin.instance_variable_get(:@tables_append_only)).to eq(%w(test_table_3))
+        expect(plugin.instance_variable_get(:@tables_append_only)).to eq(%w(test_table_3 error_table_4))
       end
 
       it 'sets sync_fm' do
@@ -273,7 +283,15 @@ EOT
       end
 
       it 'sets omit_events' do
-        expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table]})
+        expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table],'error_table_4' => [:delete, :truncate_table]})
+      end
+
+      it 'sets pk_override' do
+        expect(plugin.instance_variable_get(:@pk_override)).to eq({"test_table_1" => ["id1", "id2"], "test_table_3" => ["uid"]})
+      end
+
+      it 'set table_attributes' do
+        expect(plugin.instance_variable_get(:@table_attributes)).to eq(table_attributes_array)
       end
     end
 

data/spec/flydata/fluent-plugins/in_postgresql_query_based_flydata_spec.rb

@@ -14,7 +14,9 @@ module Fluent
   tag #{TEST_TAG}
   database #{TEST_DB}
   tables test_table,test_table_1,test_table_2
-  tables_append_only test_table_3
+  tables_append_only test_table_3,error_table_4
+  pk_override {"test_table_1":["id1","id2"],"test_table_3":["uid"]}
+  table_attributes [{"table_name":"test_table","status":"init_sync_completed"},{"table_name":"test_table_1","status":"init_sync_completed","pk_override":["id1","id2"]},{"table_name":"test_table_2","status":"init_sync_pending"},{"table_name":"test_table_3","omit_events":["delete"],"status":"init_sync_pending","uk_as_pk":["uid"]},{"table_name":"error_table_4","omit_events":["delete"],"status":"init_sync_pending","invalid_table_reason":"table does not exist in the MySQL database"}]
   position_file #{TEST_POSITION_FILE}
   host localhost
   port 5433
@@ -32,6 +34,13 @@ EOT
       let(:config) { default_config }
       subject { Test.configure_plugin(plugin, config) }
 
+      let(:table_attributes_array) {
+        [{"table_name"=>"test_table", "status"=>"init_sync_completed"},
+        {"table_name"=>"test_table_1", "status"=>"init_sync_completed", "pk_override"=>["id1", "id2"]},
+        {"table_name"=>"test_table_2", "status"=>"init_sync_pending"},
+        {"table_name"=>"test_table_3", "omit_events"=>["delete"], "status"=>"init_sync_pending", "uk_as_pk"=>["uid"]},
+        {"table_name"=>"error_table_4", "omit_events"=>["delete"], "status"=>"init_sync_pending", "invalid_table_reason"=>"table does not exist in the MySQL database"}]
+      }
       before do
         create_file(TEST_POSITION_FILE, "1001:1001:\t\t")
         create_table_pos_file('test_table', '0')
@@ -48,8 +57,9 @@ EOT
           'postgresql_data_entry_preference' => {
             'database' => 'test_db',
             'tables' => 'test_table,test_table_1,test_table_2',
-            'tables_append_only' => 'test_table_3',
-            'pk_override' => {},
+            'tables_append_only' => 'test_table_3,error_table_4',
+            'pk_override' => {"test_table_1" => ["id1", "id2"], "test_table_3" => ["uid"]},
+            'table_attributes' => table_attributes_array,
             'host' => 'localhost',
             'port' => 5433,
             'username' => 'test_admin',
@@ -68,9 +78,17 @@ EOT
       end
 
       it 'sets omit_events' do
-        expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table]})
+        expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table],'error_table_4' => [:delete, :truncate_table]})
       end
 
+      it 'sets pk_override' do
+        expect(plugin.instance_variable_get(:@pk_override)).to eq({"test_table_1" => ["id1", "id2"], "test_table_3" => ["uid"]})
+      end
+
+      it 'set table_attributes' do
+        expect(plugin.instance_variable_get(:@table_attributes)).to eq(table_attributes_array)
+       end
+
       it 'sets context' do
         expect(plugin.instance_variable_get(:@context)).to be_kind_of(Flydata::SourcePostgresql::PluginSupport::Context)
       end

data/spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb

@@ -36,6 +36,11 @@ describe SyncGenerateTableDdl do
   end
   describe '#generate_flydata_tabledef' do
     subject { subject_object.generate_flydata_tabledef(tables, options) }
+    before do
+      de['mysql_data_entry_preference']['user_pk_override'] = {}
+      de['mysql_data_entry_preference']['uk_as_pk_override'] = {}
+      de['mysql_data_entry_preference']['pk_override'] = {}
+    end
 
     let(:tables) { %w| table1 table2 table4 table3 | }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.6
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-02 00:00:00.000000000 Z
+date: 2016-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -758,6 +758,7 @@ files:
 - lib/flydata/source_zendesk/sync_generate_table_ddl.rb
 - lib/flydata/source_zendesk/zendesk_flydata_tabledefs.rb
 - lib/flydata/sync_file_manager.rb
+- lib/flydata/table_attribute.rb
 - lib/flydata/table_ddl.rb
 - lib/flydata/table_meta.rb
 - lib/flydata/util/encryptor.rb
@@ -869,7 +870,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: FlyData Agent