flydata 0.7.2.1 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 041b42f8c7a79544e1075015c40b5f4d617eb983
-  data.tar.gz: 18a938ab66c2754e67821cdb93178f0450991d8d
+  metadata.gz: 35042672a68ea10f4ec090aa77f218c885aed555
+  data.tar.gz: 4223e306573350bb767eda1259e71ccc9d7d182a
 SHA512:
-  metadata.gz: 0afce08535788fde831998db168317d125ab4e4249f36dd9d10fee443ca3e62bb3dfce478bc3953e04d5d57dd7fa560e4ba2a1948c517021378f52a0686abc58
-  data.tar.gz: e7af6dd1ec9470fd25a5c853858123112deb8f90c1a7543d09dc99c5564d75398c63f934125906f29bbf91b53d8cc8fa923f014b32c7b164f7554bf0058b44bd
+  metadata.gz: 143b46f4b1fb7c7e7472932b668caeb66c2201f91bf2bc334553583cf03a6f7800b4b1d895e50174e6043c2328abcf8836b2dc7c2e1848457bf1e0993d3149d9
+  data.tar.gz: 22ec19871da8e8d1545cf019ec09f4113fddf5ed59da2fe5f285cbce2e616db8f5ecb9320169efefd4444d79594b1907b55a507d8674a9999e2255f7c2ee69ca

data/VERSION

@@ -1 +1 @@
-0.7.2.1
+0.7.4

data/flydata-core/lib/flydata-core/mysql/command_generator.rb

@@ -120,18 +120,13 @@ module FlydataCore
       def self._each_mysql_tabledef(tables, option)
         command = generate_mysql_ddl_dump_cmd(option.merge(tables: tables))
 
-        create_opt = {}
-        if option.has_key?(:skip_primary_key_check)
-          create_opt[:skip_primary_key_check] = option[:skip_primary_key_check]
-        end
-
         Open3.popen3(command) do |stdin, stdout, stderr|
           stdin.close
           stdout.set_encoding("utf-8", "utf-8") # mysqldump output must be in UTF-8
           create_flydata_ctl_table = true
           while !stdout.eof?
             begin
-              mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, create_opt)
+              mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, option)
               break if mysql_tabledef.nil?
               yield(mysql_tabledef, nil)
             rescue FlydataCore::TableDefError=> e

data/flydata-core/lib/flydata-core/table_def/base.rb

@@ -1,9 +1,16 @@
+require 'flydata-core/errors'
+
 module FlydataCore
 module TableDef
 
 class Base
+  def self.create(source_data, options = {})
+    params = _create(source_data, options)
+    params ? self.new(*params, options) : nil
+  end
+
   def initialize(table_def, table_name, columns, column_def, default_charset,
-                 default_source_charset, comment)
+                 default_source_charset, comment, options = {})
     @table_def = table_def
     @table_name = table_name
     @columns = columns
@@ -11,6 +18,16 @@ class Base
     @default_charset = default_charset
     @default_source_charset = default_source_charset
     @comment = comment
+
+    opts = [:skip_primary_key_check, :pk_override].inject({}) do |h, key|
+      v = options[key]
+      v ||= options[key.to_s]
+      h[key] = v if v
+      h
+    end
+
+    @pk_override = self.class.override_pk(@table_name, @columns, opts)
+    self.class.check_pk(@table_name, @columns, opts)
   end
 
   attr_reader :columns, :column_def, :table_name, :default_source_charset
@@ -22,6 +39,7 @@ class Base
     tabledef[:default_charset] = @default_charset if @default_charset
     tabledef[:comment] = @comment if @comment
     tabledef[:src_ddl] = @table_def
+    tabledef[:pk_override] = @pk_override
 
     tabledef
   end
@@ -35,6 +53,41 @@ class Base
              sort{|a,b| a[:primary_key] <=> b[:primary_key]}.
              collect{|col| col[:column]}
   end
+
+  private
+
+  def self._create(source_data, options)
+    raise "subclass must implement"
+  end
+
+  def self.override_pk(table_name, columns, options)
+    pk_override = options[:pk_override]
+    return nil unless pk_override.kind_of?(Hash) && pk_override[table_name]
+
+    pkcols = pk_override[table_name]
+    pkcols_found = []
+    columns.each do |column|
+      if pkcols.include?(column[:column])
+        column[:primary_key] = true
+        pkcols_found << column[:column]
+      else
+        column.delete(:primary_key) if column.has_key?(:primary_key)
+      end
+    end
+    pkcols_missing = pkcols - pkcols_found
+    unless pkcols_missing.empty?
+      raise "Column(s) is specified as a primary key but the column does not exist in the table '#{table_name}': #{pkcols_missing.collect{|pc| "'#{pc}'"}.join(",")}"
+    end
+
+    pkcols
+  end
+
+  def self.check_pk(table_name, columns, options)
+    options
+    unless columns.any? {|column| column[:primary_key]} or options[:skip_primary_key_check]
+      raise TableDefError, {error: "no primary key defined", table: table_name}
+    end
+  end
 end
 
 end

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -85,11 +85,6 @@ class MysqlTableDef < Base
     nil
   end
 
-  def self.create(io, options = {})
-    params = _create(io, options)
-    params ? self.new(*params) : nil
-  end
-
   def self._create(io, options)
     table_def = ''
     table_name = nil
@@ -144,9 +139,6 @@ class MysqlTableDef < Base
         end
 
       when :after_create_table
-        unless columns.any? {|column| column[:primary_key]} or options[:skip_primary_key_check]
-          raise TableDefError, {error: "no primary key defined", table: table_name}
-        end
         break
       end
     end

data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb

@@ -101,11 +101,6 @@ class PostgresqlTableDef < Base
     ret_type
   end
 
-  def self.create(information_schema_columns, options)
-    params = _create(information_schema_columns, options)
-    params ? self.new(*params) : nil
-  end
-
   private
 
   def self.get_width_values(information_schema_columns, type_hash)

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -380,6 +380,9 @@ EOS
     values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
     values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
     values << "('#{flydata_tabledef[:table_name]}', 'src_ddl', '#{escape(flydata_tabledef[:src_ddl])}')"
+    if flydata_tabledef[:pk_override]
+      values << "('#{flydata_tabledef[:table_name]}', 'pk_override', '#{escape(flydata_tabledef[:pk_override].join(","))}')"
+    end
     sql += values.join(",\n") + ";\n"
     sql
   end

data/flydata-core/spec/table_def/base_spec.rb

@@ -15,6 +15,7 @@ describe Base do
   let(:default_charset) { double('default_charset') }
   let(:default_source_charset) { double('default_source_charset') }
   let(:comment) { double('comment') }
+  let(:options) { double('options') }
 
   describe '#pk_columns' do
     subject { subject_object.pk_columns }
@@ -45,6 +46,89 @@ describe Base do
       end
     end
   end
+
+  describe '.override_pk' do
+    subject { described_class.send(:override_pk, table_name, columns, options) }
+
+    let(:columns) { [
+      col1def,
+      col2def,
+    ] }
+
+    let(:col1def) { {column: col1name, primary_key: true} }
+    let(:col2def) { {column: col2name} }
+
+    let(:col1name) { double("col1name") }
+    let(:col2name) { double("col2name") }
+
+    shared_examples "nothings changed" do
+      it {
+        subject
+        expect(col1def[:primary_key]).to be_truthy
+        expect(col2def.has_key?(:primary_key)).to be_falsey
+      }
+    end
+
+    context 'when no :pk_override attribute exists' do
+      let(:options) { {} }
+
+      it_behaves_like "nothings changed"
+    end
+
+    context 'when the table has no pk_override' do
+      let(:options) { {pk_override:{}} }
+
+      it_behaves_like "nothings changed"
+    end
+
+    context 'when the table has pk_override' do
+      let(:options) { {pk_override:{table_name => [col2name]}} }
+
+      it 'overrides primary key settings' do
+        subject
+        expect(col1def.has_key?(:primary_key)).to be_falsey
+        expect(col2def[:primary_key]).to be_truthy
+      end
+    end
+
+    context 'when the column specified in pk_override does not exist' do
+      let(:options) { {pk_override:{table_name => ["asdf"]}} }
+
+      it do
+        expect{subject}.to raise_error /the column does not exist in the table/
+      end
+    end
+  end
+
+  describe '.check_pk' do
+    subject { described_class.send(:check_pk, table_name, columns, options) }
+
+    let(:columns) { [
+      col1def,
+      col2def,
+    ] }
+
+    let(:col1def) { {} }
+    let(:col2def) { {} }
+    let(:options) { {} }
+
+    context 'when a pk column exists' do
+      let(:col1def) {  {primary_key: true} }
+      it { expect{subject}.to_not raise_error }
+
+    end
+    context 'when no pk column exists' do
+      let(:col1def) { {} }
+      let(:col2def) { {} }
+
+      it { expect{subject}.to raise_error(TableDefError) }
+
+      context 'if skip_primary_key_check option is set' do
+        let(:options) { {skip_primary_key_check: true} }
+        it { expect{subject}.to_not raise_error }
+      end
+    end
+  end
 end
 
 end

data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb

@@ -16,6 +16,7 @@ module Fluent
         config_param :data_entry_type, :string, default: nil  # data entry type
         config_param :tables, :string
         config_param :tables_append_only, :string
+        config_param :pk_override, :hash, default: {}
         config_param :tag, :string
         config_param :position_file, :string, default: 'position.binlog.pos'
       end

data/lib/flydata/fluent-plugins/in_postgresql_query_based_flydata.rb

@@ -23,8 +23,9 @@ class PostgresqlQueryBasedFlydataInput < Input
     @dbconf = FlydataCore::Postgresql::Config.opts_for_pg(@data_entry['postgresql_data_entry_preference'])
     $log.info "postgresql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{@tables_append_only}\""
 
+    opts = @dbconf.merge(pk_override:@data_entry['postgresql_data_entry_preference']['pk_override'])
     @table_meta = Flydata::SourcePostgresql::TableMeta.new(
-      @dbconf, @tables, @schema)
+      opts, @tables, @schema)
 
     @context = Flydata::SourcePostgresql::PluginSupport::Context.new(
       database: @database, tables: @tables,

data/lib/flydata/source_mysql/data_entry.rb

@@ -9,6 +9,7 @@ class DataEntry < Source::DataEntry
       database: {},
       tables: {},
       tables_append_only: {},
+      pk_override: {},
       host: {},
       port: {},
       username: {},

data/lib/flydata/source_postgresql/data_entry.rb

@@ -9,6 +9,7 @@ class DataEntry < Source::DataEntry
       database: {},
       tables: {},
       tables_append_only: {},
+      pk_override: {},
       host: {},
       port: {},
       username: {},

data/lib/flydata/source_postgresql/generate_source_dump.rb

@@ -71,6 +71,11 @@ EOS
     cli = FlydataCore::Postgresql::PGClient.new(de_prefs)
     table_meta.reload(cli)
 
+    tables_missing_meta = tables.select{|t| tm = table_meta[t]; tm.nil? || tm.empty?}
+    unless tables_missing_meta.empty?
+      raise "Tables are not available.  Check if the following table(s) exist and are visible: #{tables_missing_meta.join(",")}"
+    end
+
     context = source.sync_generate_table_ddl(dp, nil)
     source_pos = get_source_pos(table_meta.current_snapshot, &src_pos_callback)
 

data/lib/flydata/source_postgresql/query_based_sync/resource_requester.rb

@@ -10,6 +10,11 @@ module QueryBasedSync
 class ResourceRequester < Flydata::QueryBasedSync::ResourceRequester
   RESPONSE_CLASS = Flydata::SourcePostgresql::QueryBasedSync::Response
 
+  def initialize(*args)
+    super
+    @tables_missing_meta = []
+  end
+
   # Override
   def create_resource_client(context)
     FlydataCore::Postgresql::PGClient.new(context.dbconf)
@@ -20,6 +25,14 @@ class ResourceRequester < Flydata::QueryBasedSync::ResourceRequester
     table_sym = table_name.to_sym
     t_meta = context.table_meta[table_sym]
 
+    if t_meta.nil? || t_meta.empty?
+      unless @tables_missing_meta.include?(table_name)
+        @tables_missing_meta << table_name
+        log_error "No metadata is available for table '#{table_name}'.  Make sure the table exists and is visible to the PostgreSQL user."
+      end
+      return
+    end
+
     table_src_pos = context.table_src_pos_files[table_sym].pos
     master_src_pos = context.cur_src_pos_file.pos
 

data/lib/flydata/source_postgresql/table_meta.rb

@@ -66,7 +66,7 @@ WHERE
   AND c.table_schema IN (%{schema})
   AND c.table_name IN (%{tables})
 ORDER BY
-  c.table_name, c.ordinal_position;
+  c.table_name, c.ordinal_position, i.indisprimary asc;
 EOT
 
     GET_CURRENT_SNAPSHOT_SQL = "SELECT txid_current_snapshot();"
@@ -74,11 +74,11 @@ EOT
     DEFAULT_MAX_FETCH_RECORD_SIZE = 50000
     #DEFAULT_MAX_FETCH_RECORD_SIZE = 8
 
-    def initialize(dbconf, tables, schema = nil)
-      @dbconf = dbconf
-      @database = dbconf[:dbname] || dbconf[:database] || dbconf['database']
+    def initialize(options, tables, schema = nil)
+      @options = options
+      @database = options[:dbname] || options[:database] || options['database']
       @tables = tables
-      @schema = (schema || dbconf[:schema] || dbconf['schema']).to_s.strip
+      @schema = (schema || options[:schema] || options['schema']).to_s.strip
       @schema = @schema.empty? ? nil : @schema
     end
 
@@ -94,7 +94,7 @@ EOT
 
       conn = pg_client
       if conn.nil?
-        local_conn = conn = FlydataCore::Postgresql::PGClient.new(@dbconf)
+        local_conn = conn = FlydataCore::Postgresql::PGClient.new(@options)
       end
 
       # Set table_meta
@@ -126,7 +126,7 @@ EOT
       ret.each do |table_name, t_meta|
         begin
           table_def = FlydataCore::TableDef::PostgresqlTableDef.create(
-                      t_meta[:raw_columns].values, :skip_primary_key_check)
+                      t_meta[:raw_columns].values, @options)
         rescue FlydataCore::TableDefError => e
           t_meta.merge!(
             table_name: table_name,

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -249,6 +249,7 @@ EOT
             'database' => 'test_db',
             'tables' => 'test_table,test_table_1,test_table_2',
             'tables_append_only' => 'test_table_3',
+            'pk_override' => {},
             'host' => 'localhost',
             'port' => 3306,
             'username' => 'test_admin',

data/spec/flydata/fluent-plugins/in_postgresql_query_based_flydata_spec.rb

@@ -49,6 +49,7 @@ EOT
             'database' => 'test_db',
             'tables' => 'test_table,test_table_1,test_table_2',
             'tables_append_only' => 'test_table_3',
+            'pk_override' => {},
             'host' => 'localhost',
             'port' => 5433,
             'username' => 'test_admin',

data/spec/flydata/source_postgresql/generate_source_dump_spec.rb

@@ -56,6 +56,7 @@ describe GenerateSourceDump do
           expect(Flydata::SourcePostgresql::TableMeta).to receive(:new).with(de_prefs, tables).and_return(table_meta)
           expect(table_meta).to receive(:reload).with(cli).once
           expect(table_meta).to receive(:current_snapshot).and_return(snapshot)
+          expect(table_meta).to receive(:[]).and_return({something_in_it:true}).exactly(tables.count).times
           expect(subject_object).to receive(:get_source_pos).
             with(snapshot, &src_pos_callback).and_return(source_pos)
           expect(source).to receive(:sync_generate_table_ddl).with(dp, nil).
@@ -83,6 +84,7 @@ describe GenerateSourceDump do
           expect(Flydata::SourcePostgresql::TableMeta).to receive(:new).with(de_prefs, tables).and_return(table_meta)
           expect(table_meta).to receive(:reload).with(cli).once
           expect(table_meta).to receive(:current_snapshot).and_return(snapshot)
+          allow(table_meta).to receive(:[]).and_return({something_in_it:true})
           expect(subject_object).to receive(:get_source_pos).
             with(snapshot, &src_pos_callback).and_return(source_pos)
           allow(source).to receive(:sync_generate_table_ddl).with(dp, nil).

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.2.1
+  version: 0.7.4
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-21 00:00:00.000000000 Z
+date: 2016-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -481,7 +481,6 @@ dependencies:
 description: FlyData Agent
 email: sysadmin@flydata.com
 executables:
-- fdmysqldump
 - fdredshift
 - flydata
 - serverinfo
@@ -500,7 +499,6 @@ files:
 - benchmark/data/insert_parser_test_data_num_only.sql.gz
 - benchmark/insert_parser_bench.rb
 - benchmark/insert_parser_prof.rb
-- bin/fdmysqldump
 - bin/fdredshift
 - bin/flydata
 - bin/serverinfo
@@ -869,7 +867,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent

data/bin/fdmysqldump

@@ -1,62 +0,0 @@
-#!/usr/bin/env ruby
-
-#require 'optparse'
-
-lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-require 'flydata/table_def'
-
-# Deprecation warning
-$stderr.puts "WARNING: Use 'flydata sync:generate_table_ddl' command instead"
-
-if `which mysqldump`.empty?
-  abort "mysqldump is not installed."
-end
-
-# TODO Implement options instead of passing all to mysqldump.
-=begin
-opts = {}
-
-op = OptionParser.new do |o|
-  o.on('-h', '--host NAME', 'Connect to host.') do |v|
-    opts[:host] = v
-  end
-  o.on('-P', '--port NUMBER', 'Port number to use for connection.') do |v|
-    opts[:port] = v
-  end
-  o.on('-u', '--user NAME', 'User for login if not current user.') do |v|
-    opts[:user] = v
-  end
-  o.on('-p', '--password [TEXT]', "Password to use when connecting to server.  If password is not given it's solicited on the tty.") do |v|
-    opts[:password] = v
-  end
-end
-
-op.parse!
-
-=end
-command = "mysqldump -d"
-
-=begin
-command += " -u#{opts[:user]}" if opts[:user]
-command += " -h#{opts[:host]}" if opts[:host]
-command += " -P#{opts[:port]}" if opts[:port]
-command += " -p#{opts[:password]}" if opts[:password]
-=end
-command += " " + ARGV.join(' ')
-
-IO.popen(command, 'r') do |io|
-  create_flydata_ctl_table = true
-  while !io.eof?
-    mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(io)
-    if mysql_tabledef.nil?
-      # stream had no more create table definition
-      break
-    end
-    flydata_tabledef = mysql_tabledef.to_flydata_tabledef
-    puts Flydata::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table)
-    create_flydata_ctl_table = false
-  end
-end
-