flydata 0.7.2.1 → 0.7.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 041b42f8c7a79544e1075015c40b5f4d617eb983
4
- data.tar.gz: 18a938ab66c2754e67821cdb93178f0450991d8d
3
+ metadata.gz: 35042672a68ea10f4ec090aa77f218c885aed555
4
+ data.tar.gz: 4223e306573350bb767eda1259e71ccc9d7d182a
5
5
  SHA512:
6
- metadata.gz: 0afce08535788fde831998db168317d125ab4e4249f36dd9d10fee443ca3e62bb3dfce478bc3953e04d5d57dd7fa560e4ba2a1948c517021378f52a0686abc58
7
- data.tar.gz: e7af6dd1ec9470fd25a5c853858123112deb8f90c1a7543d09dc99c5564d75398c63f934125906f29bbf91b53d8cc8fa923f014b32c7b164f7554bf0058b44bd
6
+ metadata.gz: 143b46f4b1fb7c7e7472932b668caeb66c2201f91bf2bc334553583cf03a6f7800b4b1d895e50174e6043c2328abcf8836b2dc7c2e1848457bf1e0993d3149d9
7
+ data.tar.gz: 22ec19871da8e8d1545cf019ec09f4113fddf5ed59da2fe5f285cbce2e616db8f5ecb9320169efefd4444d79594b1907b55a507d8674a9999e2255f7c2ee69ca
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.2.1
1
+ 0.7.4
@@ -120,18 +120,13 @@ module FlydataCore
120
120
  def self._each_mysql_tabledef(tables, option)
121
121
  command = generate_mysql_ddl_dump_cmd(option.merge(tables: tables))
122
122
 
123
- create_opt = {}
124
- if option.has_key?(:skip_primary_key_check)
125
- create_opt[:skip_primary_key_check] = option[:skip_primary_key_check]
126
- end
127
-
128
123
  Open3.popen3(command) do |stdin, stdout, stderr|
129
124
  stdin.close
130
125
  stdout.set_encoding("utf-8", "utf-8") # mysqldump output must be in UTF-8
131
126
  create_flydata_ctl_table = true
132
127
  while !stdout.eof?
133
128
  begin
134
- mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, create_opt)
129
+ mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, option)
135
130
  break if mysql_tabledef.nil?
136
131
  yield(mysql_tabledef, nil)
137
132
  rescue FlydataCore::TableDefError=> e
@@ -1,9 +1,16 @@
1
+ require 'flydata-core/errors'
2
+
1
3
  module FlydataCore
2
4
  module TableDef
3
5
 
4
6
  class Base
7
+ def self.create(source_data, options = {})
8
+ params = _create(source_data, options)
9
+ params ? self.new(*params, options) : nil
10
+ end
11
+
5
12
  def initialize(table_def, table_name, columns, column_def, default_charset,
6
- default_source_charset, comment)
13
+ default_source_charset, comment, options = {})
7
14
  @table_def = table_def
8
15
  @table_name = table_name
9
16
  @columns = columns
@@ -11,6 +18,16 @@ class Base
11
18
  @default_charset = default_charset
12
19
  @default_source_charset = default_source_charset
13
20
  @comment = comment
21
+
22
+ opts = [:skip_primary_key_check, :pk_override].inject({}) do |h, key|
23
+ v = options[key]
24
+ v ||= options[key.to_s]
25
+ h[key] = v if v
26
+ h
27
+ end
28
+
29
+ @pk_override = self.class.override_pk(@table_name, @columns, opts)
30
+ self.class.check_pk(@table_name, @columns, opts)
14
31
  end
15
32
 
16
33
  attr_reader :columns, :column_def, :table_name, :default_source_charset
@@ -22,6 +39,7 @@ class Base
22
39
  tabledef[:default_charset] = @default_charset if @default_charset
23
40
  tabledef[:comment] = @comment if @comment
24
41
  tabledef[:src_ddl] = @table_def
42
+ tabledef[:pk_override] = @pk_override
25
43
 
26
44
  tabledef
27
45
  end
@@ -35,6 +53,41 @@ class Base
35
53
  sort{|a,b| a[:primary_key] <=> b[:primary_key]}.
36
54
  collect{|col| col[:column]}
37
55
  end
56
+
57
+ private
58
+
59
+ def self._create(source_data, options)
60
+ raise "subclass must implement"
61
+ end
62
+
63
+ def self.override_pk(table_name, columns, options)
64
+ pk_override = options[:pk_override]
65
+ return nil unless pk_override.kind_of?(Hash) && pk_override[table_name]
66
+
67
+ pkcols = pk_override[table_name]
68
+ pkcols_found = []
69
+ columns.each do |column|
70
+ if pkcols.include?(column[:column])
71
+ column[:primary_key] = true
72
+ pkcols_found << column[:column]
73
+ else
74
+ column.delete(:primary_key) if column.has_key?(:primary_key)
75
+ end
76
+ end
77
+ pkcols_missing = pkcols - pkcols_found
78
+ unless pkcols_missing.empty?
79
+ raise "Column(s) is specified as a primary key but the column does not exist in the table '#{table_name}': #{pkcols_missing.collect{|pc| "'#{pc}'"}.join(",")}"
80
+ end
81
+
82
+ pkcols
83
+ end
84
+
85
+ def self.check_pk(table_name, columns, options)
86
+ options
87
+ unless columns.any? {|column| column[:primary_key]} or options[:skip_primary_key_check]
88
+ raise TableDefError, {error: "no primary key defined", table: table_name}
89
+ end
90
+ end
38
91
  end
39
92
 
40
93
  end
@@ -85,11 +85,6 @@ class MysqlTableDef < Base
85
85
  nil
86
86
  end
87
87
 
88
- def self.create(io, options = {})
89
- params = _create(io, options)
90
- params ? self.new(*params) : nil
91
- end
92
-
93
88
  def self._create(io, options)
94
89
  table_def = ''
95
90
  table_name = nil
@@ -144,9 +139,6 @@ class MysqlTableDef < Base
144
139
  end
145
140
 
146
141
  when :after_create_table
147
- unless columns.any? {|column| column[:primary_key]} or options[:skip_primary_key_check]
148
- raise TableDefError, {error: "no primary key defined", table: table_name}
149
- end
150
142
  break
151
143
  end
152
144
  end
@@ -101,11 +101,6 @@ class PostgresqlTableDef < Base
101
101
  ret_type
102
102
  end
103
103
 
104
- def self.create(information_schema_columns, options)
105
- params = _create(information_schema_columns, options)
106
- params ? self.new(*params) : nil
107
- end
108
-
109
104
  private
110
105
 
111
106
  def self.get_width_values(information_schema_columns, type_hash)
@@ -380,6 +380,9 @@ EOS
380
380
  values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
381
381
  values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
382
382
  values << "('#{flydata_tabledef[:table_name]}', 'src_ddl', '#{escape(flydata_tabledef[:src_ddl])}')"
383
+ if flydata_tabledef[:pk_override]
384
+ values << "('#{flydata_tabledef[:table_name]}', 'pk_override', '#{escape(flydata_tabledef[:pk_override].join(","))}')"
385
+ end
383
386
  sql += values.join(",\n") + ";\n"
384
387
  sql
385
388
  end
@@ -15,6 +15,7 @@ describe Base do
15
15
  let(:default_charset) { double('default_charset') }
16
16
  let(:default_source_charset) { double('default_source_charset') }
17
17
  let(:comment) { double('comment') }
18
+ let(:options) { double('options') }
18
19
 
19
20
  describe '#pk_columns' do
20
21
  subject { subject_object.pk_columns }
@@ -45,6 +46,89 @@ describe Base do
45
46
  end
46
47
  end
47
48
  end
49
+
50
+ describe '.override_pk' do
51
+ subject { described_class.send(:override_pk, table_name, columns, options) }
52
+
53
+ let(:columns) { [
54
+ col1def,
55
+ col2def,
56
+ ] }
57
+
58
+ let(:col1def) { {column: col1name, primary_key: true} }
59
+ let(:col2def) { {column: col2name} }
60
+
61
+ let(:col1name) { double("col1name") }
62
+ let(:col2name) { double("col2name") }
63
+
64
+ shared_examples "nothings changed" do
65
+ it {
66
+ subject
67
+ expect(col1def[:primary_key]).to be_truthy
68
+ expect(col2def.has_key?(:primary_key)).to be_falsey
69
+ }
70
+ end
71
+
72
+ context 'when no :pk_override attribute exists' do
73
+ let(:options) { {} }
74
+
75
+ it_behaves_like "nothings changed"
76
+ end
77
+
78
+ context 'when the table has no pk_override' do
79
+ let(:options) { {pk_override:{}} }
80
+
81
+ it_behaves_like "nothings changed"
82
+ end
83
+
84
+ context 'when the table has pk_override' do
85
+ let(:options) { {pk_override:{table_name => [col2name]}} }
86
+
87
+ it 'overrides primary key settings' do
88
+ subject
89
+ expect(col1def.has_key?(:primary_key)).to be_falsey
90
+ expect(col2def[:primary_key]).to be_truthy
91
+ end
92
+ end
93
+
94
+ context 'when the column specified in pk_override does not exist' do
95
+ let(:options) { {pk_override:{table_name => ["asdf"]}} }
96
+
97
+ it do
98
+ expect{subject}.to raise_error /the column does not exist in the table/
99
+ end
100
+ end
101
+ end
102
+
103
+ describe '.check_pk' do
104
+ subject { described_class.send(:check_pk, table_name, columns, options) }
105
+
106
+ let(:columns) { [
107
+ col1def,
108
+ col2def,
109
+ ] }
110
+
111
+ let(:col1def) { {} }
112
+ let(:col2def) { {} }
113
+ let(:options) { {} }
114
+
115
+ context 'when a pk column exists' do
116
+ let(:col1def) { {primary_key: true} }
117
+ it { expect{subject}.to_not raise_error }
118
+
119
+ end
120
+ context 'when no pk column exists' do
121
+ let(:col1def) { {} }
122
+ let(:col2def) { {} }
123
+
124
+ it { expect{subject}.to raise_error(TableDefError) }
125
+
126
+ context 'if skip_primary_key_check option is set' do
127
+ let(:options) { {skip_primary_key_check: true} }
128
+ it { expect{subject}.to_not raise_error }
129
+ end
130
+ end
131
+ end
48
132
  end
49
133
 
50
134
  end
Binary file
@@ -16,6 +16,7 @@ module Fluent
16
16
  config_param :data_entry_type, :string, default: nil # data entry type
17
17
  config_param :tables, :string
18
18
  config_param :tables_append_only, :string
19
+ config_param :pk_override, :hash, default: {}
19
20
  config_param :tag, :string
20
21
  config_param :position_file, :string, default: 'position.binlog.pos'
21
22
  end
@@ -23,8 +23,9 @@ class PostgresqlQueryBasedFlydataInput < Input
23
23
  @dbconf = FlydataCore::Postgresql::Config.opts_for_pg(@data_entry['postgresql_data_entry_preference'])
24
24
  $log.info "postgresql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{@tables_append_only}\""
25
25
 
26
+ opts = @dbconf.merge(pk_override:@data_entry['postgresql_data_entry_preference']['pk_override'])
26
27
  @table_meta = Flydata::SourcePostgresql::TableMeta.new(
27
- @dbconf, @tables, @schema)
28
+ opts, @tables, @schema)
28
29
 
29
30
  @context = Flydata::SourcePostgresql::PluginSupport::Context.new(
30
31
  database: @database, tables: @tables,
@@ -9,6 +9,7 @@ class DataEntry < Source::DataEntry
9
9
  database: {},
10
10
  tables: {},
11
11
  tables_append_only: {},
12
+ pk_override: {},
12
13
  host: {},
13
14
  port: {},
14
15
  username: {},
@@ -9,6 +9,7 @@ class DataEntry < Source::DataEntry
9
9
  database: {},
10
10
  tables: {},
11
11
  tables_append_only: {},
12
+ pk_override: {},
12
13
  host: {},
13
14
  port: {},
14
15
  username: {},
@@ -71,6 +71,11 @@ EOS
71
71
  cli = FlydataCore::Postgresql::PGClient.new(de_prefs)
72
72
  table_meta.reload(cli)
73
73
 
74
+ tables_missing_meta = tables.select{|t| tm = table_meta[t]; tm.nil? || tm.empty?}
75
+ unless tables_missing_meta.empty?
76
+ raise "Tables are not available. Check if the following table(s) exist and are visible: #{tables_missing_meta.join(",")}"
77
+ end
78
+
74
79
  context = source.sync_generate_table_ddl(dp, nil)
75
80
  source_pos = get_source_pos(table_meta.current_snapshot, &src_pos_callback)
76
81
 
@@ -10,6 +10,11 @@ module QueryBasedSync
10
10
  class ResourceRequester < Flydata::QueryBasedSync::ResourceRequester
11
11
  RESPONSE_CLASS = Flydata::SourcePostgresql::QueryBasedSync::Response
12
12
 
13
+ def initialize(*args)
14
+ super
15
+ @tables_missing_meta = []
16
+ end
17
+
13
18
  # Override
14
19
  def create_resource_client(context)
15
20
  FlydataCore::Postgresql::PGClient.new(context.dbconf)
@@ -20,6 +25,14 @@ class ResourceRequester < Flydata::QueryBasedSync::ResourceRequester
20
25
  table_sym = table_name.to_sym
21
26
  t_meta = context.table_meta[table_sym]
22
27
 
28
+ if t_meta.nil? || t_meta.empty?
29
+ unless @tables_missing_meta.include?(table_name)
30
+ @tables_missing_meta << table_name
31
+ log_error "No metadata is available for table '#{table_name}'. Make sure the table exists and is visible to the PostgreSQL user."
32
+ end
33
+ return
34
+ end
35
+
23
36
  table_src_pos = context.table_src_pos_files[table_sym].pos
24
37
  master_src_pos = context.cur_src_pos_file.pos
25
38
 
@@ -66,7 +66,7 @@ WHERE
66
66
  AND c.table_schema IN (%{schema})
67
67
  AND c.table_name IN (%{tables})
68
68
  ORDER BY
69
- c.table_name, c.ordinal_position;
69
+ c.table_name, c.ordinal_position, i.indisprimary asc;
70
70
  EOT
71
71
 
72
72
  GET_CURRENT_SNAPSHOT_SQL = "SELECT txid_current_snapshot();"
@@ -74,11 +74,11 @@ EOT
74
74
  DEFAULT_MAX_FETCH_RECORD_SIZE = 50000
75
75
  #DEFAULT_MAX_FETCH_RECORD_SIZE = 8
76
76
 
77
- def initialize(dbconf, tables, schema = nil)
78
- @dbconf = dbconf
79
- @database = dbconf[:dbname] || dbconf[:database] || dbconf['database']
77
+ def initialize(options, tables, schema = nil)
78
+ @options = options
79
+ @database = options[:dbname] || options[:database] || options['database']
80
80
  @tables = tables
81
- @schema = (schema || dbconf[:schema] || dbconf['schema']).to_s.strip
81
+ @schema = (schema || options[:schema] || options['schema']).to_s.strip
82
82
  @schema = @schema.empty? ? nil : @schema
83
83
  end
84
84
 
@@ -94,7 +94,7 @@ EOT
94
94
 
95
95
  conn = pg_client
96
96
  if conn.nil?
97
- local_conn = conn = FlydataCore::Postgresql::PGClient.new(@dbconf)
97
+ local_conn = conn = FlydataCore::Postgresql::PGClient.new(@options)
98
98
  end
99
99
 
100
100
  # Set table_meta
@@ -126,7 +126,7 @@ EOT
126
126
  ret.each do |table_name, t_meta|
127
127
  begin
128
128
  table_def = FlydataCore::TableDef::PostgresqlTableDef.create(
129
- t_meta[:raw_columns].values, :skip_primary_key_check)
129
+ t_meta[:raw_columns].values, @options)
130
130
  rescue FlydataCore::TableDefError => e
131
131
  t_meta.merge!(
132
132
  table_name: table_name,
@@ -249,6 +249,7 @@ EOT
249
249
  'database' => 'test_db',
250
250
  'tables' => 'test_table,test_table_1,test_table_2',
251
251
  'tables_append_only' => 'test_table_3',
252
+ 'pk_override' => {},
252
253
  'host' => 'localhost',
253
254
  'port' => 3306,
254
255
  'username' => 'test_admin',
@@ -49,6 +49,7 @@ EOT
49
49
  'database' => 'test_db',
50
50
  'tables' => 'test_table,test_table_1,test_table_2',
51
51
  'tables_append_only' => 'test_table_3',
52
+ 'pk_override' => {},
52
53
  'host' => 'localhost',
53
54
  'port' => 5433,
54
55
  'username' => 'test_admin',
@@ -56,6 +56,7 @@ describe GenerateSourceDump do
56
56
  expect(Flydata::SourcePostgresql::TableMeta).to receive(:new).with(de_prefs, tables).and_return(table_meta)
57
57
  expect(table_meta).to receive(:reload).with(cli).once
58
58
  expect(table_meta).to receive(:current_snapshot).and_return(snapshot)
59
+ expect(table_meta).to receive(:[]).and_return({something_in_it:true}).exactly(tables.count).times
59
60
  expect(subject_object).to receive(:get_source_pos).
60
61
  with(snapshot, &src_pos_callback).and_return(source_pos)
61
62
  expect(source).to receive(:sync_generate_table_ddl).with(dp, nil).
@@ -83,6 +84,7 @@ describe GenerateSourceDump do
83
84
  expect(Flydata::SourcePostgresql::TableMeta).to receive(:new).with(de_prefs, tables).and_return(table_meta)
84
85
  expect(table_meta).to receive(:reload).with(cli).once
85
86
  expect(table_meta).to receive(:current_snapshot).and_return(snapshot)
87
+ allow(table_meta).to receive(:[]).and_return({something_in_it:true})
86
88
  expect(subject_object).to receive(:get_source_pos).
87
89
  with(snapshot, &src_pos_callback).and_return(source_pos)
88
90
  allow(source).to receive(:sync_generate_table_ddl).with(dp, nil).
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.2.1
4
+ version: 0.7.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2016-04-21 00:00:00.000000000 Z
15
+ date: 2016-04-28 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -481,7 +481,6 @@ dependencies:
481
481
  description: FlyData Agent
482
482
  email: sysadmin@flydata.com
483
483
  executables:
484
- - fdmysqldump
485
484
  - fdredshift
486
485
  - flydata
487
486
  - serverinfo
@@ -500,7 +499,6 @@ files:
500
499
  - benchmark/data/insert_parser_test_data_num_only.sql.gz
501
500
  - benchmark/insert_parser_bench.rb
502
501
  - benchmark/insert_parser_prof.rb
503
- - bin/fdmysqldump
504
502
  - bin/fdredshift
505
503
  - bin/flydata
506
504
  - bin/serverinfo
@@ -869,7 +867,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
869
867
  version: '0'
870
868
  requirements: []
871
869
  rubyforge_project:
872
- rubygems_version: 2.2.2
870
+ rubygems_version: 2.4.3
873
871
  signing_key:
874
872
  specification_version: 4
875
873
  summary: FlyData Agent
@@ -1,62 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- #require 'optparse'
4
-
5
- lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
6
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
7
-
8
- require 'flydata/table_def'
9
-
10
- # Deprecation warning
11
- $stderr.puts "WARNING: Use 'flydata sync:generate_table_ddl' command instead"
12
-
13
- if `which mysqldump`.empty?
14
- abort "mysqldump is not installed."
15
- end
16
-
17
- # TODO Implement options instead of passing all to mysqldump.
18
- =begin
19
- opts = {}
20
-
21
- op = OptionParser.new do |o|
22
- o.on('-h', '--host NAME', 'Connect to host.') do |v|
23
- opts[:host] = v
24
- end
25
- o.on('-P', '--port NUMBER', 'Port number to use for connection.') do |v|
26
- opts[:port] = v
27
- end
28
- o.on('-u', '--user NAME', 'User for login if not current user.') do |v|
29
- opts[:user] = v
30
- end
31
- o.on('-p', '--password [TEXT]', "Password to use when connecting to server. If password is not given it's solicited on the tty.") do |v|
32
- opts[:password] = v
33
- end
34
- end
35
-
36
- op.parse!
37
-
38
- =end
39
- command = "mysqldump -d"
40
-
41
- =begin
42
- command += " -u#{opts[:user]}" if opts[:user]
43
- command += " -h#{opts[:host]}" if opts[:host]
44
- command += " -P#{opts[:port]}" if opts[:port]
45
- command += " -p#{opts[:password]}" if opts[:password]
46
- =end
47
- command += " " + ARGV.join(' ')
48
-
49
- IO.popen(command, 'r') do |io|
50
- create_flydata_ctl_table = true
51
- while !io.eof?
52
- mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(io)
53
- if mysql_tabledef.nil?
54
- # stream had no more create table definition
55
- break
56
- end
57
- flydata_tabledef = mysql_tabledef.to_flydata_tabledef
58
- puts Flydata::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table)
59
- create_flydata_ctl_table = false
60
- end
61
- end
62
-