flydata 0.7.5 → 0.7.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 44371bbacace581ba7bfc886a977202c84028ff9
4
- data.tar.gz: bbac27c82da13ad6fce04dbbdc420feebc73a17c
3
+ metadata.gz: 0a74c7dd378333e32076bf9d8dfb6f445f2a6b07
4
+ data.tar.gz: 4fbc074d26fee614f5315b674159543db96da68b
5
5
  SHA512:
6
- metadata.gz: 9b52b6990ae5706dfdb79a1a81a10666d09de149562dac3d2f9defb23383625c8c8b429d006c8d9a84978f1362cd74c81f05d0cd5c35b042164af32c06462059
7
- data.tar.gz: bfbe4e1c24723bd23fa8650427fa3daed18d7c619ace48e3bcb1e050822a495c80e007d46989758fe6327f20e6a9b4a64f4712e4ab2b15bc44430db8b5fb6043
6
+ metadata.gz: 5e88618472bc0641320931b46ab357ba02c0f17d678b1b66e5d1ef467e2ca4b0518414d8c2037f6177c310b18c2dbb7825b090649ad6615ec621e687a9355118
7
+ data.tar.gz: ad9d6fd284b20b55f36ed25e4c86d54fc323e856d691179e7611a87c424f94c892b33dc1ea09d8e35b7b6e49b861e573ce03611272020d69913982ea9c15473e
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.5
1
+ 0.7.6
@@ -9,24 +9,28 @@ class Base
9
9
  params ? self.new(*params, options) : nil
10
10
  end
11
11
 
12
- def initialize(table_def, table_name, columns, column_def, default_charset,
12
+ def initialize(table_def, table_name, columns, column_def, unique_keys, default_charset,
13
13
  default_source_charset, comment, options = {})
14
14
  @table_def = table_def
15
15
  @table_name = table_name
16
16
  @columns = columns
17
17
  @column_def = column_def
18
+ @unique_keys = unique_keys
18
19
  @default_charset = default_charset
19
20
  @default_source_charset = default_source_charset
20
21
  @comment = comment
21
22
 
23
+ # example of opts
24
+ # { :skip_primary_key_check => true,
25
+ # :pk_override => { "table_3" => ["id", "group_id"], "table_5" => ["product_id"] }
26
+ # }
22
27
  opts = [:skip_primary_key_check, :pk_override].inject({}) do |h, key|
23
28
  v = options[key]
24
29
  v ||= options[key.to_s]
25
30
  h[key] = v if v
26
31
  h
27
32
  end
28
-
29
- @pk_override = self.class.override_pk(@table_name, @columns, opts)
33
+ @pk_override, @pk_status = get_pk_override_with_status(@table_name, @columns, @unique_keys, opts)
30
34
  self.class.check_pk(@table_name, @columns, opts)
31
35
  end
32
36
 
@@ -54,12 +58,59 @@ class Base
54
58
  collect{|col| col[:column]}
55
59
  end
56
60
 
61
+ def get_pk_override_as_hash
62
+ @pk_override ? { @table_name => @pk_override } : nil
63
+ end
64
+
65
+ def converting_uk_to_pk?
66
+ @pk_status == :pk_unique
67
+ end
68
+
57
69
  private
58
70
 
59
71
  def self._create(source_data, options)
60
72
  raise "subclass must implement"
61
73
  end
62
74
 
75
+ # Get pk_override with pk_status
76
+ # [Arguments]
77
+ # - table_name (String):
78
+ # - columns (Hash):
79
+ # columns that includes all column info from source DB
80
+ # - unique_keys (Hash):
81
+ # valid unique keys on the table which can use as a primary key
82
+ # - opts (Hash):
83
+ # including pk_override values. See a comment above.
84
+ # [Return values]
85
+ # pk_status (Symbol)
86
+ # :pk_original using original pk from source DB
87
+ # :pk_overridden using user_pk_override from web server
88
+ # :pk_unique converting unique key (uk) to pk. Need to update the table attributes (UK_AS_PK_OVERRIDE) on web server
89
+ # :pk_none none of above
90
+ # pk_override (Hash):
91
+ # pk_override or nil
92
+ def get_pk_override_with_status(table_name, columns, unique_keys, opts)
93
+
94
+ pk_override = self.class.override_pk(table_name, columns, opts)
95
+
96
+ if pk_override
97
+ pk_status = :pk_overridden
98
+ elsif self.class.pk_exist?(@columns)
99
+ # real primary key exist. Not using pk_override
100
+ pk_status = :pk_original
101
+ else
102
+ # use an unique key as a pk_override if any.
103
+ new_pk_override = self.class.uk_to_pk_override(table_name, columns, unique_keys)
104
+ if new_pk_override
105
+ pk_override = self.class.override_pk(table_name, columns, new_pk_override)
106
+ pk_status = :pk_unique
107
+ else
108
+ pk_status = :pk_none
109
+ end
110
+ end
111
+ [pk_override, pk_status]
112
+ end
113
+
63
114
  def self.override_pk(table_name, columns, options)
64
115
  pk_override = options[:pk_override]
65
116
  return nil unless pk_override.kind_of?(Hash) && pk_override[table_name]
@@ -83,11 +134,29 @@ class Base
83
134
  end
84
135
 
85
136
  def self.check_pk(table_name, columns, options)
86
- options
87
- unless columns.any? {|column| column[:primary_key]} or options[:skip_primary_key_check]
137
+ unless pk_exist?(columns) || options[:skip_primary_key_check]
88
138
  raise TableDefError, {error: "no primary key defined", table: table_name}
89
139
  end
90
140
  end
141
+
142
+ def self.pk_exist?(columns)
143
+ columns.any? {|column| column[:primary_key]}
144
+ end
145
+
146
+ def self.uk_exist?(columns, unique_keys)
147
+ unique_keys.any?
148
+ end
149
+
150
+ def self.uk_to_pk_override(table_name, columns, unique_keys)
151
+ return nil unless uk_exist?(columns, unique_keys)
152
+
153
+ # use the first unique key as a substitue of primary key
154
+ new_pk_override = {}
155
+ new_pk_override[:pk_override] = { table_name => unique_keys.first }
156
+ $log.info %Q|Using unique key as primary key: {"#{table_name}"=>#{new_pk_override[:pk_override][table_name].inspect}}|
157
+
158
+ new_pk_override
159
+ end
91
160
  end
92
161
 
93
162
  end
@@ -90,6 +90,7 @@ class MysqlTableDef < Base
90
90
  table_name = nil
91
91
  columns = []
92
92
  column_def = {}
93
+ unique_keys = []
93
94
  default_charset = nil
94
95
  default_source_charset = nil
95
96
  comment = nil
@@ -133,7 +134,8 @@ class MysqlTableDef < Base
133
134
  elsif stripped_line.start_with?("CONSTRAINT")
134
135
  # constraint definition. No acction required.
135
136
  elsif stripped_line.start_with?("UNIQUE KEY")
136
- parse_key(line, columns, :unique)
137
+ unique_key, include_null_column = parse_key(line, columns, :unique)
138
+ unique_keys << unique_key unless include_null_column
137
139
  else
138
140
  # "Unknown table definition. Skip. (#{line})"
139
141
  end
@@ -142,7 +144,7 @@ class MysqlTableDef < Base
142
144
  break
143
145
  end
144
146
  end
145
- position == :after_create_table ? [table_def, table_name, columns, column_def, default_charset, default_source_charset, comment] : nil
147
+ position == :after_create_table ? [table_def, table_name, columns, column_def, unique_keys, default_charset, default_source_charset, comment] : nil
146
148
  end
147
149
 
148
150
  # Replaces a MySQL string with an empty string ('')
@@ -307,15 +309,18 @@ class MysqlTableDef < Base
307
309
  line = /\((?:`.*?`(?:\(.*?\))?(?:,\s*)?)+\)/.match(line)[0]
308
310
  keys = line.scan(/`(.*?)`/).collect{|item| item[0]}
309
311
 
312
+ include_null_column = false
310
313
  keys.each do |key|
311
314
  column = columns.detect {|column|
312
315
  column[:column] === key
313
316
  }
314
317
  raise "Key #{key} must exist in the definition " if column.nil?
315
318
  column[type] = true
319
+ include_null_column = true unless column[:not_null]
316
320
  end
317
- end
318
321
 
322
+ [keys, include_null_column]
323
+ end
319
324
  end
320
325
 
321
326
  end
@@ -123,7 +123,7 @@ class PostgresqlTableDef < Base
123
123
  end
124
124
 
125
125
  def self._create(information_schema_columns, options)
126
- table_def = information_schema_columns.inspect
126
+ table_def = information_schema_columns.collect {|iscol| iscol.first}.inspect
127
127
  table_name = nil
128
128
  columns = []
129
129
  column_def = {}
@@ -132,8 +132,12 @@ class PostgresqlTableDef < Base
132
132
  default_charset = 'UTF_8'
133
133
  default_charset_postgresql = 'UTF8'
134
134
  comment = nil
135
+ unique_keys_hash = Hash.new {|h, k| h[k] = []}
135
136
 
136
- information_schema_columns.each do |iscol|
137
+ information_schema_columns.each do |iscol_arr|
138
+ # An iscol_arr represents a column. Column information in all elements in iscol_arr is the same.
139
+ # Only difference between elements is index information.
140
+ iscol = iscol_arr.first
137
141
  column = parse_one_column_def(iscol)
138
142
  if table_name
139
143
  unless table_name == column[:table]
@@ -145,8 +149,17 @@ class PostgresqlTableDef < Base
145
149
  columns << column
146
150
  coldef = iscol.inspect
147
151
  column_def[column[:column]] = coldef
152
+
153
+ iscol_arr.each do |iscol|
154
+ # gather information for unique keys that this column belongs to.
155
+ if iscol['is_primary'] == 'f' && iscol['is_unique'] == 't' && iscol['is_valid'] == 't' && iscol['is_partial'] == 'f'
156
+ unique_keys_hash[iscol['index_name'].to_sym] << iscol['column_name']
157
+ end
158
+ end
148
159
  end
149
- [table_def, table_name, columns, column_def, default_charset, default_charset_postgresql, comment]
160
+ unique_keys = unique_keys_hash.values
161
+
162
+ [table_def, table_name, columns, column_def, unique_keys, default_charset, default_charset_postgresql, comment]
150
163
  end
151
164
 
152
165
  def self.parse_one_column_def(information_schema_column)
@@ -5,13 +5,14 @@ module TableDef
5
5
 
6
6
  describe Base do
7
7
  let(:subject_object) { described_class.new(table_def, table_name, columns,
8
- column_def, default_charset,
8
+ column_def, unique_keys, default_charset,
9
9
  default_source_charset, comment) }
10
10
 
11
11
  let(:table_def) { double('table_def') }
12
12
  let(:table_name) { double('table_name') }
13
13
  let(:columns) { double('columns') }
14
14
  let(:column_def) { double('column_def') }
15
+ let(:unique_keys) { double('unique_keys') }
15
16
  let(:default_charset) { double('default_charset') }
16
17
  let(:default_source_charset) { double('default_source_charset') }
17
18
  let(:comment) { double('comment') }
@@ -27,7 +28,6 @@ describe Base do
27
28
  {column: "user_id", primary_key: pk2_value},
28
29
  ]
29
30
  end
30
-
31
31
 
32
32
  context 'with unordered primary keys' do
33
33
  let(:pk1_value) { 't' }
data/flydata.gemspec CHANGED
Binary file
@@ -164,11 +164,23 @@ module Flydata
164
164
  end
165
165
 
166
166
  # Update validity of tables
167
- # table_validity_hash: {"updated_tables":{ "bad_table": "error reason", "good_table": nil }}
168
- # table "bad_table" will be marked invalid with reason "error reason"
169
- # table "good table" will be marked valid, that is, clear its error reason if it's set.
170
- def update_table_validity(data_entry_id, table_validity_hash)
171
- @client.post("/#{@model_name.pluralize}/#{data_entry_id}/update_table_validity", {:headers => {:content_type => :json}}, table_validity_hash.to_json)
167
+ # -- Params ( Agent 0.7.4 and older )
168
+ # params = {
169
+ # updated_tables: { "bad_table"=>"error reason", "good_table"=>nil}
170
+ # }
171
+ # -- Params ( Agent 0.7.5 and later )
172
+ # table_update_hash = {
173
+ # updated_tables: {
174
+ # "<table-name>" => {
175
+ # "invalid_table_reason" => "<error reason>",
176
+ # "uk_as_pk" => ['xx', ...],
177
+ # }
178
+ # }
179
+ # }
180
+ # - If value is nil, the setting will be deleted.
181
+ # - If table's attribute hash doesn't have an attribute key, the setting for the attribute for the table will not be changed
182
+ def update_table_validity(data_entry_id, table_update_hash)
183
+ @client.post("/#{@model_name.pluralize}/#{data_entry_id}/update_table_validity", {:headers => {:content_type => :json}}, table_update_hash.to_json)
172
184
  end
173
185
 
174
186
  # Tells the server that an initial sync has completed
@@ -15,6 +15,7 @@ require 'flydata/output/forwarder'
15
15
  require 'flydata/parser'
16
16
  require 'flydata/preference/data_entry_preference'
17
17
  require 'flydata/sync_file_manager'
18
+ require 'flydata/table_attribute'
18
19
  require 'flydata-core/table_def'
19
20
  require 'flydata/table_ddl'
20
21
  require 'flydata/event/api_event_sender'
@@ -1181,8 +1182,9 @@ EOM
1181
1182
  raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file"
1182
1183
  end
1183
1184
 
1184
- flydata_tabledefs, error_list = context.generate_flydata_tabledef(tables,
1185
+ flydata_tabledefs, error_list, uk_as_pk_override = context.generate_flydata_tabledef(tables,
1185
1186
  skip_primary_key_check: opts.skip_primary_key_check?)
1187
+
1186
1188
  create_flydata_ctl_table = true
1187
1189
  append_only = tables & @append_only_tables
1188
1190
  flydata_tabledefs.each do |flydata_tabledef|
@@ -1191,7 +1193,8 @@ EOM
1191
1193
  flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?, skip_drop_table: skip_drop_table, skip_primary_key_check: opts.skip_primary_key_check?)
1192
1194
  create_flydata_ctl_table = false
1193
1195
  end
1194
- table_validity_hash = {}
1196
+
1197
+ table_validity_hash = Hash.new {|h,k| h[k] = {}}
1195
1198
  tables_without_error = tables
1196
1199
  unless error_list.empty?
1197
1200
  log_error_stderr("\n\nERROR: FlyData Sync will not sync the following table(s) due to an error.")
@@ -1200,7 +1203,7 @@ EOM
1200
1203
  group_error[error].each do |hash|
1201
1204
  if table = hash[:table]
1202
1205
  log_error_stderr(" - #{table} (#{error})")
1203
- table_validity_hash[table] = error
1206
+ table_validity_hash[table][TableAttribute::INVALID_TABLE_REASON] = error
1204
1207
  end
1205
1208
  end
1206
1209
  end
@@ -1215,9 +1218,13 @@ The other tables are ready to sync. To start sync, run the generated script on
1215
1218
  EOS
1216
1219
  end
1217
1220
  end
1221
+ tables_without_error.each {|table| table_validity_hash[table][TableAttribute::INVALID_TABLE_REASON] = nil }
1222
+
1223
+ # update table validity and uk_as_pk info on web server
1224
+ uk_as_pk_override.each_pair {|table, val| table_validity_hash[table][TableAttribute::UK_AS_PK_OVERRIDE] = val}
1225
+ (tables - uk_as_pk_override.keys).each {|table| table_validity_hash[table][TableAttribute::UK_AS_PK_OVERRIDE] = nil }
1218
1226
 
1219
- tables_without_error.each {|table| table_validity_hash[table] = nil }
1220
- flydata.data_entry.update_table_validity(de['id'], {updated_tables: table_validity_hash}) unless table_validity_hash.empty?
1227
+ flydata.data_entry.update_table_validity(de['id'], updated_tables: table_validity_hash) unless table_validity_hash.empty?
1221
1228
 
1222
1229
  sync_fm = create_sync_file_manager(de)
1223
1230
  sync_fm.save_generated_ddl(tables_without_error, TableDdl::VERSION)
@@ -3,6 +3,7 @@ require 'flydata/source'
3
3
  require 'flydata-core/fluent/config_helper'
4
4
  require 'flydata/fluent-plugins/flydata_plugin_ext/flush_support'
5
5
  require 'flydata/fluent-plugins/flydata_plugin_ext/transaction_support'
6
+ require 'flydata/table_attribute'
6
7
 
7
8
  module Fluent
8
9
 
@@ -17,6 +18,7 @@ module Fluent
17
18
  config_param :tables, :string
18
19
  config_param :tables_append_only, :string
19
20
  config_param :pk_override, :hash, default: {}
21
+ config_param :table_attributes, :array, default: []
20
22
  config_param :tag, :string
21
23
  config_param :position_file, :string, default: 'position.binlog.pos'
22
24
  end
@@ -61,6 +63,26 @@ module Fluent
61
63
  h
62
64
  end
63
65
 
66
+ # Re-construct pk_override & table_attributes
67
+ # @pk_override should include both USER_PK_OVERRIDE and UK_AS_PK_OVERRIDE.
68
+ # If mismatch happens here, @pk_override value should come from a local conf file.
69
+ #TODO support UK_AS_PK_OVERRIDE for tables existing only in local conf, not registered on the web.
70
+
71
+ pk_override_from_table_attrs = Flydata::TableAttribute::generate_pk_override(@table_attributes)
72
+
73
+ if @pk_override != pk_override_from_table_attrs
74
+ # @pk_override in a local conf file only includes USER_PK_OVERRIDE. Update table_attributes.
75
+ Flydata::TableAttribute::delete_attribute(@table_attributes, Flydata::TableAttribute::USER_PK_OVERRIDE)
76
+ Flydata::TableAttribute::save_attribute(@table_attributes, @pk_override, Flydata::TableAttribute::USER_PK_OVERRIDE)
77
+
78
+ @pk_override = Flydata::TableAttribute::generate_pk_override(@table_attributes)
79
+ $log.info "Update primary key override. Using #{@pk_override}"
80
+
81
+ prefs = @data_entry['mysql_data_entry_preference'] || @data_entry['postgresql_data_entry_preference']
82
+ prefs['table_attributes'] = @table_attributes
83
+ prefs['pk_override'] = @pk_override
84
+ end
85
+
64
86
  $log.info("Source position - resume_pos:'#{@source_position_file.read rescue IOError}' " +
65
87
  "sent_pos:'#{@sent_position_file.read rescue nil}'")
66
88
  end
@@ -1,6 +1,7 @@
1
1
  require 'flydata/source'
2
2
  require 'flydata/source/component'
3
3
  require 'flydata/source/errors'
4
+ require 'flydata/table_attribute'
4
5
 
5
6
  module Flydata
6
7
  module Source
@@ -68,6 +69,35 @@ class Sync < Component
68
69
  prefs['new_tables'] =
69
70
  prefs['new_tables'].kind_of?(String) ?
70
71
  prefs['new_tables'].split(/(?:\s*,\s*|\s+)/).uniq : []
72
+
73
+ setup_pk_override_prefs(prefs)
74
+ end
75
+
76
+ def setup_pk_override_prefs(prefs)
77
+ prefs['pk_override'] = {} if prefs['pk_override'].nil?
78
+ prefs['user_pk_override'] = TableAttribute::extract_attribute(prefs['table_attributes'],
79
+ TableAttribute::USER_PK_OVERRIDE)
80
+ prefs['uk_as_pk_override'] = TableAttribute::extract_attribute(prefs['table_attributes'],
81
+ TableAttribute::UK_AS_PK_OVERRIDE)
82
+
83
+ # Re-construct pk_override & table_attributes
84
+ # prefs['pk_override'] should include both USER_PK_OVERRIDE and UK_AS_PK_OVERRIDE.
85
+ # If mismatch happens here, prefs['pk_override'] value should come from a local conf file.
86
+ #TODO support UK_AS_PK_OVERRIDE for tables existing only in local conf, not registered on the web.
87
+
88
+ pk_override_from_table_attrs = TableAttribute::generate_pk_override(prefs['table_attributes'])
89
+
90
+ if prefs['pk_override'] != pk_override_from_table_attrs
91
+ # prefs['pk_override'] in a local conf file only includes USER_PK_OVERRIDE.
92
+ prefs['user_pk_override'] = prefs['pk_override']
93
+
94
+ # Update table_attributes.
95
+ TableAttribute::delete_attribute(prefs['table_attributes'], TableAttribute::USER_PK_OVERRIDE)
96
+ TableAttribute::save_attribute(prefs['table_attributes'], prefs['user_pk_override'], TableAttribute::USER_PK_OVERRIDE)
97
+
98
+ prefs['pk_override'] = TableAttribute::generate_pk_override(prefs['table_attributes'])
99
+ $log.info "Primary key override is updated. Using pk_override: #{prefs['pk_override']}"
100
+ end
71
101
  end
72
102
  end
73
103
 
@@ -28,21 +28,30 @@ class SyncGenerateTableDdl < Component
28
28
 
29
29
  def generate_flydata_tabledef(tables, options)
30
30
  prefs = data_entry_prefs
31
+ reset_uk_as_pk_override(tables, prefs)
31
32
  options = options.merge(prefs)
32
33
  flydata_tabledefs = []
33
34
  error_list = []
35
+ uk_as_pk_override = {}
36
+
34
37
  missing_tables = each_source_tabledef(tables, options) do |source_tabledef, error|
35
38
  if error
36
39
  error_list << error.err_hash
37
40
  next
38
41
  end
42
+
43
+ if source_tabledef.converting_uk_to_pk?
44
+ new_pk_override = source_tabledef.get_pk_override_as_hash
45
+ uk_as_pk_override.merge!(new_pk_override)
46
+ end
39
47
  flydata_tabledefs << source_tabledef.to_flydata_tabledef
40
48
  end
49
+
41
50
  if missing_tables
42
51
  missing_tables.each {|missing_table| error_list << { error: "table does not exist in the #{data_source_type_display_name}", table: missing_table } }
43
52
  end
44
53
 
45
- [flydata_tabledefs, error_list]
54
+ [flydata_tabledefs, error_list, uk_as_pk_override]
46
55
  end
47
56
 
48
57
  private
@@ -77,6 +86,21 @@ class SyncGenerateTableDdl < Component
77
86
  def each_source_tabledef(tables, options, &block)
78
87
  raise UnsupportedSourceError, "subclass must implement"
79
88
  end
89
+
90
+ def reset_uk_as_pk_override(target_tables, prefs)
91
+ # Reset uk_as_pk_override for target tables.
92
+ prefs['uk_as_pk_override'].delete_if do |table_name, val|
93
+ target_tables.include?(table_name)
94
+ end
95
+ update_pk_override_pref(prefs)
96
+ end
97
+
98
+ def update_pk_override_pref(prefs)
99
+ # pk_override = user_pk_override + uk_as_pk_override
100
+ # (`user_pk_override` takes precedence)
101
+ prefs['pk_override'] = prefs['uk_as_pk_override'].merge( prefs['user_pk_override'] )
102
+ $log.info "primary key override: #{prefs['pk_override']}"
103
+ end
80
104
  end
81
105
 
82
106
  end
@@ -9,10 +9,33 @@ module Flydata
9
9
 
10
10
  def generate_command(dbconf, args)
11
11
  dbconf.delete('tables')
12
- dbconf[:custom_option_end] = args.join(' ')
12
+ dbconf[:custom_option_end] = build_custom_option_end(args)
13
13
  dbconf[:command] = 'mysqldump'
14
14
  FlydataCore::Mysql::CommandGenerator::generate_mysql_cmd(dbconf)
15
15
  end
16
+
17
+ # Workaround to build --where options correctly
18
+ #
19
+ # [Background]
20
+ # Arguments including single/double quotations don't work because those quotations
21
+ # are removed from the system.
22
+ #
23
+ # User's command:
24
+ # flydata mysqldump table_a --where "id = 1001"
25
+ # ARGV:
26
+ # ["table_a", "--where", "id", "=", "1001"]
27
+ #
28
+ def build_custom_option_end(args)
29
+ ret = []
30
+ while v = args.shift
31
+ ret << v
32
+ if v == '--where'
33
+ ret << %Q|"#{args.join(' ')}"|
34
+ break
35
+ end
36
+ end
37
+ ret.join(' ')
38
+ end
16
39
  end
17
40
  end
18
41
  end
@@ -10,6 +10,7 @@ class DataEntry < Source::DataEntry
10
10
  tables: {},
11
11
  tables_append_only: {},
12
12
  pk_override: {},
13
+ table_attributes: {},
13
14
  host: {},
14
15
  port: {},
15
16
  username: {},
@@ -475,11 +475,16 @@ EOS
475
475
  def parse(line)
476
476
  start_ruby_prof
477
477
  bench_start_time = Time.now
478
- _parse(line)
479
- ensure
480
- stop_ruby_prof
481
- if ENV['FLYDATA_BENCHMARK']
482
- puts " -> time:#{Time.now.to_f - bench_start_time.to_f} size:#{target_line.size}"
478
+ begin
479
+ _parse(line)
480
+ rescue DumpParseError => e
481
+ $log.error "Encountered an error while parsing this chunk:\n #{e.message}. backtrace: #{e.backtrace}"
482
+ raise
483
+ ensure
484
+ stop_ruby_prof
485
+ if ENV['FLYDATA_BENCHMARK']
486
+ puts " -> time:#{Time.now.to_f - bench_start_time.to_f} size:#{target_line.size}"
487
+ end
483
488
  end
484
489
  end
485
490
 
@@ -10,6 +10,7 @@ class DataEntry < Source::DataEntry
10
10
  tables: {},
11
11
  tables_append_only: {},
12
12
  pk_override: {},
13
+ table_attributes: {},
13
14
  host: {},
14
15
  port: {},
15
16
  username: {},
@@ -50,7 +50,11 @@ SELECT
50
50
  FROM (SELECT length(substring('999999'::money::char varying, '\.([0-9]+)$')) AS scale) s)
51
51
  ELSE c.numeric_scale END AS numeric_scale,
52
52
  c.datetime_precision,
53
- i.indisprimary AS is_primary
53
+ i.indexrelid::regclass as index_name,
54
+ i.indisprimary as is_primary,
55
+ i.indisunique as is_unique,
56
+ i.indisvalid as is_valid,
57
+ CASE WHEN i.indpred!='NULL' THEN true ELSE false END AS is_partial
54
58
  FROM
55
59
  pg_index i
56
60
  JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
@@ -66,7 +70,7 @@ WHERE
66
70
  AND c.table_schema IN (%{schema})
67
71
  AND c.table_name IN (%{tables})
68
72
  ORDER BY
69
- c.table_name, c.ordinal_position, i.indisprimary asc;
73
+ c.table_name, c.ordinal_position, i.indisprimary desc, i.indisunique desc, i.indisvalid asc;
70
74
  EOT
71
75
 
72
76
  GET_CURRENT_SNAPSHOT_SQL = "SELECT txid_current_snapshot();"
@@ -119,8 +123,8 @@ EOT
119
123
  column_name = col['column_name'].to_sym
120
124
  table_name = col['table_name'].to_sym
121
125
  t_meta = ret[table_name]
122
- t_meta[:raw_columns] ||= {}
123
- t_meta[:raw_columns][column_name] = col
126
+ t_meta[:raw_columns] = Hash.new {|h,k| h[k] = []} unless t_meta[:raw_columns]
127
+ t_meta[:raw_columns][column_name] << col
124
128
  end
125
129
 
126
130
  ret.each do |table_name, t_meta|
@@ -514,7 +514,7 @@ module Flydata
514
514
  OLD_DUMP_DIR_PROPERTY = 'mysqldump_dir'
515
515
  DUMP_DIR_PROPERTY = 'dump_dir'
516
516
  def dump_dir
517
- pref = @data_entry['mysql_data_entry_preference']
517
+ pref = @data_entry['mysql_data_entry_preference'] || @data_entry['postgresql_data_entry_preference']
518
518
  dump_dir = nil
519
519
  dump_dir_property = DUMP_DIR_PROPERTY
520
520
 
@@ -0,0 +1,58 @@
1
+ module Flydata
2
+ module TableAttribute
3
+ USER_PK_OVERRIDE = 'pk_override'
4
+ UK_AS_PK_OVERRIDE = 'uk_as_pk'
5
+ INVALID_TABLE_REASON = 'invalid_table_reason'
6
+
7
+ def self.generate_pk_override(table_attributes)
8
+ user_pk_override = extract_attribute(table_attributes, USER_PK_OVERRIDE)
9
+ uk_as_pk_override = extract_attribute(table_attributes, UK_AS_PK_OVERRIDE)
10
+
11
+ uk_as_pk_override.merge(user_pk_override)
12
+ end
13
+
14
+ def self.delete_attribute(table_attributes, key)
15
+ table_attributes.each {|tbl_attr| tbl_attr.delete(key.to_s)}
16
+ table_attributes
17
+ end
18
+
19
+ # Extract {<table_name> => <value_for_a_given_key>} pairs from table attributes
20
+ # arguments:
21
+ # - table_attributes (hash):
22
+ # { "<table-name1>" => {
23
+ # "invalid_table_reason" => "<error reason>",
24
+ # "pk_override" => ['id1', ...],
25
+ # "uk_as_pk" => ['uid', ...],...
26
+ # },
27
+ # "<table-name2> => {...},
28
+ # }
29
+ #
30
+ # - key (symbol or String): hash key in table_attributes
31
+ #
32
+ # return value (hash):
33
+ # { "table_1" => <value_1>, "table_2" => <value_2>,...}
34
+ def self.extract_attribute(table_attributes, key)
35
+ tbl_value_hash = {}
36
+ table_attributes.each do |tbl_attr|
37
+
38
+ table_name = tbl_attr['table_name']
39
+ next unless tbl_attr.has_key?(key.to_s)
40
+ tbl_value_hash[table_name] = tbl_attr[key.to_s]
41
+ end
42
+ tbl_value_hash
43
+ end
44
+
45
+ def self.save_attribute(table_attributes, values_hash, key)
46
+ return unless values_hash
47
+ values_hash.each_pair do |tbl_name, val|
48
+ tbl_attr = table_attributes.find { |ta| ta['table_name'] == tbl_name }
49
+ unless tbl_attr
50
+ # Add a new table
51
+ tbl_attr = { 'table_name' => tbl_name }
52
+ table_attributes << tbl_attr
53
+ end
54
+ tbl_attr[key.to_s] = val
55
+ end
56
+ end
57
+ end
58
+ end
@@ -175,7 +175,13 @@ module Flydata
175
175
  subject { subject_object.send(:data_entry) }
176
176
 
177
177
  let(:de) { { 'mysql_data_entry_preference' => mp } }
178
- let(:mp) { { 'tables' => 'Users,Addresses' } }
178
+ let(:mp) { { 'tables' => 'Users,Addresses',
179
+ 'table_attributes' => [
180
+ {"table_name"=>"Users", "status"=>"init_sync_pending"},
181
+ {"table_name"=>"Addresses", "status"=>"init_sync_pending"}
182
+ ],
183
+ 'pk_override' => {"Users"=>["id"]}
184
+ } }
179
185
 
180
186
  let(:sfm) { double('sfm') }
181
187
  let(:ssl_ca_content) { double('ssl_ca_content') }
@@ -200,7 +206,14 @@ module Flydata
200
206
  end
201
207
  end
202
208
  context 'with tables_append_only' do
203
- before { mp['tables_append_only'] = 'Invoices,Sessions,Addresses' }
209
+ before do
210
+ mp['tables_append_only'] = 'Invoices,Sessions,Addresses'
211
+ mp['table_attributes'] += [
212
+ {"table_name"=>"Invoices", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
213
+ {"table_name"=>"Sessions", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
214
+ {"table_name"=>"Addresses", "omit_events"=>["delete"], "status"=>"init_sync_pending"}
215
+ ]
216
+ end
204
217
  it "creates an array of tables from 'tables' and 'tables_append_only' combined" do
205
218
  subject
206
219
  expect(mp['tables']).to eq %w(Users Addresses Invoices Sessions)
@@ -15,7 +15,9 @@ module Fluent
15
15
  tag #{TEST_TAG}
16
16
  database #{TEST_DB}
17
17
  tables test_table,test_table_1,test_table_2
18
- tables_append_only test_table_3
18
+ tables_append_only test_table_3,error_table_4
19
+ pk_override {"test_table_1":["id1","id2"],"test_table_3":["uid"]}
20
+ table_attributes [{"table_name":"test_table","status":"init_sync_completed"},{"table_name":"test_table_1","status":"init_sync_completed","pk_override":["id1","id2"]},{"table_name":"test_table_2","status":"init_sync_pending"},{"table_name":"test_table_3","omit_events":["delete"],"status":"init_sync_pending","uk_as_pk":["uid"]},{"table_name":"error_table_4","omit_events":["delete"],"status":"init_sync_pending","invalid_table_reason":"table does not exist in the MySQL database"}]
19
21
  position_file #{TEST_POSITION_FILE}
20
22
  host localhost
21
23
  port 3306
@@ -237,6 +239,13 @@ EOT
237
239
 
238
240
  describe '#configure' do
239
241
  let(:config) { default_config }
242
+ let(:table_attributes_array) {
243
+ [{"table_name"=>"test_table", "status"=>"init_sync_completed"},
244
+ {"table_name"=>"test_table_1", "status"=>"init_sync_completed", "pk_override"=>["id1", "id2"]},
245
+ {"table_name"=>"test_table_2", "status"=>"init_sync_pending"},
246
+ {"table_name"=>"test_table_3", "omit_events"=>["delete"], "status"=>"init_sync_pending", "uk_as_pk"=>["uid"]},
247
+ {"table_name"=>"error_table_4", "omit_events"=>["delete"], "status"=>"init_sync_pending", "invalid_table_reason"=>"table does not exist in the MySQL database"}]
248
+ }
240
249
  before do
241
250
  Test.configure_plugin(plugin, config)
242
251
  end
@@ -248,8 +257,9 @@ EOT
248
257
  'mysql_data_entry_preference' => {
249
258
  'database' => 'test_db',
250
259
  'tables' => 'test_table,test_table_1,test_table_2',
251
- 'tables_append_only' => 'test_table_3',
252
- 'pk_override' => {},
260
+ 'tables_append_only' => 'test_table_3,error_table_4',
261
+ 'pk_override' => {"test_table_1" => ["id1", "id2"], "test_table_3" => ["uid"]},
262
+ 'table_attributes' => table_attributes_array,
253
263
  'host' => 'localhost',
254
264
  'port' => 3306,
255
265
  'username' => 'test_admin',
@@ -265,7 +275,7 @@ EOT
265
275
  end
266
276
 
267
277
  it 'sets data tables_append_only as array' do
268
- expect(plugin.instance_variable_get(:@tables_append_only)).to eq(%w(test_table_3))
278
+ expect(plugin.instance_variable_get(:@tables_append_only)).to eq(%w(test_table_3 error_table_4))
269
279
  end
270
280
 
271
281
  it 'sets sync_fm' do
@@ -273,7 +283,15 @@ EOT
273
283
  end
274
284
 
275
285
  it 'sets omit_events' do
276
- expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table]})
286
+ expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table],'error_table_4' => [:delete, :truncate_table]})
287
+ end
288
+
289
+ it 'sets pk_override' do
290
+ expect(plugin.instance_variable_get(:@pk_override)).to eq({"test_table_1" => ["id1", "id2"], "test_table_3" => ["uid"]})
291
+ end
292
+
293
+ it 'set table_attributes' do
294
+ expect(plugin.instance_variable_get(:@table_attributes)).to eq(table_attributes_array)
277
295
  end
278
296
  end
279
297
 
@@ -14,7 +14,9 @@ module Fluent
14
14
  tag #{TEST_TAG}
15
15
  database #{TEST_DB}
16
16
  tables test_table,test_table_1,test_table_2
17
- tables_append_only test_table_3
17
+ tables_append_only test_table_3,error_table_4
18
+ pk_override {"test_table_1":["id1","id2"],"test_table_3":["uid"]}
19
+ table_attributes [{"table_name":"test_table","status":"init_sync_completed"},{"table_name":"test_table_1","status":"init_sync_completed","pk_override":["id1","id2"]},{"table_name":"test_table_2","status":"init_sync_pending"},{"table_name":"test_table_3","omit_events":["delete"],"status":"init_sync_pending","uk_as_pk":["uid"]},{"table_name":"error_table_4","omit_events":["delete"],"status":"init_sync_pending","invalid_table_reason":"table does not exist in the MySQL database"}]
18
20
  position_file #{TEST_POSITION_FILE}
19
21
  host localhost
20
22
  port 5433
@@ -32,6 +34,13 @@ EOT
32
34
  let(:config) { default_config }
33
35
  subject { Test.configure_plugin(plugin, config) }
34
36
 
37
+ let(:table_attributes_array) {
38
+ [{"table_name"=>"test_table", "status"=>"init_sync_completed"},
39
+ {"table_name"=>"test_table_1", "status"=>"init_sync_completed", "pk_override"=>["id1", "id2"]},
40
+ {"table_name"=>"test_table_2", "status"=>"init_sync_pending"},
41
+ {"table_name"=>"test_table_3", "omit_events"=>["delete"], "status"=>"init_sync_pending", "uk_as_pk"=>["uid"]},
42
+ {"table_name"=>"error_table_4", "omit_events"=>["delete"], "status"=>"init_sync_pending", "invalid_table_reason"=>"table does not exist in the MySQL database"}]
43
+ }
35
44
  before do
36
45
  create_file(TEST_POSITION_FILE, "1001:1001:\t\t")
37
46
  create_table_pos_file('test_table', '0')
@@ -48,8 +57,9 @@ EOT
48
57
  'postgresql_data_entry_preference' => {
49
58
  'database' => 'test_db',
50
59
  'tables' => 'test_table,test_table_1,test_table_2',
51
- 'tables_append_only' => 'test_table_3',
52
- 'pk_override' => {},
60
+ 'tables_append_only' => 'test_table_3,error_table_4',
61
+ 'pk_override' => {"test_table_1" => ["id1", "id2"], "test_table_3" => ["uid"]},
62
+ 'table_attributes' => table_attributes_array,
53
63
  'host' => 'localhost',
54
64
  'port' => 5433,
55
65
  'username' => 'test_admin',
@@ -68,9 +78,17 @@ EOT
68
78
  end
69
79
 
70
80
  it 'sets omit_events' do
71
- expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table]})
81
+ expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table],'error_table_4' => [:delete, :truncate_table]})
72
82
  end
73
83
 
84
+ it 'sets pk_override' do
85
+ expect(plugin.instance_variable_get(:@pk_override)).to eq({"test_table_1" => ["id1", "id2"], "test_table_3" => ["uid"]})
86
+ end
87
+
88
+ it 'set table_attributes' do
89
+ expect(plugin.instance_variable_get(:@table_attributes)).to eq(table_attributes_array)
90
+ end
91
+
74
92
  it 'sets context' do
75
93
  expect(plugin.instance_variable_get(:@context)).to be_kind_of(Flydata::SourcePostgresql::PluginSupport::Context)
76
94
  end
@@ -36,6 +36,11 @@ describe SyncGenerateTableDdl do
36
36
  end
37
37
  describe '#generate_flydata_tabledef' do
38
38
  subject { subject_object.generate_flydata_tabledef(tables, options) }
39
+ before do
40
+ de['mysql_data_entry_preference']['user_pk_override'] = {}
41
+ de['mysql_data_entry_preference']['uk_as_pk_override'] = {}
42
+ de['mysql_data_entry_preference']['pk_override'] = {}
43
+ end
39
44
 
40
45
  let(:tables) { %w| table1 table2 table4 table3 | }
41
46
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.5
4
+ version: 0.7.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2016-06-02 00:00:00.000000000 Z
15
+ date: 2016-06-15 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -758,6 +758,7 @@ files:
758
758
  - lib/flydata/source_zendesk/sync_generate_table_ddl.rb
759
759
  - lib/flydata/source_zendesk/zendesk_flydata_tabledefs.rb
760
760
  - lib/flydata/sync_file_manager.rb
761
+ - lib/flydata/table_attribute.rb
761
762
  - lib/flydata/table_ddl.rb
762
763
  - lib/flydata/table_meta.rb
763
764
  - lib/flydata/util/encryptor.rb
@@ -869,7 +870,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
869
870
  version: '0'
870
871
  requirements: []
871
872
  rubyforge_project:
872
- rubygems_version: 2.0.14.1
873
+ rubygems_version: 2.2.2
873
874
  signing_key:
874
875
  specification_version: 4
875
876
  summary: FlyData Agent