flydata 0.6.14 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/Gemfile +1 -0
- data/flydata-core/Gemfile.lock +5 -0
- data/flydata-core/lib/flydata-core/errors.rb +4 -2
- data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb +4 -0
- data/flydata-core/lib/flydata-core/postgresql/compatibility_checker.rb +119 -0
- data/flydata-core/lib/flydata-core/postgresql/config.rb +58 -0
- data/flydata-core/lib/flydata-core/postgresql/pg_client.rb +170 -0
- data/flydata-core/lib/flydata-core/postgresql/snapshot.rb +49 -0
- data/flydata-core/lib/flydata-core/postgresql/source_pos.rb +71 -10
- data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +1 -1
- data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb +76 -17
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +59 -10
- data/flydata-core/spec/mysql/binlog_pos_spec.rb +10 -2
- data/flydata-core/spec/postgresql/compatibility_checker_spec.rb +148 -0
- data/flydata-core/spec/postgresql/config_spec.rb +85 -0
- data/flydata-core/spec/postgresql/pg_client_spec.rb +195 -0
- data/flydata-core/spec/postgresql/snapshot_spec.rb +55 -0
- data/flydata-core/spec/postgresql/source_pos_spec.rb +70 -8
- data/flydata-core/spec/table_def/postgresql_table_def_spec.rb +80 -19
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +211 -14
- data/flydata.gemspec +0 -0
- data/lib/flydata.rb +1 -0
- data/lib/flydata/command/sender.rb +10 -7
- data/lib/flydata/command/sync.rb +4 -1
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/base.rb +1 -0
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/fluent_log_ext.rb +73 -0
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb +35 -10
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based.rb +29 -0
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based.rb +26 -0
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb +29 -13
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +10 -18
- data/lib/flydata/fluent-plugins/in_postgresql_query_based_flydata.rb +64 -0
- data/lib/flydata/helpers.rb +1 -3
- data/lib/flydata/plugin_support/context.rb +14 -2
- data/lib/flydata/plugin_support/source_position_file.rb +35 -0
- data/lib/flydata/plugin_support/sync_record_emittable.rb +2 -1
- data/lib/flydata/query_based_sync/client.rb +101 -0
- data/lib/flydata/query_based_sync/record_size_estimator.rb +39 -0
- data/lib/flydata/query_based_sync/resource_requester.rb +70 -0
- data/lib/flydata/query_based_sync/response.rb +122 -0
- data/lib/flydata/query_based_sync/response_handler.rb +30 -0
- data/lib/flydata/source/sync_generate_table_ddl.rb +1 -1
- data/lib/flydata/source_mysql/plugin_support/binlog_record_dispatcher.rb +2 -2
- data/lib/flydata/source_mysql/plugin_support/binlog_record_handler.rb +3 -9
- data/lib/flydata/source_mysql/plugin_support/context.rb +26 -2
- data/lib/flydata/source_mysql/plugin_support/source_position_file.rb +14 -0
- data/lib/flydata/source_mysql/table_ddl.rb +3 -3
- data/lib/flydata/source_mysql/{plugin_support/table_meta.rb → table_meta.rb} +3 -10
- data/lib/flydata/source_postgresql/generate_source_dump.rb +44 -63
- data/lib/flydata/source_postgresql/parse_dump_and_send.rb +2 -0
- data/lib/flydata/source_postgresql/plugin_support/context.rb +13 -0
- data/lib/flydata/source_postgresql/plugin_support/source_position_file.rb +14 -0
- data/lib/flydata/source_postgresql/query_based_sync/client.rb +16 -0
- data/lib/flydata/source_postgresql/query_based_sync/diff_query_generator.rb +135 -0
- data/lib/flydata/source_postgresql/query_based_sync/resource_requester.rb +86 -0
- data/lib/flydata/source_postgresql/query_based_sync/response.rb +12 -0
- data/lib/flydata/source_postgresql/query_based_sync/response_handler.rb +12 -0
- data/lib/flydata/source_postgresql/sync_generate_table_ddl.rb +25 -79
- data/lib/flydata/source_postgresql/table_meta.rb +168 -0
- data/lib/flydata/sync_file_manager.rb +5 -5
- data/lib/flydata/table_meta.rb +19 -0
- data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_context.rb +85 -0
- data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based_shared_examples.rb +36 -0
- data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based_shared_examples.rb +37 -0
- data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_shared_examples.rb +67 -0
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +119 -96
- data/spec/flydata/fluent-plugins/in_postgresql_query_based_flydata_spec.rb +82 -0
- data/spec/flydata/fluent-plugins/sync_source_plugin_context.rb +29 -0
- data/spec/flydata/plugin_support/context_spec.rb +37 -3
- data/spec/flydata/query_based_sync/client_spec.rb +79 -0
- data/spec/flydata/query_based_sync/query_based_sync_context.rb +116 -0
- data/spec/flydata/query_based_sync/record_size_estimator_spec.rb +54 -0
- data/spec/flydata/query_based_sync/resource_requester_spec.rb +58 -0
- data/spec/flydata/query_based_sync/response_handler_spec.rb +36 -0
- data/spec/flydata/query_based_sync/response_spec.rb +157 -0
- data/spec/flydata/source_mysql/plugin_support/context_spec.rb +7 -1
- data/spec/flydata/source_mysql/plugin_support/dml_record_handler_spec.rb +2 -15
- data/spec/flydata/source_mysql/plugin_support/drop_database_query_handler_spec.rb +1 -1
- data/spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb +12 -11
- data/spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb +53 -0
- data/spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb +1 -1
- data/spec/flydata/source_mysql/table_ddl_spec.rb +5 -5
- data/spec/flydata/source_mysql/{plugin_support/table_meta_spec.rb → table_meta_spec.rb} +6 -7
- data/spec/flydata/source_postgresql/generate_source_dump_spec.rb +165 -77
- data/spec/flydata/source_postgresql/query_based_sync/diff_query_generator_spec.rb +213 -0
- data/spec/flydata/source_postgresql/query_based_sync/query_based_sync_postgresql_context.rb +76 -0
- data/spec/flydata/source_postgresql/query_based_sync/resource_requester_spec.rb +70 -0
- data/spec/flydata/source_postgresql/table_meta_spec.rb +77 -0
- metadata +49 -6
- data/lib/flydata/source_mysql/plugin_support/binlog_position_file.rb +0 -23
- data/lib/flydata/source_postgresql/pg_client.rb +0 -43
@@ -1,23 +0,0 @@
|
|
1
|
-
module Flydata
|
2
|
-
module SourceMysql
|
3
|
-
|
4
|
-
module PluginSupport
|
5
|
-
class BinLogPositionFile
|
6
|
-
attr_accessor :path
|
7
|
-
|
8
|
-
def initialize(path)
|
9
|
-
@path = path
|
10
|
-
end
|
11
|
-
|
12
|
-
def exists?
|
13
|
-
File.exists?(@path)
|
14
|
-
end
|
15
|
-
|
16
|
-
def read
|
17
|
-
File.open(@path) {|f| f.read }
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
end
|
23
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'pg'
|
2
|
-
|
3
|
-
module Flydata
|
4
|
-
module SourcePostgresql
|
5
|
-
|
6
|
-
class PGClient
|
7
|
-
def initialize(de_prefs)
|
8
|
-
@de_prefs = de_prefs
|
9
|
-
@cli = PG::Connection.new(pg_opts_from_de_prefs(de_prefs))
|
10
|
-
end
|
11
|
-
|
12
|
-
def query(query, params = [], opts = {})
|
13
|
-
if opts.has_key?(:placeholder_start_num)
|
14
|
-
placeholders = placeholder_string(opts[:placeholder_size],
|
15
|
-
opts[:placeholder_start_num])
|
16
|
-
q = query % [placeholders]
|
17
|
-
else
|
18
|
-
q = query
|
19
|
-
end
|
20
|
-
|
21
|
-
@cli.query(q, params)
|
22
|
-
end
|
23
|
-
|
24
|
-
private
|
25
|
-
|
26
|
-
def pg_opts_from_de_prefs(de_prefs)
|
27
|
-
{
|
28
|
-
host: de_prefs['host'],
|
29
|
-
port: de_prefs['port'],
|
30
|
-
dbname: de_prefs['database'],
|
31
|
-
user: de_prefs['username'],
|
32
|
-
password: de_prefs['password'],
|
33
|
-
sslmode: :prefer,
|
34
|
-
}
|
35
|
-
end
|
36
|
-
|
37
|
-
def placeholder_string(num_items, start_num)
|
38
|
-
num_items.times.collect{|i| "$#{i + start_num}"}.join(",")
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
end
|
43
|
-
end
|