flydata 0.6.14 → 0.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/flydata-core/Gemfile +1 -0
  4. data/flydata-core/Gemfile.lock +5 -0
  5. data/flydata-core/lib/flydata-core/errors.rb +4 -2
  6. data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb +4 -0
  7. data/flydata-core/lib/flydata-core/postgresql/compatibility_checker.rb +119 -0
  8. data/flydata-core/lib/flydata-core/postgresql/config.rb +58 -0
  9. data/flydata-core/lib/flydata-core/postgresql/pg_client.rb +170 -0
  10. data/flydata-core/lib/flydata-core/postgresql/snapshot.rb +49 -0
  11. data/flydata-core/lib/flydata-core/postgresql/source_pos.rb +71 -10
  12. data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +1 -1
  13. data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb +76 -17
  14. data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +59 -10
  15. data/flydata-core/spec/mysql/binlog_pos_spec.rb +10 -2
  16. data/flydata-core/spec/postgresql/compatibility_checker_spec.rb +148 -0
  17. data/flydata-core/spec/postgresql/config_spec.rb +85 -0
  18. data/flydata-core/spec/postgresql/pg_client_spec.rb +195 -0
  19. data/flydata-core/spec/postgresql/snapshot_spec.rb +55 -0
  20. data/flydata-core/spec/postgresql/source_pos_spec.rb +70 -8
  21. data/flydata-core/spec/table_def/postgresql_table_def_spec.rb +80 -19
  22. data/flydata-core/spec/table_def/redshift_table_def_spec.rb +211 -14
  23. data/flydata.gemspec +0 -0
  24. data/lib/flydata.rb +1 -0
  25. data/lib/flydata/command/sender.rb +10 -7
  26. data/lib/flydata/command/sync.rb +4 -1
  27. data/lib/flydata/fluent-plugins/flydata_plugin_ext/base.rb +1 -0
  28. data/lib/flydata/fluent-plugins/flydata_plugin_ext/fluent_log_ext.rb +73 -0
  29. data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb +35 -10
  30. data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based.rb +29 -0
  31. data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based.rb +26 -0
  32. data/lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb +29 -13
  33. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +10 -18
  34. data/lib/flydata/fluent-plugins/in_postgresql_query_based_flydata.rb +64 -0
  35. data/lib/flydata/helpers.rb +1 -3
  36. data/lib/flydata/plugin_support/context.rb +14 -2
  37. data/lib/flydata/plugin_support/source_position_file.rb +35 -0
  38. data/lib/flydata/plugin_support/sync_record_emittable.rb +2 -1
  39. data/lib/flydata/query_based_sync/client.rb +101 -0
  40. data/lib/flydata/query_based_sync/record_size_estimator.rb +39 -0
  41. data/lib/flydata/query_based_sync/resource_requester.rb +70 -0
  42. data/lib/flydata/query_based_sync/response.rb +122 -0
  43. data/lib/flydata/query_based_sync/response_handler.rb +30 -0
  44. data/lib/flydata/source/sync_generate_table_ddl.rb +1 -1
  45. data/lib/flydata/source_mysql/plugin_support/binlog_record_dispatcher.rb +2 -2
  46. data/lib/flydata/source_mysql/plugin_support/binlog_record_handler.rb +3 -9
  47. data/lib/flydata/source_mysql/plugin_support/context.rb +26 -2
  48. data/lib/flydata/source_mysql/plugin_support/source_position_file.rb +14 -0
  49. data/lib/flydata/source_mysql/table_ddl.rb +3 -3
  50. data/lib/flydata/source_mysql/{plugin_support/table_meta.rb → table_meta.rb} +3 -10
  51. data/lib/flydata/source_postgresql/generate_source_dump.rb +44 -63
  52. data/lib/flydata/source_postgresql/parse_dump_and_send.rb +2 -0
  53. data/lib/flydata/source_postgresql/plugin_support/context.rb +13 -0
  54. data/lib/flydata/source_postgresql/plugin_support/source_position_file.rb +14 -0
  55. data/lib/flydata/source_postgresql/query_based_sync/client.rb +16 -0
  56. data/lib/flydata/source_postgresql/query_based_sync/diff_query_generator.rb +135 -0
  57. data/lib/flydata/source_postgresql/query_based_sync/resource_requester.rb +86 -0
  58. data/lib/flydata/source_postgresql/query_based_sync/response.rb +12 -0
  59. data/lib/flydata/source_postgresql/query_based_sync/response_handler.rb +12 -0
  60. data/lib/flydata/source_postgresql/sync_generate_table_ddl.rb +25 -79
  61. data/lib/flydata/source_postgresql/table_meta.rb +168 -0
  62. data/lib/flydata/sync_file_manager.rb +5 -5
  63. data/lib/flydata/table_meta.rb +19 -0
  64. data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_context.rb +85 -0
  65. data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based_shared_examples.rb +36 -0
  66. data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based_shared_examples.rb +37 -0
  67. data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_shared_examples.rb +67 -0
  68. data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +119 -96
  69. data/spec/flydata/fluent-plugins/in_postgresql_query_based_flydata_spec.rb +82 -0
  70. data/spec/flydata/fluent-plugins/sync_source_plugin_context.rb +29 -0
  71. data/spec/flydata/plugin_support/context_spec.rb +37 -3
  72. data/spec/flydata/query_based_sync/client_spec.rb +79 -0
  73. data/spec/flydata/query_based_sync/query_based_sync_context.rb +116 -0
  74. data/spec/flydata/query_based_sync/record_size_estimator_spec.rb +54 -0
  75. data/spec/flydata/query_based_sync/resource_requester_spec.rb +58 -0
  76. data/spec/flydata/query_based_sync/response_handler_spec.rb +36 -0
  77. data/spec/flydata/query_based_sync/response_spec.rb +157 -0
  78. data/spec/flydata/source_mysql/plugin_support/context_spec.rb +7 -1
  79. data/spec/flydata/source_mysql/plugin_support/dml_record_handler_spec.rb +2 -15
  80. data/spec/flydata/source_mysql/plugin_support/drop_database_query_handler_spec.rb +1 -1
  81. data/spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb +12 -11
  82. data/spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb +53 -0
  83. data/spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb +1 -1
  84. data/spec/flydata/source_mysql/table_ddl_spec.rb +5 -5
  85. data/spec/flydata/source_mysql/{plugin_support/table_meta_spec.rb → table_meta_spec.rb} +6 -7
  86. data/spec/flydata/source_postgresql/generate_source_dump_spec.rb +165 -77
  87. data/spec/flydata/source_postgresql/query_based_sync/diff_query_generator_spec.rb +213 -0
  88. data/spec/flydata/source_postgresql/query_based_sync/query_based_sync_postgresql_context.rb +76 -0
  89. data/spec/flydata/source_postgresql/query_based_sync/resource_requester_spec.rb +70 -0
  90. data/spec/flydata/source_postgresql/table_meta_spec.rb +77 -0
  91. metadata +49 -6
  92. data/lib/flydata/source_mysql/plugin_support/binlog_position_file.rb +0 -23
  93. data/lib/flydata/source_postgresql/pg_client.rb +0 -43
@@ -1,23 +0,0 @@
1
- module Flydata
2
- module SourceMysql
3
-
4
- module PluginSupport
5
- class BinLogPositionFile
6
- attr_accessor :path
7
-
8
- def initialize(path)
9
- @path = path
10
- end
11
-
12
- def exists?
13
- File.exists?(@path)
14
- end
15
-
16
- def read
17
- File.open(@path) {|f| f.read }
18
- end
19
- end
20
- end
21
-
22
- end
23
- end
@@ -1,43 +0,0 @@
1
- require 'pg'
2
-
3
- module Flydata
4
- module SourcePostgresql
5
-
6
- class PGClient
7
- def initialize(de_prefs)
8
- @de_prefs = de_prefs
9
- @cli = PG::Connection.new(pg_opts_from_de_prefs(de_prefs))
10
- end
11
-
12
- def query(query, params = [], opts = {})
13
- if opts.has_key?(:placeholder_start_num)
14
- placeholders = placeholder_string(opts[:placeholder_size],
15
- opts[:placeholder_start_num])
16
- q = query % [placeholders]
17
- else
18
- q = query
19
- end
20
-
21
- @cli.query(q, params)
22
- end
23
-
24
- private
25
-
26
- def pg_opts_from_de_prefs(de_prefs)
27
- {
28
- host: de_prefs['host'],
29
- port: de_prefs['port'],
30
- dbname: de_prefs['database'],
31
- user: de_prefs['username'],
32
- password: de_prefs['password'],
33
- sslmode: :prefer,
34
- }
35
- end
36
-
37
- def placeholder_string(num_items, start_num)
38
- num_items.times.collect{|i| "$#{i + start_num}"}.join(",")
39
- end
40
- end
41
-
42
- end
43
- end