flydata 0.3.0 → 0.3.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 68f311d20207752e80dd5fb47b905b085e840520
4
- data.tar.gz: 6bfa38fa63a7db7e1f4b7109324017e99c82c117
3
+ metadata.gz: 9b34f397692efd9d9bc9575588d19263b9c0df2c
4
+ data.tar.gz: 21b42d4c19267a6656eb8f8d8c21607cf54501f1
5
5
  SHA512:
6
- metadata.gz: 01469b9c9b268bc8dad0bda81a3dbcc3a98a55901108cac17b50a6688313485f1d7cdd8a01b26ea20cbea0d8fa6f4c096055394f55cea58ee59a949acb2963fe
7
- data.tar.gz: cde53114405b00f034e7444fd0b9ba9df9519a2cd3979eade6a99d48e80e326615a1ba16fb0d5c014c51d23272ca5cf2257b2f318135ad6efef858f4d6d4479b
6
+ metadata.gz: 147ed1f61574abacfc850460e4745953c37bba5827985bd82adc51c9ef6750cb6188f3e56f52a3e855f73096c9fb6c05e97039ed68ac0982029a259f8a9a8c78
7
+ data.tar.gz: a73489d7abcfb93db8720819143ad751d78493389d911407ede925a8f2417ba9161d7a742365b97de56595ca168139832ef179f37a025ce47e015da97b03829d
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.3.0
1
+ 0.3.1
@@ -0,0 +1,115 @@
1
+ module FlydataCore
2
+ module Config
3
+
4
+ # This class manages user maintenance mode
5
+ # We have two user maintenance mode as following and controll user's mode by putting a file.
6
+ #
7
+ # ## User Mode
8
+ #
9
+ # - maintenance mode <stop actions against user's resource like Redshift>
10
+ # - fluent-output-plugin#format: Append data to buffer file
11
+ # - fluent-output-plugin#write: Stop processing chunk file and throw an error (repeat retry)
12
+ # - copy-handler: Stop processing.
13
+ #
14
+ #- discard mode
15
+ # - fluent-output-plugin#format: Discard data
16
+ # - fluent-output-plugin#format: Return true without processing chunk files
17
+ # - copy-handler: Stop processing.
18
+ #
19
+ #
20
+ # ## Directory Structure
21
+ #
22
+ # /mnt/flydata/system/user_maintenance
23
+ # + [data-port-id]
24
+ # - maintenance_[data-port-id]_[data-entry-id]_[(schema-name.)table-name]
25
+ # - discard_[data-port-id]_[data-entry-id]_[(schema-name.)table-name]
26
+ #
27
+ class UserMaintenance
28
+ def initialize(base_dir_path)
29
+ @base_dir_path = base_dir_path
30
+ end
31
+
32
+ attr_reader :base_dir_path
33
+
34
+ # Argument "params" of methods
35
+ #
36
+ # params: Hash
37
+ # data_port_id
38
+ # data_entry_id
39
+ # table_name
40
+
41
+ # For flydata-web
42
+
43
+ def maintenance_mode_file_path(params)
44
+ mode_file_path(:maintenance, params)
45
+ end
46
+
47
+ def discard_mode_file_path(params)
48
+ mode_file_path(:discard, params)
49
+ end
50
+
51
+ # For fluentd plugin and copy handler
52
+
53
+ def maintenance_mode?(params)
54
+ check_mode(:maintenance, params)
55
+ end
56
+
57
+ def discard_mode?(params)
58
+ check_mode(:discard, params)
59
+ end
60
+
61
+ def current_user_mode(params)
62
+ if discard_mode?(params)
63
+ :discard
64
+ elsif maintenance_mode?(params)
65
+ :maintenance
66
+ else
67
+ nil
68
+ end
69
+ end
70
+
71
+ private
72
+
73
+ def validate_params(params)
74
+ raise ArgumentError.new("params must be hash") unless params.kind_of?(Hash)
75
+ raise ArgumentError.new("params must have 'data_port_id' key") unless params[:data_port_id]
76
+ end
77
+
78
+ def check_mode(mode, params)
79
+ validate_params(params)
80
+ return false unless params[:data_port_id]
81
+ return true if File.exists?(mode_file_path(mode, data_port_id: params[:data_port_id]))
82
+ return false unless params[:data_entry_id]
83
+ de_mode_file_path = mode_file_path(mode, data_port_id: params[:data_port_id], data_entry_id: params[:data_entry_id])
84
+ return true if File.exists?(de_mode_file_path)
85
+ return false unless params[:table_name]
86
+
87
+ # Ignore case of table_name
88
+ table_name = params[:table_name].downcase
89
+ tn_mode_file_path = mode_file_path(mode, data_port_id: params[:data_port_id], data_entry_id: params[:data_entry_id], table_name: table_name)
90
+ Dir.glob("#{de_mode_file_path}_*").any?{|fp| File.join(File.dirname(fp), File.basename(fp).downcase) == tn_mode_file_path}
91
+ end
92
+
93
+ # /mnt/flydata/system/user_maintenance/[dp-id]/[mode]_[data-port-id]_[data-entry-id]_[table-name]
94
+ def mode_file_path(mode, params)
95
+ validate_params(params)
96
+ File.join(dp_mode_dir_path(mode, params), mode_file_name(mode, params))
97
+ end
98
+
99
+ # /mnt/flydata/system/user_maintenance/[dp-id]
100
+ def dp_mode_dir_path(mode, params)
101
+ File.join(@base_dir_path, params[:data_port_id].to_s)
102
+ end
103
+
104
+ # [mode]_[data-port-id]_[data-entry-id]_[table-name]
105
+ def mode_file_name(mode, params)
106
+ return nil unless params[:data_port_id]
107
+ path = "#{mode}_#{params[:data_port_id]}"
108
+ return path unless params[:data_entry_id]
109
+ path << "_#{params[:data_entry_id]}"
110
+ return path unless params[:table_name]
111
+ path << "_#{params[:table_name]}"
112
+ end
113
+ end
114
+ end
115
+ end
@@ -89,7 +89,7 @@ EOS
89
89
  DROP TABLE %s;
90
90
  CREATE TABLE %s (
91
91
  %s
92
- );
92
+ )%s;
93
93
  EOS
94
94
 
95
95
  def self.create_table_sql(flydata_tabledef, schema_name)
@@ -99,9 +99,11 @@ EOS
99
99
 
100
100
  contents = lines.join(",\n")
101
101
 
102
+ dk_sk_def = distkey_sortkey_sql(flydata_tabledef)
103
+
102
104
  table_name = flydata_tabledef[:table_name]
103
105
  redshift_tbl = table_name_for_ddl(table_name, schema_name)
104
- CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents]
106
+ CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents, dk_sk_def]
105
107
  end
106
108
 
107
109
  def self.column_def_sql(column, opt = {})
@@ -173,10 +175,20 @@ EOS
173
175
  end
174
176
 
175
177
  def self.primary_key_sql(flydata_tabledef)
176
- pks = flydata_tabledef[:columns].select{|col| col[:primary_key]}.collect{|col| col[:column]}
178
+ pks = primary_keys(flydata_tabledef)
177
179
  pks.empty? ? nil : " PRIMARY KEY (#{pks.join(',')})"
178
180
  end
179
181
 
182
+ def self.primary_keys(flydata_tabledef)
183
+ flydata_tabledef[:columns].select{|col| col[:primary_key]}.collect{|col| col[:column]}
184
+ end
185
+
186
+ def self.distkey_sortkey_sql(flydata_tabledef)
187
+ pks = primary_keys(flydata_tabledef)
188
+ return nil if pks.empty?
189
+ " DISTKEY(#{pks.first}) SORTKEY(#{pks.join(',')})"
190
+ end
191
+
180
192
  def self.comment_sql(flydata_tabledef, schema_name)
181
193
  sql = ""
182
194
  flydata_tabledef[:columns].each do |col|
@@ -0,0 +1,78 @@
1
+ require_relative '../spec_helper'
2
+
3
+ require 'tempfile'
4
+ require 'flydata-core/config/user_maintenance'
5
+
6
+ module FlydataCore::Config
7
+ describe UserMaintenance do
8
+ let(:temp_dir) { Dir.mktmpdir('flydatads_rspec_home') }
9
+ let(:user_maintenance_dir_path) { File.join(temp_dir, 'user_maintenance') }
10
+ let(:default_user_maintenance) { described_class.new(user_maintenance_dir_path) }
11
+ let(:user_maintenance) { default_user_maintenance }
12
+ let(:default_params) { { data_port_id: 1, data_entry_id: 2, table_name: 'test_table' } }
13
+ let(:params) { default_params }
14
+
15
+ describe '#maintenance_mode_file_path' do
16
+ subject { user_maintenance.maintenance_mode_file_path(params) }
17
+
18
+ context 'with data_port_id' do
19
+ let(:params) { { data_port_id: 1 } }
20
+ it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'maintenance_1')) }
21
+ end
22
+
23
+ context 'with data_port_id and data_entry_id' do
24
+ let(:params) { { data_port_id: 1, data_entry_id: 2 } }
25
+ it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'maintenance_1_2')) }
26
+ end
27
+
28
+ context 'with data_port_id, data_entry_id and table_name' do
29
+ it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'maintenance_1_2_test_table')) }
30
+ end
31
+
32
+ context 'with invalid object' do
33
+ let(:params) { nil }
34
+ it { expect{subject}.to raise_error(ArgumentError) }
35
+ end
36
+
37
+ context 'with empty hash' do
38
+ let(:params) { {} }
39
+ it { expect{subject}.to raise_error(ArgumentError) }
40
+ end
41
+ end
42
+
43
+ describe '#discard_mode_file_path' do
44
+ subject { user_maintenance.discard_mode_file_path(params) }
45
+
46
+ context 'with data_port_id' do
47
+ let(:params) { { data_port_id: 1 } }
48
+ it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'discard_1')) }
49
+ end
50
+
51
+ context 'with data_port_id and data_entry_id' do
52
+ let(:params) { { data_port_id: 1, data_entry_id: 2 } }
53
+ it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'discard_1_2')) }
54
+ end
55
+
56
+ context 'with data_port_id, data_entry_id and table_name' do
57
+ it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'discard_1_2_test_table')) }
58
+ end
59
+
60
+ context 'with invalid object' do
61
+ let(:params) { nil }
62
+ it { expect{subject}.to raise_error(ArgumentError) }
63
+ end
64
+
65
+ context 'with empty hash' do
66
+ let(:params) { {} }
67
+ it { expect{subject}.to raise_error(ArgumentError) }
68
+ end
69
+ end
70
+
71
+ describe '#maintenance_mode?' do
72
+ subject { user_maintenance.maintenance_mode?(params) }
73
+
74
+ context 'when in maintenance mode per data-port' do
75
+ end
76
+ end
77
+ end
78
+ end
@@ -66,7 +66,7 @@ CREATE TABLE "test_table_all" (
66
66
  "col_varbinary" varchar(512) DEFAULT NULL,
67
67
  "col_varchar" varchar(372) DEFAULT NULL,
68
68
  PRIMARY KEY (id)
69
- );
69
+ ) DISTKEY(id) SORTKEY(id);
70
70
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_all';
71
71
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
72
72
  ('test_table_all', 'id', 'int8(20)', 1),
@@ -120,7 +120,7 @@ CREATE TABLE "bit_test_def_1" (
120
120
  "bit_value" bigint DEFAULT 1,
121
121
  "int_value" int4 DEFAULT 16,
122
122
  PRIMARY KEY (id)
123
- );
123
+ ) DISTKEY(id) SORTKEY(id);
124
124
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'bit_test_def_1';
125
125
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
126
126
  ('bit_test_def_1', 'id', 'int4(11)', 1),
@@ -151,7 +151,7 @@ CREATE TABLE "product_order" (
151
151
  "product_id" int4 NOT NULL,
152
152
  "customer_id" int4 NOT NULL,
153
153
  PRIMARY KEY (no)
154
- );
154
+ ) DISTKEY(no) SORTKEY(no);
155
155
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'product_order';
156
156
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
157
157
  ('product_order', 'no', 'int4(11)', 1),
@@ -181,7 +181,7 @@ CREATE TABLE "test_table_column_comment" (
181
181
  "id" int4 NOT NULL DEFAULT '0',
182
182
  "value" varchar(max),
183
183
  PRIMARY KEY (id)
184
- );
184
+ ) DISTKEY(id) SORTKEY(id);
185
185
  COMMENT ON COLUMN "test_table_column_comment"."id"
186
186
  IS 'this is primary key';
187
187
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_column_comment';
@@ -213,7 +213,7 @@ CREATE TABLE "test_table_enum" (
213
213
  "enum_2" varchar encode bytedict DEFAULT 'a',
214
214
  "enum_3" varchar encode bytedict NOT NULL,
215
215
  PRIMARY KEY (id)
216
- );
216
+ ) DISTKEY(id) SORTKEY(id);
217
217
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_enum';
218
218
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
219
219
  ('test_table_enum', 'id', 'int4(11)', 1),
@@ -244,7 +244,7 @@ CREATE TABLE "test_table_multi_pk" (
244
244
  "id2" int4 NOT NULL DEFAULT '0',
245
245
  "value" varchar(max),
246
246
  PRIMARY KEY (id1,id2)
247
- );
247
+ ) DISTKEY(id1) SORTKEY(id1,id2);
248
248
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_multi_pk';
249
249
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
250
250
  ('test_table_multi_pk', 'id1', 'int4(11)', 1),
@@ -283,7 +283,7 @@ CREATE TABLE "sample1" (
283
283
  "name" varchar(max),
284
284
  "num" int4 DEFAULT NULL,
285
285
  PRIMARY KEY (id)
286
- );
286
+ ) DISTKEY(id) SORTKEY(id);
287
287
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'sample1';
288
288
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
289
289
  ('sample1', 'id', 'int4(11)', 1),
@@ -315,7 +315,7 @@ CREATE TABLE "sample1" (
315
315
  "name" varchar(max),
316
316
  "num" int4 DEFAULT NULL,
317
317
  PRIMARY KEY (id)
318
- );
318
+ ) DISTKEY(id) SORTKEY(id);
319
319
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'sample1';
320
320
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
321
321
  ('sample1', 'id', 'int4(11)', 1),
@@ -361,7 +361,7 @@ CREATE TABLE "invoice_items" (
361
361
  "stripe_invoice_id" varchar(765) DEFAULT NULL,
362
362
  "is_vat" int2 NOT NULL DEFAULT '0',
363
363
  PRIMARY KEY (id)
364
- );
364
+ ) DISTKEY(id) SORTKEY(id);
365
365
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'invoice_items';
366
366
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
367
367
  ('invoice_items', 'id', 'int4(11)', 1),
@@ -410,7 +410,7 @@ CREATE TABLE "zerofill_table" (
410
410
  "name" varchar(768) DEFAULT NULL,
411
411
  "value_small_int" int4 DEFAULT NULL,
412
412
  PRIMARY KEY (id)
413
- );
413
+ ) DISTKEY(id) SORTKEY(id);
414
414
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'zerofill_table';
415
415
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
416
416
  ('zerofill_table', 'id', 'int4(11)', 1),
@@ -28,7 +28,7 @@ CREATE TABLE "test_table" (
28
28
  "age" int8,
29
29
  "value" varchar(max),
30
30
  PRIMARY KEY (id)
31
- );
31
+ ) DISTKEY(id) SORTKEY(id);
32
32
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
33
33
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
34
34
  ('test_table', 'id', 'int4(11)', 1),
@@ -58,7 +58,7 @@ CREATE TABLE "test_table" (
58
58
  "age" int8,
59
59
  "value" varchar(max),
60
60
  PRIMARY KEY (id)
61
- );
61
+ ) DISTKEY(id) SORTKEY(id);
62
62
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
63
63
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
64
64
  ('test_table', 'id', 'int4(11)', 1),
@@ -86,7 +86,7 @@ CREATE TABLE "test_table" (
86
86
  "value" varchar(max),
87
87
  "cmnt" varchar(max),
88
88
  PRIMARY KEY (id)
89
- );
89
+ ) DISTKEY(id) SORTKEY(id);
90
90
  COMMENT ON COLUMN "test_table"."cmnt"
91
91
  IS 'helloworld';
92
92
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
@@ -127,7 +127,7 @@ CREATE TABLE "test_schema"."test_table" (
127
127
  "value" varchar(max),
128
128
  "cmnt" varchar(max),
129
129
  PRIMARY KEY (id)
130
- );
130
+ ) DISTKEY(id) SORTKEY(id);
131
131
  COMMENT ON COLUMN "test_schema"."test_table"."cmnt"
132
132
  IS 'helloworld';
133
133
  DELETE FROM "test_schema"."flydata_ctl_columns" WHERE table_name = 'test_table';
data/flydata.gemspec CHANGED
@@ -2,16 +2,14 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: flydata 0.3.0 ruby lib
6
5
 
7
6
  Gem::Specification.new do |s|
8
7
  s.name = "flydata"
9
- s.version = "0.3.0"
8
+ s.version = "0.3.1"
10
9
 
11
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
12
- s.require_paths = ["lib"]
13
11
  s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
14
- s.date = "2015-02-11"
12
+ s.date = "2015-02-24"
15
13
  s.description = "FlyData Agent"
16
14
  s.email = "sysadmin@flydata.com"
17
15
  s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -37,6 +35,7 @@ Gem::Specification.new do |s|
37
35
  "flydata-core/Gemfile.lock",
38
36
  "flydata-core/circle.yml",
39
37
  "flydata-core/lib/flydata-core.rb",
38
+ "flydata-core/lib/flydata-core/config/user_maintenance.rb",
40
39
  "flydata-core/lib/flydata-core/core_ext.rb",
41
40
  "flydata-core/lib/flydata-core/core_ext/module.rb",
42
41
  "flydata-core/lib/flydata-core/core_ext/module/include.rb",
@@ -48,6 +47,7 @@ Gem::Specification.new do |s|
48
47
  "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
49
48
  "flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
50
49
  "flydata-core/lib/flydata-core/thread_context.rb",
50
+ "flydata-core/spec/config/user_maintenance_spec.rb",
51
51
  "flydata-core/spec/logger_spec.rb",
52
52
  "flydata-core/spec/spec_helper.rb",
53
53
  "flydata-core/spec/table_def/mysql_table_def_spec.rb",
@@ -150,7 +150,8 @@ Gem::Specification.new do |s|
150
150
  ]
151
151
  s.homepage = "http://flydata.com/"
152
152
  s.licenses = ["All right reserved."]
153
- s.rubygems_version = "2.4.3"
153
+ s.require_paths = ["lib"]
154
+ s.rubygems_version = "2.0.14"
154
155
  s.summary = "FlyData Agent"
155
156
 
156
157
  if s.respond_to? :specification_version then
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2015-02-11 00:00:00.000000000 Z
15
+ date: 2015-02-24 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -440,6 +440,7 @@ files:
440
440
  - flydata-core/Gemfile.lock
441
441
  - flydata-core/circle.yml
442
442
  - flydata-core/lib/flydata-core.rb
443
+ - flydata-core/lib/flydata-core/config/user_maintenance.rb
443
444
  - flydata-core/lib/flydata-core/core_ext.rb
444
445
  - flydata-core/lib/flydata-core/core_ext/module.rb
445
446
  - flydata-core/lib/flydata-core/core_ext/module/include.rb
@@ -451,6 +452,7 @@ files:
451
452
  - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
452
453
  - flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
453
454
  - flydata-core/lib/flydata-core/thread_context.rb
455
+ - flydata-core/spec/config/user_maintenance_spec.rb
454
456
  - flydata-core/spec/logger_spec.rb
455
457
  - flydata-core/spec/spec_helper.rb
456
458
  - flydata-core/spec/table_def/mysql_table_def_spec.rb
@@ -570,7 +572,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
570
572
  version: '0'
571
573
  requirements: []
572
574
  rubyforge_project:
573
- rubygems_version: 2.4.3
575
+ rubygems_version: 2.0.14
574
576
  signing_key:
575
577
  specification_version: 4
576
578
  summary: FlyData Agent