flydata 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68f311d20207752e80dd5fb47b905b085e840520
-  data.tar.gz: 6bfa38fa63a7db7e1f4b7109324017e99c82c117
+  metadata.gz: 9b34f397692efd9d9bc9575588d19263b9c0df2c
+  data.tar.gz: 21b42d4c19267a6656eb8f8d8c21607cf54501f1
 SHA512:
-  metadata.gz: 01469b9c9b268bc8dad0bda81a3dbcc3a98a55901108cac17b50a6688313485f1d7cdd8a01b26ea20cbea0d8fa6f4c096055394f55cea58ee59a949acb2963fe
-  data.tar.gz: cde53114405b00f034e7444fd0b9ba9df9519a2cd3979eade6a99d48e80e326615a1ba16fb0d5c014c51d23272ca5cf2257b2f318135ad6efef858f4d6d4479b
+  metadata.gz: 147ed1f61574abacfc850460e4745953c37bba5827985bd82adc51c9ef6750cb6188f3e56f52a3e855f73096c9fb6c05e97039ed68ac0982029a259f8a9a8c78
+  data.tar.gz: a73489d7abcfb93db8720819143ad751d78493389d911407ede925a8f2417ba9161d7a742365b97de56595ca168139832ef179f37a025ce47e015da97b03829d

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.3.1

data/flydata-core/lib/flydata-core/config/user_maintenance.rb

@@ -0,0 +1,115 @@
+module FlydataCore
+  module Config
+
+    # This class manages user maintenance mode
+    # We have two user maintenance mode as following and controll user's mode by putting a file.
+    #
+    # ## User Mode
+    #
+    # - maintenance mode <stop actions against user's resource like Redshift>
+    #   - fluent-output-plugin#format: Append data to buffer file
+    #   - fluent-output-plugin#write: Stop processing chunk file and throw an error (repeat retry)
+    #   - copy-handler: Stop processing.
+    #
+    #- discard mode
+    #   - fluent-output-plugin#format: Discard data
+    #   - fluent-output-plugin#format: Return true without processing chunk files
+    #   - copy-handler: Stop processing.
+    #
+    #
+    # ## Directory Structure
+    #
+    # /mnt/flydata/system/user_maintenance
+    #   + [data-port-id]
+    #     - maintenance_[data-port-id]_[data-entry-id]_[(schema-name.)table-name]
+    #     - discard_[data-port-id]_[data-entry-id]_[(schema-name.)table-name]
+    #
+    class UserMaintenance
+      def initialize(base_dir_path)
+        @base_dir_path = base_dir_path
+      end
+
+      attr_reader :base_dir_path
+
+      # Argument "params" of methods
+      #
+      # params: Hash
+      #   data_port_id
+      #   data_entry_id
+      #   table_name
+
+      # For flydata-web
+
+      def maintenance_mode_file_path(params)
+        mode_file_path(:maintenance, params)
+      end
+
+      def discard_mode_file_path(params)
+        mode_file_path(:discard, params)
+      end
+
+      # For fluentd plugin and copy handler
+
+      def maintenance_mode?(params)
+        check_mode(:maintenance, params)
+      end
+
+      def discard_mode?(params)
+        check_mode(:discard, params)
+      end
+
+      def current_user_mode(params)
+        if discard_mode?(params)
+          :discard
+        elsif maintenance_mode?(params)
+          :maintenance
+        else
+          nil
+        end
+      end
+
+      private
+
+      def validate_params(params)
+        raise ArgumentError.new("params must be hash")  unless params.kind_of?(Hash)
+        raise ArgumentError.new("params must have 'data_port_id' key")  unless params[:data_port_id]
+      end
+
+      def check_mode(mode, params)
+        validate_params(params)
+        return false unless params[:data_port_id]
+        return true if File.exists?(mode_file_path(mode, data_port_id: params[:data_port_id]))
+        return false unless params[:data_entry_id]
+        de_mode_file_path = mode_file_path(mode, data_port_id: params[:data_port_id], data_entry_id: params[:data_entry_id])
+        return true if File.exists?(de_mode_file_path)
+        return false unless params[:table_name]
+
+        # Ignore case of table_name
+        table_name = params[:table_name].downcase
+        tn_mode_file_path = mode_file_path(mode, data_port_id: params[:data_port_id], data_entry_id: params[:data_entry_id], table_name: table_name)
+        Dir.glob("#{de_mode_file_path}_*").any?{|fp| File.join(File.dirname(fp), File.basename(fp).downcase) == tn_mode_file_path}
+      end
+
+      # /mnt/flydata/system/user_maintenance/[dp-id]/[mode]_[data-port-id]_[data-entry-id]_[table-name]
+      def mode_file_path(mode, params)
+        validate_params(params)
+        File.join(dp_mode_dir_path(mode, params), mode_file_name(mode, params))
+      end
+
+      # /mnt/flydata/system/user_maintenance/[dp-id]
+      def dp_mode_dir_path(mode, params)
+        File.join(@base_dir_path, params[:data_port_id].to_s)
+      end
+
+      # [mode]_[data-port-id]_[data-entry-id]_[table-name]
+      def mode_file_name(mode, params)
+        return nil unless params[:data_port_id]
+        path = "#{mode}_#{params[:data_port_id]}"
+        return path unless params[:data_entry_id]
+        path << "_#{params[:data_entry_id]}"
+        return path unless params[:table_name]
+        path << "_#{params[:table_name]}"
+      end
+    end
+  end
+end

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -89,7 +89,7 @@ EOS
 DROP TABLE %s;
 CREATE TABLE %s (
 %s
-);
+)%s;
 EOS
 
   def self.create_table_sql(flydata_tabledef, schema_name)
@@ -99,9 +99,11 @@ EOS
 
     contents = lines.join(",\n")
 
+    dk_sk_def = distkey_sortkey_sql(flydata_tabledef)
+
     table_name = flydata_tabledef[:table_name]
     redshift_tbl = table_name_for_ddl(table_name, schema_name)
-    CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents]
+    CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents, dk_sk_def]
   end
 
   def self.column_def_sql(column, opt = {})
@@ -173,10 +175,20 @@ EOS
   end
 
   def self.primary_key_sql(flydata_tabledef)
-    pks = flydata_tabledef[:columns].select{|col| col[:primary_key]}.collect{|col| col[:column]}
+    pks = primary_keys(flydata_tabledef)
     pks.empty? ? nil : "  PRIMARY KEY (#{pks.join(',')})"
   end
 
+  def self.primary_keys(flydata_tabledef)
+    flydata_tabledef[:columns].select{|col| col[:primary_key]}.collect{|col| col[:column]}
+  end
+
+  def self.distkey_sortkey_sql(flydata_tabledef)
+    pks = primary_keys(flydata_tabledef)
+    return nil if pks.empty?
+    " DISTKEY(#{pks.first}) SORTKEY(#{pks.join(',')})"
+  end
+
   def self.comment_sql(flydata_tabledef, schema_name)
     sql = ""
     flydata_tabledef[:columns].each do |col|

data/flydata-core/spec/config/user_maintenance_spec.rb

@@ -0,0 +1,78 @@
+require_relative '../spec_helper'
+
+require 'tempfile'
+require 'flydata-core/config/user_maintenance'
+
+module FlydataCore::Config
+  describe UserMaintenance do
+    let(:temp_dir) { Dir.mktmpdir('flydatads_rspec_home') }
+    let(:user_maintenance_dir_path) { File.join(temp_dir, 'user_maintenance') }
+    let(:default_user_maintenance) { described_class.new(user_maintenance_dir_path) }
+    let(:user_maintenance) { default_user_maintenance }
+    let(:default_params) { { data_port_id: 1, data_entry_id: 2, table_name: 'test_table' } }
+    let(:params) { default_params }
+
+    describe '#maintenance_mode_file_path' do
+      subject { user_maintenance.maintenance_mode_file_path(params) }
+
+      context 'with data_port_id' do
+        let(:params) { { data_port_id: 1 } }
+        it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'maintenance_1')) }
+      end
+
+      context 'with data_port_id and data_entry_id' do
+        let(:params) { { data_port_id: 1, data_entry_id: 2 } }
+        it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'maintenance_1_2')) }
+      end
+
+      context 'with data_port_id, data_entry_id and table_name' do
+        it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'maintenance_1_2_test_table')) }
+      end
+
+      context 'with invalid object' do
+        let(:params) { nil }
+        it { expect{subject}.to raise_error(ArgumentError) }
+      end
+
+      context 'with empty hash' do
+        let(:params) { {} }
+        it { expect{subject}.to raise_error(ArgumentError) }
+      end
+    end
+
+    describe '#discard_mode_file_path' do
+      subject { user_maintenance.discard_mode_file_path(params) }
+
+      context 'with data_port_id' do
+        let(:params) { { data_port_id: 1 } }
+        it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'discard_1')) }
+      end
+
+      context 'with data_port_id and data_entry_id' do
+        let(:params) { { data_port_id: 1, data_entry_id: 2 } }
+        it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'discard_1_2')) }
+      end
+
+      context 'with data_port_id, data_entry_id and table_name' do
+        it { is_expected.to eq(File.join(user_maintenance_dir_path, '1', 'discard_1_2_test_table')) }
+      end
+
+      context 'with invalid object' do
+        let(:params) { nil }
+        it { expect{subject}.to raise_error(ArgumentError) }
+      end
+
+      context 'with empty hash' do
+        let(:params) { {} }
+        it { expect{subject}.to raise_error(ArgumentError) }
+      end
+    end
+
+    describe '#maintenance_mode?' do
+      subject { user_maintenance.maintenance_mode?(params) }
+
+      context 'when in maintenance mode per data-port' do
+      end
+    end
+  end
+end

data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb

@@ -66,7 +66,7 @@ CREATE TABLE "test_table_all" (
   "col_varbinary" varchar(512) DEFAULT NULL,
   "col_varchar" varchar(372) DEFAULT NULL,
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_all';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table_all', 'id', 'int8(20)', 1),
@@ -120,7 +120,7 @@ CREATE TABLE "bit_test_def_1" (
   "bit_value" bigint DEFAULT 1,
   "int_value" int4 DEFAULT 16,
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'bit_test_def_1';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('bit_test_def_1', 'id', 'int4(11)', 1),
@@ -151,7 +151,7 @@ CREATE TABLE "product_order" (
   "product_id" int4 NOT NULL,
   "customer_id" int4 NOT NULL,
   PRIMARY KEY (no)
-);
+) DISTKEY(no) SORTKEY(no);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'product_order';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('product_order', 'no', 'int4(11)', 1),
@@ -181,7 +181,7 @@ CREATE TABLE "test_table_column_comment" (
   "id" int4 NOT NULL DEFAULT '0',
   "value" varchar(max),
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 COMMENT ON COLUMN "test_table_column_comment"."id"
   IS 'this is primary key';
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_column_comment';
@@ -213,7 +213,7 @@ CREATE TABLE "test_table_enum" (
   "enum_2" varchar encode bytedict DEFAULT 'a',
   "enum_3" varchar encode bytedict NOT NULL,
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_enum';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table_enum', 'id', 'int4(11)', 1),
@@ -244,7 +244,7 @@ CREATE TABLE "test_table_multi_pk" (
   "id2" int4 NOT NULL DEFAULT '0',
   "value" varchar(max),
   PRIMARY KEY (id1,id2)
-);
+) DISTKEY(id1) SORTKEY(id1,id2);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_multi_pk';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table_multi_pk', 'id1', 'int4(11)', 1),
@@ -283,7 +283,7 @@ CREATE TABLE "sample1" (
   "name" varchar(max),
   "num" int4 DEFAULT NULL,
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'sample1';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('sample1', 'id', 'int4(11)', 1),
@@ -315,7 +315,7 @@ CREATE TABLE "sample1" (
   "name" varchar(max),
   "num" int4 DEFAULT NULL,
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'sample1';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('sample1', 'id', 'int4(11)', 1),
@@ -361,7 +361,7 @@ CREATE TABLE "invoice_items" (
   "stripe_invoice_id" varchar(765) DEFAULT NULL,
   "is_vat" int2 NOT NULL DEFAULT '0',
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'invoice_items';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('invoice_items', 'id', 'int4(11)', 1),
@@ -410,7 +410,7 @@ CREATE TABLE "zerofill_table" (
   "name" varchar(768) DEFAULT NULL,
   "value_small_int" int4 DEFAULT NULL,
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'zerofill_table';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('zerofill_table', 'id', 'int4(11)', 1),

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -28,7 +28,7 @@ CREATE TABLE "test_table" (
   "age" int8,
   "value" varchar(max),
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table', 'id', 'int4(11)', 1),
@@ -58,7 +58,7 @@ CREATE TABLE "test_table" (
   "age" int8,
   "value" varchar(max),
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table', 'id', 'int4(11)', 1),
@@ -86,7 +86,7 @@ CREATE TABLE "test_table" (
   "value" varchar(max),
   "cmnt" varchar(max),
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 COMMENT ON COLUMN "test_table"."cmnt"
   IS 'helloworld';
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
@@ -127,7 +127,7 @@ CREATE TABLE "test_schema"."test_table" (
   "value" varchar(max),
   "cmnt" varchar(max),
   PRIMARY KEY (id)
-);
+) DISTKEY(id) SORTKEY(id);
 COMMENT ON COLUMN "test_schema"."test_table"."cmnt"
   IS 'helloworld';
 DELETE FROM "test_schema"."flydata_ctl_columns" WHERE table_name = 'test_table';

data/flydata.gemspec

@@ -2,16 +2,14 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.0"
+  s.version = "0.3.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-02-11"
+  s.date = "2015-02-24"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -37,6 +35,7 @@ Gem::Specification.new do |s|
     "flydata-core/Gemfile.lock",
     "flydata-core/circle.yml",
     "flydata-core/lib/flydata-core.rb",
+    "flydata-core/lib/flydata-core/config/user_maintenance.rb",
     "flydata-core/lib/flydata-core/core_ext.rb",
     "flydata-core/lib/flydata-core/core_ext/module.rb",
     "flydata-core/lib/flydata-core/core_ext/module/include.rb",
@@ -48,6 +47,7 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
     "flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
     "flydata-core/lib/flydata-core/thread_context.rb",
+    "flydata-core/spec/config/user_maintenance_spec.rb",
     "flydata-core/spec/logger_spec.rb",
     "flydata-core/spec/spec_helper.rb",
     "flydata-core/spec/table_def/mysql_table_def_spec.rb",
@@ -150,7 +150,8 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.3"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.0.14"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-11 00:00:00.000000000 Z
+date: 2015-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -440,6 +440,7 @@ files:
 - flydata-core/Gemfile.lock
 - flydata-core/circle.yml
 - flydata-core/lib/flydata-core.rb
+- flydata-core/lib/flydata-core/config/user_maintenance.rb
 - flydata-core/lib/flydata-core/core_ext.rb
 - flydata-core/lib/flydata-core/core_ext/module.rb
 - flydata-core/lib/flydata-core/core_ext/module/include.rb
@@ -451,6 +452,7 @@ files:
 - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
 - flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
 - flydata-core/lib/flydata-core/thread_context.rb
+- flydata-core/spec/config/user_maintenance_spec.rb
 - flydata-core/spec/logger_spec.rb
 - flydata-core/spec/spec_helper.rb
 - flydata-core/spec/table_def/mysql_table_def_spec.rb
@@ -570,7 +572,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: FlyData Agent