flydata 0.3.20 → 0.3.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e06e855feb4eedb44d34602d0e6d447f72eddc2f
-  data.tar.gz: 960b559b4006bfcea9fecb6b96ef0457f2eed949
+  metadata.gz: 0c4aa0461989a989042cf8117321b6fb0a25290b
+  data.tar.gz: 489443d11c79f35f1736bf52c61b438b7dfe7e54
 SHA512:
-  metadata.gz: 0cf8f7f398d49b1081be49e96d0111a5e9bb88102bdea6fb148a015629cd62d0140e11f22a0ad0e6bd490a3993d362a0d5c7af29fa58829434649487d0dcf7b2
-  data.tar.gz: cf814b03327a2230bb90058c82c72b3c6e1b06225a2048dff20dd6744656614418cf609bd7c4a452570d229ff34c1ba4eca54e33b99f202f6a6c54850fab183c
+  metadata.gz: f1cf691ead10def34ef2f61fc1be6d879a1dc3761e4cda2610b47afd05e227f8b435eece8b75dfbc4b338eb774848f0bd70474ed3fa53e702da8d4fbddd7bad9
+  data.tar.gz: 73ba527eecb4925518fcf34f432cc397ee6059ff4c166c199666c5dd9e3601fd8a5fb245d32a0fcae4f4ea62bf9ba4b9c6738a88c8f8e99be302c5bfc60bd0b6

data/VERSION

@@ -1 +1 @@
-0.3.20
+0.3.21

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -4,6 +4,8 @@ require 'flydata-core/errors'
 module FlydataCore
 module TableDef
 
+class NameTooLongError < RuntimeError; end
+
 class RedshiftTableDef
   TYPE_MAP_F2R = {
     'binary' => {type: 'varchar', use_params: true, default_value: ''},
@@ -146,7 +148,7 @@ EOS
               else
                 type_info[:type]
               end
-    line = %Q|  "#{column[:column]}" #{rs_type}|
+    line = %Q|  "#{convert_to_valid_column_name(column[:column])}" #{rs_type}|
     line += " NOT NULL" if column[:not_null]
     if (column.has_key?(:default))
       val = replace_default_value(type, type_info[:type], column[:default])
@@ -271,6 +273,37 @@ EOS
     sql
   end
 
+  # convert key to valid a column name
+  #   http://docs.aws.amazon.com/redshift/latest/dg/r_names.html
+  #   http://www.postgresql.org/docs/8.0/static/ddl-system-columns.html
+  def self.convert_to_valid_column_name(key)
+    convert_to_valid_name(key) {|n| redshift_reserved_column_name?(n) }
+  end
+
+  def self.convert_to_valid_table_name(key)
+    convert_to_valid_name(key)
+  end
+
+  MAX_COLUMNNAME_LENGTH = 127
+
+  def self.convert_to_valid_name(key)
+    name = key.downcase.gsub(/[^a-z0-9_$]/, '_')
+    do_convert = yield name if block_given?
+    name = "_#{name}" if do_convert || name =~ /^[0-9$]/
+    if name.length > MAX_COLUMNNAME_LENGTH
+      raise NameTooLongError, "Converted name is too long. size: #{name.size} name: #{name}"
+    end
+    name
+  end
+
+  REDSHIFT_RESERVED_COLUMNS = %w[oid tableoid xmin cmin xmax cmax ctid deletexid insertxid]
+  REDSHIFT_RESERVED_COLUMNS_HASH = REDSHIFT_RESERVED_COLUMNS.inject({}) {|h, word| h[word.to_sym] = true; h}
+
+  def self.redshift_reserved_column_name?(name)
+    return false unless name
+    REDSHIFT_RESERVED_COLUMNS_HASH[name.to_sym] == true
+  end
+
   def self.escape(text)
     text.gsub("'", "\\\\'")
   end

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -5,6 +5,7 @@ module FlydataCore
 module TableDef
 
 describe RedshiftTableDef do
+  let(:table_name) { 'test_table' }
 
   describe '.from_flydata_tabledef' do
     subject { described_class.from_flydata_tabledef(flydata_tabledef, option) }
@@ -127,6 +128,56 @@ EOT
   describe '.column_def_sql' do
     subject { described_class.column_def_sql(column, opt) }
     let(:column) { {} }
+    let(:opt) { {} }
+
+    shared_examples "test generated Redshift SQL for column" do
+      it do
+         expect(subject).to eq(expected_query)
+      end
+    end
+
+    context "test different column names" do
+      before do
+        column[:type] = "varchar(12)"
+      end
+      let(:expected_query) { %Q|  "#{expected_column_name}" varchar(12)| }
+
+      context "normal column name" do
+        before do
+          column[:column] = "all_lower_case_not_reserved"
+        end
+        let(:expected_column_name) { "all_lower_case_not_reserved" }
+
+        it_behaves_like "test generated Redshift SQL for column"
+      end
+
+      context "capitalized column name" do
+        before do
+          column[:column] = "CapitalizedNotReserved"
+        end
+        let(:expected_column_name) { "capitalizednotreserved" }
+
+        it_behaves_like "test generated Redshift SQL for column"
+      end
+
+      context "reserved column name" do
+        before do
+          column[:column] = "oid"
+        end
+        let(:expected_column_name) { "_oid" }
+
+        it_behaves_like "test generated Redshift SQL for column"
+      end
+
+      context "captalized reserved column name" do
+        before do
+          column[:column] = "OID"
+        end
+        let(:expected_column_name) { "_oid" }
+
+        it_behaves_like "test generated Redshift SQL for column"
+      end
+    end
 
     let(:no_default_sql) { "" }
     shared_examples "default values" do |*examples|
@@ -171,12 +222,6 @@ EOT
       end
     end
 
-    shared_examples "test generated Redshift SQL for column" do
-      it do
-         expect(subject).to eq(expected_query)
-      end
-    end
-
     shared_examples "test column types" do |*examples|
       context 'with varchar column def' do
         context 'when size is smaller than max size' do
@@ -478,6 +523,53 @@ EOT
     end
   end
 
+  describe '.flydata_ctl_columns_sql' do
+    subject { described_class.flydata_ctl_columns_sql(flydata_tabledef, schema_name) }
+    let(:flydata_tabledef) { {
+        table_name: table_name,
+        columns: [
+          { column: column_name, type: "varchar(12)" },
+        ],
+        default_charset: "UTF_8"} }
+    let(:schema_name) { nil }
+
+    let(:expected_sql) { <<EOS.strip }
+DELETE FROM "flydata_ctl_columns" WHERE table_name = '#{table_name}';
+INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
+('#{table_name}', '#{expected_column_name}', 'varchar(12)', 1);
+EOS
+
+    context "test different column names" do
+      context "normal column name" do
+        let(:column_name) { "all_lower_case_not_reserved" }
+        let(:expected_column_name) { column_name }
+
+        it { is_expected.to eq expected_sql }
+      end
+
+      context "capitalized column name" do
+        let(:column_name) { "CapitalizedNotReserved" }
+        let(:expected_column_name) { column_name }
+
+        it { is_expected.to eq expected_sql }
+      end
+
+      context "reserved column name" do
+        let(:column_name) { "oid" }
+        let(:expected_column_name) { column_name }
+
+        it { is_expected.to eq expected_sql }
+      end
+
+      context "captalized reserved column name" do
+        let(:column_name) { "OID" }
+        let(:expected_column_name) { column_name }
+
+        it { is_expected.to eq expected_sql }
+      end
+    end
+  end
+
   describe '.parse_timestamp' do
     let(:value) { nil }
     subject { described_class.parse_timestamp(value) }

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.20 ruby lib
+# stub: flydata 0.3.21 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.20"
+  s.version = "0.3.21"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-05-13"
+  s.date = "2015-05-14"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -181,7 +181,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.6"
+  s.rubygems_version = "2.4.3"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.20
+  version: 0.3.21
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-13 00:00:00.000000000 Z
+date: 2015-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -621,7 +621,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent