flydata 0.3.21 → 0.3.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c4aa0461989a989042cf8117321b6fb0a25290b
-  data.tar.gz: 489443d11c79f35f1736bf52c61b438b7dfe7e54
+  metadata.gz: 9e5b9816ca7b0901b9716d3361a344f8842f4e67
+  data.tar.gz: 6c1061fd528b620f93273b5ba737da1f78d81803
 SHA512:
-  metadata.gz: f1cf691ead10def34ef2f61fc1be6d879a1dc3761e4cda2610b47afd05e227f8b435eece8b75dfbc4b338eb774848f0bd70474ed3fa53e702da8d4fbddd7bad9
-  data.tar.gz: 73ba527eecb4925518fcf34f432cc397ee6059ff4c166c199666c5dd9e3601fd8a5fb245d32a0fcae4f4ea62bf9ba4b9c6738a88c8f8e99be302c5bfc60bd0b6
+  metadata.gz: 3b8295cd89f049cde9393e02bd8ea07b37f0cc08d0584bfde3ae54b735af78172bcc6157045f58a81dfffab05361df8dc83886be48f881777f4656b3d2626825
+  data.tar.gz: 6f67eac41f7ba8a7794d696c1b7bfe2466628a54153be62b40ace4a8fce70b4f4201ec44807e9609dd565b5021b8682551e68072f22101cb78fb1763e76173c8

data/VERSION

@@ -1 +1 @@
-0.3.21
+0.3.22

data/flydata-core/lib/flydata-core/redshift/string.rb

@@ -2,22 +2,35 @@ module FlydataCore
 module Redshift
 
 class String
+  NULL_CHAR_CODE_POINTS = "\\0".codepoints
+
   # Redshift supports UTF-8 but it enforces stricter rule than other
   # implementations such as MySQL or Ruby.  This method returns a
   # Redshift-safe string from the given string.
   def self.encode(string, options = {})
-    result = string.encoding == Encoding::UTF_8 ?
-              string.encode(Encoding::UTF_16, options).encode(Encoding::UTF_8) :
-              string.encode(Encoding::UTF_8, options)
-    rep_str = options[:replace] || "\uFFFD"
-    result.each_codepoint.with_index(0) do |cp, i|
+    encoded_str = string.encoding == Encoding::UTF_8 ?
+                  string.encode(Encoding::UTF_16, options).encode(Encoding::UTF_8) :
+                  string.encode(Encoding::UTF_8, options)
+
+    ret_cpts = []
+    rep_str_cpts = (options[:replace] || "\uFFFD").codepoints
+    replaced = false
+
+    encoded_str.each_codepoint do |cp|
       # Per Redshift document
       # http://docs.aws.amazon.com/redshift/latest/dg/multi-byte-character-load-errors.html
       if cp >= 0xFDD0 && cp <= 0xFDEF || cp == 0xFFFE || cp == 0xFFFF
-        result[i] = rep_str
+        ret_cpts.concat(rep_str_cpts)
+        replaced = true
+      elsif cp == 0x0000
+        ret_cpts.concat(NULL_CHAR_CODE_POINTS)
+        replaced = true
+      else
+        ret_cpts << cp
       end
     end
-    result
+
+    replaced ? ret_cpts.pack("U*") : encoded_str
   end
 end
 

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -211,17 +211,17 @@ EOS
 
   def self.primary_key_sql(flydata_tabledef)
     pks = primary_keys(flydata_tabledef)
-    pks.empty? ? nil : "  PRIMARY KEY (#{pks.join(',')})"
+    pks.empty? ? nil : "  PRIMARY KEY (#{pks.collect{|pk| %Q|"#{pk}"|}.join(',')})"
   end
 
   def self.primary_keys(flydata_tabledef)
-    flydata_tabledef[:columns].select{|col| col[:primary_key]}.collect{|col| col[:column]}
+    flydata_tabledef[:columns].select{|col| col[:primary_key]}.collect{|col| convert_to_valid_column_name(col[:column])}
   end
 
   def self.distkey_sortkey_sql(flydata_tabledef)
     pks = primary_keys(flydata_tabledef)
     return nil if pks.empty?
-    " DISTKEY(#{pks.first}) SORTKEY(#{pks.join(',')})"
+    %Q| DISTKEY("#{pks.first}") SORTKEY(#{pks.collect{|pk| %Q|"#{pk}"|}.join(',')})|
   end
 
   def self.comment_sql(flydata_tabledef, schema_name)
@@ -230,7 +230,7 @@ EOS
       next unless col[:comment]
 
       sql += <<EOS
-COMMENT ON COLUMN #{table_name_for_ddl(flydata_tabledef[:table_name], schema_name)}."#{col[:column]}"
+COMMENT ON COLUMN #{table_name_for_ddl(flydata_tabledef[:table_name], schema_name)}."#{convert_to_valid_column_name(col[:column])}"
   IS '#{col[:comment]}';
 EOS
     end
@@ -289,7 +289,7 @@ EOS
   def self.convert_to_valid_name(key)
     name = key.downcase.gsub(/[^a-z0-9_$]/, '_')
     do_convert = yield name if block_given?
-    name = "_#{name}" if do_convert || name =~ /^[0-9$]/
+    name = "_#{name}" if do_convert
     if name.length > MAX_COLUMNNAME_LENGTH
       raise NameTooLongError, "Converted name is too long. size: #{name.size} name: #{name}"
     end
@@ -358,7 +358,7 @@ EOS
   def self.parse_date(value)
     return nil if value.nil?
     value_str = value.to_s
-    dt = Date.parse(convert_year_into_date(value_str))
+    dt = Date.parse(convert_date(value_str))
     dt.strftime('%Y-%m-%d')
   rescue ArgumentError => ae
     # '0000-00-00' is valid for mysql date column
@@ -366,18 +366,28 @@ EOS
     raise ae
   end
 
+  CONVERT_ZERO_DATE_REGEXP = /^0000|-00/
+  CONVERT_ZERO_DATE_HASH_TABLE = {"0000" => "0001", "-00" => "-01" }
+
+  def self.convert_date(value)
+    value = convert_year_into_date(value)
+    # Convert any 00 value to 01
+    value.gsub(CONVERT_ZERO_DATE_REGEXP, CONVERT_ZERO_DATE_HASH_TABLE)
+  end
+
   def self.convert_year_into_date(value)
-    if value == '0' || value == '0000'
-      converted_value = '0001-01-01' # '0001-01-01' is the "base" date
+    return value unless /^\d+$/.match(value)
+    # Only positive integers are allowed for year type
+    val_i = value.to_i
+
+    if val_i == 0  # '0' or '0000'
+      '0001-01-01' # '0001-01-01' is the "base" date
+    elsif value.length == 4
+      "#{value}-01-01"
+    elsif value.length == 2
+      "#{val_i >= 70 ? "19" : "20"}#{value}-01-01"
     else
-      case value
-      when /^\d{4}$/
-        converted_value = "#{value}-01-01"
-      when /^\d{2}$/
-        converted_value = "#{value.to_i >= 70 ? "19" : "20"}#{value}-01-01"
-      else
-        converted_value = value # Return the value as is
-      end
+      value # Return the value as is
     end
   end
 end

data/flydata-core/spec/redshift/string_spec.rb

@@ -39,7 +39,7 @@ describe String do
         end
         context 'invalid char' do
           let(:string) { "\xeb\x13\x00" }
-          let(:result) { "\uFFFD\u0013\u0000".encode(Encoding::UTF_8) }
+          let(:result) { "\uFFFD\u0013\\0".encode(Encoding::UTF_8) }
 
           it_behaves_like "converting string to a Redshift friendly string"
         end
@@ -53,6 +53,12 @@ describe String do
           let(:string) { "\xef\xbf\xbf" }
           let(:result) { "\uFFFD".encode(Encoding::UTF_8) }
 
+          it_behaves_like "converting string to a Redshift friendly string"
+        end
+        context 'null chars' do
+          let(:string) { "\x00\x00\x00aabbcc\x00\x00ddeeff\x00あい\x00う\x00\x00" }
+          let(:result) { "\\0\\0\\0aabbcc\\0\\0ddeeff\\0あい\\0う\\0\\0".encode(Encoding::UTF_8) }
+
           it_behaves_like "converting string to a Redshift friendly string"
         end
       end

data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb

@@ -76,8 +76,8 @@ CREATE TABLE "test_table_all" (
   "col_year" date DEFAULT NULL,
   "col_year_4" date DEFAULT NULL,
   "col_year_2" date DEFAULT NULL,
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_all';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table_all', 'id', 'int8(20)', 1),
@@ -178,8 +178,8 @@ CREATE TABLE "bit_test_def_1" (
   "id" int4 NOT NULL,
   "bit_value" bigint DEFAULT 1,
   "int_value" int4 DEFAULT 16,
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'bit_test_def_1';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('bit_test_def_1', 'id', 'int4(11)', 1),
@@ -227,8 +227,8 @@ CREATE TABLE "product_order" (
   "product_category" int4 NOT NULL,
   "product_id" int4 NOT NULL,
   "customer_id" int4 NOT NULL,
-  PRIMARY KEY (no)
-) DISTKEY(no) SORTKEY(no);
+  PRIMARY KEY ("no")
+) DISTKEY("no") SORTKEY("no");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'product_order';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('product_order', 'no', 'int4(11)', 1),
@@ -280,8 +280,8 @@ DROP TABLE IF EXISTS "test_table_column_comment";
 CREATE TABLE "test_table_column_comment" (
   "id" int4 NOT NULL DEFAULT '0',
   "value" varchar(max),
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
 COMMENT ON COLUMN "test_table_column_comment"."id"
   IS 'this is primary key';
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_column_comment';
@@ -329,8 +329,8 @@ CREATE TABLE "test_table_enum" (
   "enum_1" varchar encode bytedict DEFAULT NULL,
   "enum_2" varchar encode bytedict DEFAULT 'a',
   "enum_3" varchar encode bytedict NOT NULL,
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_enum';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table_enum', 'id', 'int4(11)', 1),
@@ -379,8 +379,8 @@ CREATE TABLE "test_table_multi_pk" (
   "id1" int4 NOT NULL DEFAULT '0',
   "id2" int4 NOT NULL DEFAULT '0',
   "value" varchar(max),
-  PRIMARY KEY (id1,id2)
-) DISTKEY(id1) SORTKEY(id1,id2);
+  PRIMARY KEY ("id1","id2")
+) DISTKEY("id1") SORTKEY("id1","id2");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_multi_pk';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table_multi_pk', 'id1', 'int4(11)', 1),
@@ -436,8 +436,8 @@ CREATE TABLE "sample1" (
   "title" varchar(768) DEFAULT NULL,
   "name" varchar(max),
   "num" int4 DEFAULT NULL,
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'sample1';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('sample1', 'id', 'int4(11)', 1),
@@ -489,8 +489,8 @@ CREATE TABLE "sample1" (
   "title" varchar(768) DEFAULT NULL,
   "name" varchar(max),
   "num" int4 DEFAULT NULL,
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'sample1';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('sample1', 'id', 'int4(11)', 1),
@@ -556,8 +556,8 @@ CREATE TABLE "invoice_items" (
   "bill_ahead_resolved_at" timestamp DEFAULT NULL,
   "stripe_invoice_id" varchar(765) DEFAULT NULL,
   "is_vat" int2 NOT NULL DEFAULT '0',
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'invoice_items';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('invoice_items', 'id', 'int4(11)', 1),
@@ -639,8 +639,8 @@ CREATE TABLE "zerofill_table" (
   "value_double" float8 DEFAULT NULL,
   "name" varchar(768) DEFAULT NULL,
   "value_small_int" int4 DEFAULT NULL,
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'zerofill_table';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('zerofill_table', 'id', 'int4(11)', 1),

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -5,10 +5,11 @@ module FlydataCore
 module TableDef
 
 describe RedshiftTableDef do
+  let(:subject_object) { described_class }
   let(:table_name) { 'test_table' }
 
   describe '.from_flydata_tabledef' do
-    subject { described_class.from_flydata_tabledef(flydata_tabledef, option) }
+    subject { subject_object.from_flydata_tabledef(flydata_tabledef, option) }
 
     let(:value_column_hash) { { column: "value", type: "text" } }
     let(:value_column) { value_column_hash }
@@ -33,8 +34,8 @@ CREATE TABLE #{schema_prefix}"test_table" (
   "id" int4 NOT NULL,
   "age" int8,
   "value" varchar(max),
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
 EOT
     let(:flydata_ctl_update) { <<EOT.strip }
 DELETE FROM #{schema_prefix}"flydata_ctl_columns" WHERE table_name = 'test_table';
@@ -126,7 +127,7 @@ EOT
   end
 
   describe '.column_def_sql' do
-    subject { described_class.column_def_sql(column, opt) }
+    subject { subject_object.column_def_sql(column, opt) }
     let(:column) { {} }
     let(:opt) { {} }
 
@@ -524,7 +525,7 @@ EOT
   end
 
   describe '.flydata_ctl_columns_sql' do
-    subject { described_class.flydata_ctl_columns_sql(flydata_tabledef, schema_name) }
+    subject { subject_object.flydata_ctl_columns_sql(flydata_tabledef, schema_name) }
     let(:flydata_tabledef) { {
         table_name: table_name,
         columns: [
@@ -572,7 +573,7 @@ EOS
 
   describe '.parse_timestamp' do
     let(:value) { nil }
-    subject { described_class.parse_timestamp(value) }
+    subject { subject_object.parse_timestamp(value) }
 
     context 'with w3c format value' do
       context 'with sec' do
@@ -721,7 +722,7 @@ EOS
 
   describe '.parse_date' do
     let(:value) { nil }
-    subject { described_class.parse_date(value) }
+    subject { subject_object.parse_date(value) }
 
     context 'with year values' do
       context 'with value string 0' do
@@ -758,6 +759,26 @@ EOS
         let(:value){ '69' }
         it { is_expected.to eq('2069-01-01') }
       end
+
+      context 'with value 259392-00-00' do
+        let(:value) { '259392-00-00' }
+        it { is_expected.to eq('259392-01-01') }
+      end
+
+      context 'with value 259392-00-04' do
+        let(:value) { '259392-00-04' }
+        it { is_expected.to eq('259392-01-04') }
+      end
+
+      context 'with value 0000-01-04' do
+        let(:value) { '0000-01-04' }
+        it { is_expected.to eq('0001-01-04') }
+      end
+
+      context 'with value 259392-40-04' do
+        let(:value) { '259392-40-04' }
+        it { expect{subject}.to raise_error(ArgumentError) }
+      end
     end
 
     context 'with date values' do
@@ -779,6 +800,45 @@ EOS
       it { is_expected.to eq(nil) }
     end
   end
+
+  shared_examples "expecting no underscore in front of numbers" do
+    # Redshift accepts a table/column name starting with numbers (e.g. 20grams) or even a number-only name (e.g. 12345)
+    context "with a key including numbers" do
+      let(:key) { "year2000" }
+      let(:expected_value) { key }
+      it { is_expected.to eq expected_value }
+    end
+
+    context "with a key only including numbers" do
+      let(:key) { "2000" }
+      let(:expected_value) { key }
+      it { is_expected.to eq expected_value }
+    end
+
+    context "with a key starting with numbers" do
+      let(:key) { "2000stars" }
+      let(:expected_value) { key }
+      it { is_expected.to eq expected_value }
+    end
+  end
+
+  describe ".convert_to_valid_name" do
+    subject { subject_object.convert_to_valid_name(key) }
+
+    it_behaves_like "expecting no underscore in front of numbers"
+  end
+
+  describe ".convert_to_valid_column_name" do
+    subject { subject_object.convert_to_valid_name(key) }
+
+    it_behaves_like "expecting no underscore in front of numbers"
+  end
+
+  describe ".convert_to_valid_table_name" do
+    subject { subject_object.convert_to_valid_name(key) }
+
+    it_behaves_like "expecting no underscore in front of numbers"
+  end
 end
 
 end

data/flydata.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.21 ruby lib
+# stub: flydata 0.3.22 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.21"
+  s.version = "0.3.22"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
@@ -181,7 +181,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.3"
+  s.rubygems_version = "2.2.2"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.21
+  version: 0.3.22
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -621,7 +621,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: FlyData Agent