flydata 0.3.16 → 0.3.17

data/flydata-core/spec/table_def/mysqldump_test_column_charset.dump

@@ -0,0 +1,45 @@
+-- MySQL dump 10.13  Distrib 5.6.21, for debian-linux-gnu (x86_64)
+--
+-- Host: ubertas    Database: mak_development
+-- ------------------------------------------------------
+-- Server version	5.6.21-1+deb.sury.org~precise+1-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `items`
+--
+
+DROP TABLE IF EXISTS `items`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `items` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(40) CHARACTER SET latin1 DEFAULT NULL,
+  `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+  `bin` binary(34) DEFAULT NULL,
+  `bin2` blob,
+  `varbin` varchar(34) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=57 DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2015-03-26 23:36:52

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -7,43 +7,47 @@ module TableDef
 describe RedshiftTableDef do
 
   describe '.from_flydata_tabledef' do
+    subject { described_class.from_flydata_tabledef(flydata_tabledef, option) }
+
+    let(:value_column_hash) { { column: "value", type: "text" } }
+    let(:value_column) { value_column_hash }
+    let(:value_type_body) { 'text' }
+    let(:value_type) { value_type_body }
     let(:flydata_tabledef) { {
         table_name: "test_table",
         columns: [
           { column: "id", type: "int4(11)", not_null: true, primary_key: true },
           { column: "age", type: "int4(11) unsigned" },
-          { column: "value", type: "text" },
+          value_column,
         ],
-        default_charset: "utf8;"} }
+        default_charset: "UTF_8"} }
     let(:option) { { flydata_ctl_table: false } }
 
-    subject { described_class.from_flydata_tabledef(flydata_tabledef, option) }
+    let(:schema_prefix) { "" }
 
-    context 'with simple flydatadef' do
-      it 'should return ddl' do
-        expect(subject).to eq( <<EOT.strip )
-DROP TABLE IF EXISTS "test_table";
-CREATE TABLE "test_table" (
+    let(:create_table_queries) { <<EOT.strip }
+DROP TABLE IF EXISTS #{schema_prefix}"test_table";
+CREATE TABLE #{schema_prefix}"test_table" (
   "id" int4 NOT NULL,
   "age" int8,
   "value" varchar(max),
   PRIMARY KEY (id)
 ) DISTKEY(id) SORTKEY(id);
-DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
-INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
+EOT
+    let(:flydata_ctl_update) { <<EOT.strip }
+DELETE FROM #{schema_prefix}"flydata_ctl_columns" WHERE table_name = 'test_table';
+INSERT INTO #{schema_prefix}"flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table', 'id', 'int4(11)', 1),
 ('test_table', 'age', 'int4(11) unsigned', 2),
-('test_table', 'value', 'text', 3);
+('test_table', 'value', '#{value_type}', 3);
+DELETE FROM #{schema_prefix}"flydata_ctl_tables" WHERE table_name = 'test_table';
+INSERT INTO #{schema_prefix}"flydata_ctl_tables" (table_name, attribute, value) VALUES
+('test_table', 'cs', 'UTF_8'),
+('test_table', 'revision', 1);
 EOT
-      end
-    end
-
-    context 'with flydata_ctl_columns option true' do
-      let(:option) { { flydata_ctl_columns: true } }
 
-      it 'should return ddl including flydata_ctl_columns creation' do
-        expect(subject).to eq( <<EOT.strip )
-CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
+    let(:flydata_ctl_create) { <<EOT.strip }
+CREATE TABLE IF NOT EXISTS #{schema_prefix}"flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -52,92 +56,70 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE IF EXISTS "test_table";
-CREATE TABLE "test_table" (
-  "id" int4 NOT NULL,
-  "age" int8,
-  "value" varchar(max),
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
-DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
-INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
-('test_table', 'id', 'int4(11)', 1),
-('test_table', 'age', 'int4(11) unsigned', 2),
-('test_table', 'value', 'text', 3);
+CREATE TABLE IF NOT EXISTS #{schema_prefix}"flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
+EOT
+
+    let(:comment_subquery) { <<EOT.strip }
+COMMENT ON COLUMN #{schema_prefix}"test_table"."value"
+  IS 'helloworld';
+EOT
+
+    let(:create_schema_query) { <<EOT.strip }
+CREATE SCHEMA IF NOT EXISTS "test_schema";
 EOT
+
+    context 'with simple flydatadef' do
+      it 'should return ddl' do
+        expect(subject).to eq "#{create_table_queries}\n#{flydata_ctl_update}"
+      end
+    end
+
+    context 'with flydata_ctl_columns option true' do
+      let(:option) { { flydata_ctl_columns: true } }
+
+      it 'should return ddl including flydata_ctl_columns creation' do
+        expect(subject).to eq "#{flydata_ctl_create}\n#{create_table_queries}\n#{flydata_ctl_update}"
       end
     end
 
     context 'with flydatadef with comment' do
-      let(:flydata_tabledef) { {
-        table_name: "test_table",
-        columns: [
-          { column: "id", type: "int4(11)", not_null: true, primary_key: true },
-          { column: "value", type: "text" },
-          { column: "cmnt", type: "text", comment: "helloworld" },
-        ],
-        default_charset: "utf8;"} }
+      let(:value_column) { value_column_hash.merge(comment: "helloworld") }
 
       it 'should add comment creation ddl' do
-        expect(subject).to eq( <<EOT.strip )
-DROP TABLE IF EXISTS "test_table";
-CREATE TABLE "test_table" (
-  "id" int4 NOT NULL,
-  "value" varchar(max),
-  "cmnt" varchar(max),
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
-COMMENT ON COLUMN "test_table"."cmnt"
-  IS 'helloworld';
-DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
-INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
-('test_table', 'id', 'int4(11)', 1),
-('test_table', 'value', 'text', 2),
-('test_table', 'cmnt', 'text', 3);
-EOT
+        expect(subject).to eq "#{create_table_queries}\n#{comment_subquery}\n#{flydata_ctl_update}"
       end
     end
 
     context 'with schema_name' do
-      let(:flydata_tabledef) { {
-        table_name: "test_table",
-        columns: [
-          { column: "id", type: "int4(11)", not_null: true, primary_key: true },
-          { column: "value", type: "text" },
-          { column: "cmnt", type: "text", comment: "helloworld" },
-        ],
-        default_charset: "utf8;"} }
+      let(:value_column) { value_column_hash.merge(comment: "helloworld") }
       let(:option) { { flydata_ctl_table: true, schema_name: 'test_schema' } }
+      let(:schema_prefix) { '"test_schema".' }
 
       it 'should preappend schema name to table name' do
-        expect(subject).to eq( <<EOT.strip )
-CREATE SCHEMA "test_schema";
-CREATE TABLE IF NOT EXISTS "test_schema"."flydata_ctl_columns"(
-  id integer NOT NULL IDENTITY(1,1),
-  table_name varchar(128) NOT NULL,
-  column_name varchar(128) NOT NULL,
-  src_data_type varchar(1024) NOT NULL,
-  revision int NOT NULL DEFAULT 1,
-  ordinal_position int NOT NULL,
-  PRIMARY KEY(id)
-) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE IF EXISTS "test_schema"."test_table";
-CREATE TABLE "test_schema"."test_table" (
-  "id" int4 NOT NULL,
-  "value" varchar(max),
-  "cmnt" varchar(max),
-  PRIMARY KEY (id)
-) DISTKEY(id) SORTKEY(id);
-COMMENT ON COLUMN "test_schema"."test_table"."cmnt"
-  IS 'helloworld';
-DELETE FROM "test_schema"."flydata_ctl_columns" WHERE table_name = 'test_table';
-INSERT INTO "test_schema"."flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
-('test_table', 'id', 'int4(11)', 1),
-('test_table', 'value', 'text', 2),
-('test_table', 'cmnt', 'text', 3);
+        expect(subject).to eq <<EOT.strip
+#{create_schema_query}
+#{flydata_ctl_create}
+#{create_table_queries}
+#{comment_subquery}
+#{flydata_ctl_update}
 EOT
       end
     end
+
+    context 'with column charset' do
+      let(:value_column) { value_column_hash.merge(charset: "ISO_8859_1") }
+      let(:value_type) { "#{value_type_body} cs:ISO_8859_1" }
+      it 'should return ddl' do
+        expect(subject).to eq "#{create_table_queries}\n#{flydata_ctl_update}"
+      end
+    end
   end
 
   describe '.column_def_sql' do

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.16 ruby lib
+# stub: flydata 0.3.17 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.16"
+  s.version = "0.3.17"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-04-21"
+  s.date = "2015-04-22"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -47,6 +47,7 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb",
     "flydata-core/lib/flydata-core/fluent/config_helper.rb",
     "flydata-core/lib/flydata-core/logger.rb",
+    "flydata-core/lib/flydata-core/record/record.rb",
     "flydata-core/lib/flydata-core/table_def.rb",
     "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
     "flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
@@ -58,6 +59,7 @@ Gem::Specification.new do |s|
     "flydata-core/spec/table_def/mysql_table_def_spec.rb",
     "flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb",
     "flydata-core/spec/table_def/mysqldump_test_bit_table.dump",
+    "flydata-core/spec/table_def/mysqldump_test_column_charset.dump",
     "flydata-core/spec/table_def/mysqldump_test_foreign_key.dump",
     "flydata-core/spec/table_def/mysqldump_test_table_all.dump",
     "flydata-core/spec/table_def/mysqldump_test_table_column_comment.dump",
@@ -102,7 +104,6 @@ Gem::Specification.new do |s|
     "lib/flydata/fluent-plugins/idle_event_detector.rb",
     "lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb",
     "lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb",
-    "lib/flydata/fluent-plugins/mysql/binlog_position.rb",
     "lib/flydata/fluent-plugins/mysql/binlog_position_file.rb",
     "lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb",
     "lib/flydata/fluent-plugins/mysql/binlog_query_handler.rb",
@@ -121,6 +122,9 @@ Gem::Specification.new do |s|
     "lib/flydata/heroku/configuration_methods.rb",
     "lib/flydata/heroku/instance_methods.rb",
     "lib/flydata/log_monitor.rb",
+    "lib/flydata/mysql/binlog_position.rb",
+    "lib/flydata/mysql/mysql_util.rb",
+    "lib/flydata/mysql/table_ddl.rb",
     "lib/flydata/output/forwarder.rb",
     "lib/flydata/parser/mysql/dump_parser.rb",
     "lib/flydata/parser/mysql/mysql_alter_table.treetop",
@@ -129,7 +133,6 @@ Gem::Specification.new do |s|
     "lib/flydata/proxy.rb",
     "lib/flydata/sync_file_manager.rb",
     "lib/flydata/util/encryptor.rb",
-    "lib/flydata/util/mysql_util.rb",
     "spec/fluent_plugins_spec_helper.rb",
     "spec/fly_data_model_spec.rb",
     "spec/flydata/api/data_entry_spec.rb",
@@ -155,26 +158,28 @@ Gem::Specification.new do |s|
     "spec/flydata/fluent-plugins/idle_event_detector_spec.rb",
     "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
     "spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb",
-    "spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
     "spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb",
     "spec/flydata/fluent-plugins/mysql/ddl_query_handler_spec.rb",
+    "spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb",
     "spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb",
     "spec/flydata/fluent-plugins/mysql/table_meta_spec.rb",
     "spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb",
     "spec/flydata/heroku_spec.rb",
+    "spec/flydata/mysql/binlog_position_spec.rb",
+    "spec/flydata/mysql/mysql_util_spec.rb",
+    "spec/flydata/mysql/table_ddl_spec.rb",
     "spec/flydata/output/forwarder_spec.rb",
     "spec/flydata/parser/mysql/alter_table_parser_spec.rb",
     "spec/flydata/parser/mysql/dump_parser_spec.rb",
     "spec/flydata/sync_file_manager_spec.rb",
     "spec/flydata/util/encryptor_spec.rb",
-    "spec/flydata/util/mysql_util_spec.rb",
     "spec/flydata_spec.rb",
     "spec/spec_helper.rb",
     "tmpl/redshift_mysql_data_entry.conf.tmpl"
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.6"
+  s.rubygems_version = "2.4.3"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/setup.rb

@@ -40,7 +40,7 @@ What's next?
     - To manage the FlyData Agent, use the 'flydata' command (type 'flydata' for
       help)
     - If you encounter an issue,
-        please check our documentation (https://www.flydata.com/docs/) or
+        please check our documentation (https://www.flydata.com/resources/) or
         contact our customer support team (support@flydata.com)
 
 Thank you for using FlyData!
@@ -67,13 +67,13 @@ What's next?
 3. Start Sync
     Run the following command on your server. The command will start synchronizing data between MySQL and Redshift!
 
-    $ flydata sync
+    $ flydata start
 
       EOM
 
       NO_DE_CANCEL_MESSAGE_TEMPLATE = <<-EOM
-FlyData Agent has been installed on your computer.
-However, you need to create at least a data entry before start using FlyData.
+FlyData Agent has been installed on your server successfully.
+However, you need to create at least a data entry before you can start using FlyData.
 
 What's next?
 

data/lib/flydata/command/sync.rb

@@ -1,6 +1,5 @@
 require 'fiber'
 require 'msgpack'
-require 'open3'
 require 'mysql2'
 require 'rest_client'
 require 'sys/filesystem'
@@ -14,8 +13,9 @@ require 'flydata/output/forwarder'
 require 'flydata/parser/mysql/dump_parser'
 require 'flydata/preference/data_entry_preference'
 require 'flydata/sync_file_manager'
-require 'flydata/util/mysql_util'
+require 'flydata/mysql/mysql_util'
 require 'flydata-core/table_def'
+require 'flydata/mysql/table_ddl'
 #require 'ruby-prof'
 
 module Flydata
@@ -534,6 +534,7 @@ EOM
             Proc.new { |mysql_table, values_set|
               mysql_table_name = mysql_table.table_name
               records = values_set.collect do |values|
+                values = convert_to_flydata_values(mysql_table, values)
                 json = generate_json(mysql_table, values)
                 {table_name: mysql_table_name, log: json}
               end
@@ -673,6 +674,11 @@ EOM
         sync_fm.close
       end
 
+      def convert_to_flydata_values(mysql_table, values)
+        types = mysql_table.columns.each_value.collect{|col_attrs| col_attrs[:format_type]}
+        types.size.times.collect{|i| FlydataCore::TableDef::MysqlTableDef.convert_to_flydata_value(values[i], types[i]) }
+      end
+
       def generate_json(mysql_table, values)
         h = {}
         mysql_table.columns.each_key.with_index do |k, i|
@@ -739,33 +745,16 @@ EOM
           raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file"
         end
 
-        command = Util::MysqlUtil.generate_mysql_ddl_dump_cmd(mp.merge(tables: tables))
-
-        Open3.popen3(command) do |stdin, stdout, stderr|
-          stdin.close
-          stdout.set_encoding("utf-8") # mysqldump output must be in UTF-8
-          create_flydata_ctl_table = @full_initial_sync
-          while !stdout.eof?
-            begin
-              mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, skip_primary_key_check: opts.skip_primary_key_check?)
-              if mysql_tabledef.nil?
-                # stream had no more create table definition
-                break
-              end
-              flydata_tabledef = mysql_tabledef.to_flydata_tabledef
-              puts FlydataCore::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
-            rescue FlydataCore::TableDefError=> e
-              error_list << e.err_hash
-              next
-            end
-            create_flydata_ctl_table = false
-          end
-          errors = ""
-          while !stderr.eof?
-            line = stderr.gets.gsub('mysqldump: ', '')
-            errors << line unless /Warning: Using a password on the command line interface can be insecure./ === line
+        create_flydata_ctl_table = true
+        option = {skip_parimay_key_check: opts.skip_primary_key_check?}.merge(mp)
+        Mysql::MysqlUtil.each_mysql_tabledef(tables, option) do |mysql_tabledef, error|
+          if error
+            error_list << error.err_hash
+            next
           end
-          raise errors unless errors.empty?
+          flydata_tabledef = mysql_tabledef.to_flydata_tabledef
+          puts FlydataCore::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
+          create_flydata_ctl_table = false
         end
         unless error_list.empty?
           log_error_stderr("We have noticed the following error(s):")
@@ -781,7 +770,7 @@ EOM
         tables_without_error = tables - error_list.inject([]){|arr, err| arr << err[:table] if err[:table]}
 
         sync_fm = create_sync_file_manager(de)
-        sync_fm.mark_generated_tables(tables_without_error)
+        sync_fm.save_generated_ddl(tables_without_error, Mysql::TableDdl::VERSION)
         sync_fm.close
       end