flydata 0.1.1 → 0.1.3

data/Gemfile

@@ -6,7 +6,7 @@ gem "i18n", "~> 0.6.5"
 gem "activesupport", "~> 4.0.0"
 gem "json", "~> 1.8.0"
 gem "highline", "~> 1.6.19"
-gem "fluentd", "0.10.35"
+gem "fluentd", "0.10.46"
 gem "ruby-binlog", ">= 1.0.0"
 gem "fluent-plugin-mysql-binlog", "~> 0.0.2"
 gem "mysql2", "~> 0.3.11"

data/Gemfile.lock

@@ -21,8 +21,7 @@ GEM
     arel (4.0.2)
     atomic (1.1.14)
     builder (3.1.4)
-    cool.io (1.1.1)
-      iobuffer (>= 1.0.0)
+    cool.io (1.2.3)
     diff-lcs (1.2.4)
     faraday (0.8.8)
       multipart-post (~> 1.2.0)
@@ -30,11 +29,12 @@ GEM
       activesupport
       fluentd
       kodama
-    fluentd (0.10.35)
-      cool.io (~> 1.1.0)
-      http_parser.rb (~> 0.5.1)
+    fluentd (0.10.46)
+      cool.io (>= 1.1.1, < 2.0.0, != 1.2.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
       json (>= 1.4.3)
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      sigdump (~> 0.2.2)
       yajl-ruby (~> 1.0)
     git (1.2.6)
     github_api (0.10.1)
@@ -46,10 +46,9 @@ GEM
       oauth2
     hashie (2.0.5)
     highline (1.6.19)
-    http_parser.rb (0.5.3)
+    http_parser.rb (0.6.0)
     httpauth (0.2.0)
     i18n (0.6.5)
-    iobuffer (1.1.2)
     jeweler (1.8.8)
       builder
       bundler (~> 1.0)
@@ -59,14 +58,14 @@ GEM
       nokogiri (= 1.5.10)
       rake
       rdoc
-    json (1.8.0)
+    json (1.8.1)
     jwt (0.1.8)
       multi_json (>= 1.5)
     kodama (0.1.1)
       ruby-binlog (>= 0.1.9)
     mime-types (1.25)
     minitest (4.7.5)
-    msgpack (0.5.6)
+    msgpack (0.5.8)
     multi_json (1.8.2)
     multi_xml (0.5.5)
     multipart-post (1.2.0)
@@ -97,12 +96,13 @@ GEM
     rspec-mocks (2.14.3)
     ruby-binlog (1.0.0)
     ruby-prof (0.14.2)
+    sigdump (0.2.2)
     sqlite3 (1.3.8)
     thread_safe (0.1.3)
       atomic
     timecop (0.7.1)
     tzinfo (0.3.38)
-    yajl-ruby (1.1.0)
+    yajl-ruby (1.2.0)
 
 PLATFORMS
   ruby
@@ -113,7 +113,7 @@ DEPENDENCIES
   activesupport (~> 4.0.0)
   bundler
   fluent-plugin-mysql-binlog (~> 0.0.2)
-  fluentd (= 0.10.35)
+  fluentd (= 0.10.46)
   highline (~> 1.6.19)
   i18n (~> 0.6.5)
   jeweler

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.3

data/bin/fdmysqldump

@@ -7,6 +7,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'flydata/table_def'
 
+# Deprecation warning
+$stderr.puts "WARNING: Use 'flydata sync:generate_table_ddl' command instead"
+
 if `which mysqldump`.empty?
   abort "mysqldump is not installed."
 end

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.1.1"
+  s.version = "0.1.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-05-01"
+  s.date = "2014-05-10"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
   s.executables = ["fdmysqldump", "flydata"]
@@ -86,7 +86,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://flydata.co/"
   s.licenses = ["All right reserved."]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.23"
+  s.rubygems_version = "1.8.24"
   s.summary = "FlyData CLI"
 
   if s.respond_to? :specification_version then
@@ -98,7 +98,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<activesupport>, ["~> 4.0.0"])
       s.add_runtime_dependency(%q<json>, ["~> 1.8.0"])
       s.add_runtime_dependency(%q<highline>, ["~> 1.6.19"])
-      s.add_runtime_dependency(%q<fluentd>, ["= 0.10.35"])
+      s.add_runtime_dependency(%q<fluentd>, ["= 0.10.46"])
       s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.0"])
       s.add_runtime_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
       s.add_runtime_dependency(%q<mysql2>, ["~> 0.3.11"])
@@ -117,7 +117,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<activesupport>, ["~> 4.0.0"])
       s.add_dependency(%q<json>, ["~> 1.8.0"])
       s.add_dependency(%q<highline>, ["~> 1.6.19"])
-      s.add_dependency(%q<fluentd>, ["= 0.10.35"])
+      s.add_dependency(%q<fluentd>, ["= 0.10.46"])
       s.add_dependency(%q<ruby-binlog>, [">= 1.0.0"])
       s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
       s.add_dependency(%q<mysql2>, ["~> 0.3.11"])
@@ -137,7 +137,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<activesupport>, ["~> 4.0.0"])
     s.add_dependency(%q<json>, ["~> 1.8.0"])
     s.add_dependency(%q<highline>, ["~> 1.6.19"])
-    s.add_dependency(%q<fluentd>, ["= 0.10.35"])
+    s.add_dependency(%q<fluentd>, ["= 0.10.46"])
     s.add_dependency(%q<ruby-binlog>, [">= 1.0.0"])
     s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
     s.add_dependency(%q<mysql2>, ["~> 0.3.11"])

data/lib/flydata/command/sync.rb

@@ -1,6 +1,7 @@
 require 'msgpack'
 require 'open3'
 require 'flydata/sync_file_manager'
+require 'flydata/table_def'
 #require 'ruby-prof'
 
 module Flydata
@@ -68,12 +69,54 @@ module Flydata
         puts "Run 'flydata start' to start continuous sync."
       end
 
+      def generate_table_ddl
+        de = retrieve_data_entries.first
+        raise "There are no data entry." unless de
+        case de['type']
+        when 'RedshiftMysqlDataEntry'
+          do_generate_table_ddl(de)
+        else
+          raise "No supported data entry. Only mysql-redshift sync is supported."
+        end
+      end
+
       private
 
       def do_check(de)
         flydata.data_entry.buffer_stat(de['id'], env_mode)
       end
 
+      DDL_DUMP_CMD_TEMPLATE = "mysqldump -d -h %s -P %s -u %s %s %s %s"
+      def do_generate_table_ddl(de)
+        if `which mysqldump`.empty?
+          raise "mysqldump is not installed.  mysqldump is required to run the command"
+        end
+
+        mp = de['mysql_data_entry_preference']
+        params = []
+        if mp['host'] then params << mp['host'] else raise "MySQL `host` is not defined in the data entry" end
+        params << (mp['port'] or '3306')
+        if mp['username'] then params << mp['username'] else raise "MySQL `username` is not defined is not defined in the data entry" end
+        params << (mp['password'] ? "-p#{mp['password']}" : "")
+        if mp['database'] then params << mp['database'] else raise "`database` is not defined in the data entry" end
+        if mp['tables'] then params << mp['tables'].gsub(/,/, ' ') else raise "`tables` is not defined in the data entry" end
+        command = DDL_DUMP_CMD_TEMPLATE % params
+
+        IO.popen(command, 'r') do |io|
+          create_flydata_ctl_table = true
+          while !io.eof?
+            mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(io)
+            if mysql_tabledef.nil?
+              # stream had no more create table definition
+              break
+            end
+            flydata_tabledef = mysql_tabledef.to_flydata_tabledef
+            puts Flydata::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table)
+            create_flydata_ctl_table = false
+          end
+        end
+      end
+
       def sync_mysql_to_redshift(de)
         dp = flydata.data_port.get
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)

data/spec/flydata/command/sync_spec.rb

@@ -5,6 +5,126 @@ require 'socket'
 module Flydata
   module Command
     describe Sync do
+      let(:default_mysqldump_dir) do
+        File.join('/tmp', "sync_dump_#{Time.now.to_i}")
+      end
+      let(:default_data_entry) do
+        {"id"=>93,
+        "name"=>"flydata_sync_mysql",
+        "data_port_id"=>52,
+        "display_name"=>"flydata_sync_mysql",
+        "enabled"=>true,
+        "heroku_log_type"=>nil,
+        "heroku_resource_id"=>nil,
+        "log_deletion"=>nil,
+        "log_file_delimiter"=>nil,
+        "log_file_type"=>nil,
+        "log_path"=>nil,
+        "redshift_schema_name"=>"",
+        "redshift_table_name"=>nil,
+        "created_at"=>"2014-01-22T18:58:43Z",
+        "updated_at"=>"2014-01-30T02:42:26Z",
+        "type"=>"RedshiftMysqlDataEntry",
+        "tag_name"=>"flydata.a458c641_dp52.flydata_mysql",
+        "tag_name_dev"=>"flydata.a458c641_dp52.flydata_mysql.dev",
+        "data_port_key"=>"a458c641",
+        "mysql_data_entry_preference" =>
+          { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
+            "password"=>"welcome", "database"=>"sync_test", "tables"=>"table1, table2",
+            "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
+            "data_servers"=>"localhost:9905" }
+         }
+      end
+
+      after :each do
+        if Dir.exists?(default_mysqldump_dir)
+          Dir.delete(default_mysqldump_dir) rescue nil
+        end
+        if File.exists?(default_mysqldump_dir)
+          File.delete(default_mysqldump_dir) rescue nil
+        end
+      end
+
+      describe '#do_generate_table_ddl' do
+        subject { Sync.new }
+        context 'with full options' do
+          it 'issues mysqldump command with expected parameters' do
+            expect(IO).to receive(:popen).with(
+             'mysqldump -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
+            subject.send(:do_generate_table_ddl, default_data_entry)
+          end
+        end
+        context 'without_host' do
+          before do
+            default_data_entry['mysql_data_entry_preference'].delete('host')
+          end
+          it "throws an error" do
+            expect {
+              subject.send(:do_generate_table_ddl, default_data_entry)
+            }.to raise_error
+          end
+        end
+        context 'without_port' do
+          before do
+            default_data_entry['mysql_data_entry_preference'].delete('port')
+          end
+          it "uses the default port" do
+            expect(IO).to receive(:popen).with(
+             'mysqldump -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
+            subject.send(:do_generate_table_ddl, default_data_entry)
+          end
+        end
+        context 'with_port_override' do
+          before do
+            default_data_entry['mysql_data_entry_preference']['port'] = 1234
+          end
+          it "uses the specified port" do
+            expect(IO).to receive(:popen).with(
+             'mysqldump -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
+            subject.send(:do_generate_table_ddl, default_data_entry)
+          end
+        end
+        context 'without_username' do
+          before do
+            default_data_entry['mysql_data_entry_preference'].delete('username')
+          end
+          it "throws an error" do
+            expect {
+              subject.send(:do_generate_table_ddl, default_data_entry)
+            }.to raise_error
+          end
+        end
+        context 'without_password' do
+          before do
+            default_data_entry['mysql_data_entry_preference'].delete('password')
+          end
+          it "call mysqldump without -p option" do
+            expect(IO).to receive(:popen).with(
+             'mysqldump -d -h localhost -P 3306 -u masashi  sync_test table1  table2', 'r').and_call_original
+            subject.send(:do_generate_table_ddl, default_data_entry)
+          end
+        end
+        context 'without_database' do
+          before do
+            default_data_entry['mysql_data_entry_preference'].delete('database')
+          end
+          it "throws an error" do
+            expect {
+              subject.send(:do_generate_table_ddl, default_data_entry)
+            }.to raise_error
+          end
+        end
+        context 'without_tables' do
+          before do
+            default_data_entry['mysql_data_entry_preference'].delete('tables')
+          end
+          it "throws an error" do
+            expect {
+              subject.send(:do_generate_table_ddl, default_data_entry)
+            }.to raise_error
+          end
+        end
+      end
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-01 00:00:00.000000000 Z
+date: 2014-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -98,7 +98,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.10.35
+        version: 0.10.46
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.10.35
+        version: 0.10.46
 - !ruby/object:Gem::Dependency
   name: ruby-binlog
   requirement: !ruby/object:Gem::Requirement
@@ -390,7 +390,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2999852327908013183
+      hash: -1932242309607756156
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -399,7 +399,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: FlyData CLI