flydata 0.3.13 → 0.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 317626ac900b0a0adf96c84648e554f6d2e7436a
-  data.tar.gz: e54dcf2b46068be7c27e3d6ce63c338d111e0c20
+  metadata.gz: de14c5ba0b4b0b3c1b98e93f9e41c59a32db18fd
+  data.tar.gz: 685e70c882291fe387d7aaf14657a879b8c1d73e
 SHA512:
-  metadata.gz: 935c6cc893c91fba078e92306b88a06697195c3a21b5241f0d08936a9cfb8acdff7c9eb32febbdc5c77a04b5dbe9627c757cf123ceee0420ae29784e4e10e1b4
-  data.tar.gz: 593a4ade45a37713981a11c5d98456eaaa04a592870cab4b2aff48a162e17d36c4780a138df5c38636d07bc9c3879d3ddde02e6331c94be916c62cc9b1d2f58a
+  metadata.gz: c735cbf41f4e9448175bb38c5dac9b5e5365e7a8f9710f86e7399feab2eecd5e018165f2224e8749e8c7169941b33bc1a01e122d7eb1d510056010508c0a793a
+  data.tar.gz: efc559f6b9536c45078f63b1416fc5da759bbd99249176b51dc15e7ec4016a5dfb5942262db0e4a87d13ba843d90ae00a91a9b731d670881c5467b5b642fac87

data/Gemfile

@@ -7,14 +7,13 @@ gem "activesupport", '~> 4.0.0'
 gem "json", '~> 1.8', '>= 1.8.0'
 gem "highline", '~> 1.6', '>= 1.6.19'
 gem "fluentd", "0.10.46"
-gem "ruby-binlog", '~> 1.0', '>= 1.0.2'
 gem "fluent-plugin-mysql-binlog", '~> 0.0', '>= 0.0.2'
-gem "mysql2", '~> 0.3', '>= 0.3.11'
+gem "mysql2", '~> 0.3', '>= 0.3.17'
 gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
 gem "io-console", '~> 0.4.2', '>= 0.4.2'
-gem "kodama", '~> 0.1.2', '>= 0.1.2'
+gem "kodama", '~> 0.1.2', '>= 0.1.3'
 
 group :development do
   gem "jeweler", '~> 1.8', '>= 1.8.8'

data/Gemfile.lock

@@ -61,8 +61,8 @@ GEM
       rdoc
     json (1.8.1)
     jwt (1.0.0)
-    kodama (0.1.2)
-      ruby-binlog (>= 0.1.9)
+    kodama (0.1.3)
+      ruby-binlog (~> 1.0, >= 1.0.4)
     method_source (0.8.2)
     mime-types (2.3)
     minitest (4.7.5)
@@ -70,7 +70,7 @@ GEM
     multi_json (1.10.1)
     multi_xml (0.5.5)
     multipart-post (1.2.0)
-    mysql2 (0.3.16)
+    mysql2 (0.3.18)
     netrc (0.7.7)
     nokogiri (1.5.10)
     oauth2 (1.0.0)
@@ -105,7 +105,7 @@ GEM
     rspec-mocks (3.0.3)
       rspec-support (~> 3.0.0)
     rspec-support (3.0.3)
-    ruby-binlog (1.0.2)
+    ruby-binlog (1.0.4)
     ruby-prof (0.15.1)
     sigdump (0.2.2)
     slop (3.6.0)
@@ -133,13 +133,12 @@ DEPENDENCIES
   io-console (~> 0.4.2, >= 0.4.2)
   jeweler (~> 1.8, >= 1.8.8)
   json (~> 1.8, >= 1.8.0)
-  kodama (~> 0.1.2, >= 0.1.2)
-  mysql2 (~> 0.3, >= 0.3.11)
+  kodama (~> 0.1.2, >= 0.1.3)
+  mysql2 (~> 0.3, >= 0.3.17)
   protected_attributes (~> 1.0, >= 1.0.8)
   pry
   rest-client (~> 1.6, >= 1.6.7)
   rspec (~> 3.0)
-  ruby-binlog (~> 1.0, >= 1.0.2)
   ruby-prof (~> 0.15, >= 0.15.1)
   slop (~> 3.4, >= 3.4.6)
   sqlite3 (~> 1.3, >= 1.3.9)

data/VERSION

@@ -1 +1 @@
-0.3.13
+0.3.14

data/flydata-core/lib/flydata-core/config/user_maintenance.rb

@@ -25,6 +25,9 @@ module FlydataCore
     #     - discard_[data-port-id]_[data-entry-id]_[(schema-name.)table-name]
     #
     class UserMaintenance
+
+      SUPPORTED_MODES = [:maintenance, :discard]
+
       def initialize(base_dir_path)
         @base_dir_path = base_dir_path
       end
@@ -48,6 +51,18 @@ module FlydataCore
         mode_file_path(:discard, params)
       end
 
+      # /mnt/flydata/system/user_maintenance/[dp-id]/[mode]_[data-port-id]_[data-entry-id]_[table-name]
+      def mode_file_path(mode, params)
+        validate_mode(mode)
+        validate_params(params)
+        File.join(dp_mode_dir_path(params), mode_file_name(mode, params))
+      end
+
+      # /mnt/flydata/system/user_maintenance/[dp-id]
+      def dp_mode_dir_path(params)
+        File.join(@base_dir_path, params[:data_port_id].to_s)
+      end
+
       # For fluentd plugin and copy handler
 
       def maintenance_mode?(params)
@@ -70,6 +85,10 @@ module FlydataCore
 
       private
 
+      def validate_mode(mode)
+        raise ArgumentError.new("Unsupported mode:#{mode}") unless SUPPORTED_MODES.include?(mode)
+      end
+
       def validate_params(params)
         raise ArgumentError.new("params must be hash")  unless params.kind_of?(Hash)
         raise ArgumentError.new("params must have 'data_port_id' key")  unless params[:data_port_id]
@@ -90,17 +109,6 @@ module FlydataCore
         Dir.glob("#{de_mode_file_path}_*").any?{|fp| File.join(File.dirname(fp), File.basename(fp).downcase) == tn_mode_file_path}
       end
 
-      # /mnt/flydata/system/user_maintenance/[dp-id]/[mode]_[data-port-id]_[data-entry-id]_[table-name]
-      def mode_file_path(mode, params)
-        validate_params(params)
-        File.join(dp_mode_dir_path(mode, params), mode_file_name(mode, params))
-      end
-
-      # /mnt/flydata/system/user_maintenance/[dp-id]
-      def dp_mode_dir_path(mode, params)
-        File.join(@base_dir_path, params[:data_port_id].to_s)
-      end
-
       # [mode]_[data-port-id]_[data-entry-id]_[table-name]
       def mode_file_name(mode, params)
         return nil unless params[:data_port_id]

data/flydata-core/lib/flydata-core/fluent/config_helper.rb

@@ -0,0 +1,21 @@
+module FlydataCore
+  module Fluent
+    class ConfigHelper
+      ESCAPE_REGEXP = /\r|\n|\t|"|\\|/
+      ESCAPE_HASH_TABLE = {"\r" => "\\r", "\n" => "\\n", "\t" => "\\t", '"' => '\\"', "\\" => "\\\\"}
+
+      UNESCAPE_REGEXP = /\\r|\\n|\\t|\\"|\\\\|/
+      UNESCAPE_HASH_TABLE = {"\\r" => "\r", "\\n" => "\n", "\\t" => "\t", '\\"' => '"', "\\\\" => "\\"}
+
+      def self.escape_conf(str)
+        return "" if str.nil? || str.empty?
+        str.gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
+      end
+
+      def self.unescape_conf(str)
+        return "" if str.nil? || str.empty?
+        str.gsub(UNESCAPE_REGEXP, UNESCAPE_HASH_TABLE)
+      end
+    end
+  end
+end

data/flydata-core/spec/config/user_maintenance_spec.rb

@@ -68,6 +68,16 @@ module FlydataCore::Config
       end
     end
 
+    describe '#mode_file_path' do
+      let(:mode) { nil }
+      subject { user_maintenance.mode_file_path(mode, params) }
+
+      context 'with invalid mode' do
+        let(:mode) { :unknown }
+        it { expect{ subject }.to raise_error(ArgumentError, "Unsupported mode:unknown") }
+      end
+    end
+
     describe '#maintenance_mode?' do
       subject { user_maintenance.maintenance_mode?(params) }
 

data/flydata-core/spec/fluent/config_helper_spec.rb

@@ -0,0 +1,78 @@
+require_relative '../spec_helper'
+require 'flydata-core/fluent/config_helper'
+
+module FlydataCore::Fluent
+  describe ConfigHelper do
+    describe '.escape_conf' do
+
+      let(:input) { "" }
+      let(:output) { "" }
+      subject { ConfigHelper.escape_conf(input) }
+
+      shared_examples 'escape conf expectedly' do
+        it { is_expected.to eq(output) }
+      end
+
+      context 'when input contains escape characters' do
+        let(:input) { %Q|aa\nbb\r\ncc\tdd"ee\\ff| }
+        let(:output) { %Q|aa\\nbb\\r\\ncc\\tdd\\"ee\\\\ff| }
+
+        it_behaves_like 'escape conf expectedly'
+      end
+
+      context 'when input does not contain escape characters' do
+        let(:input) { %Q|aabbccddeeff| }
+        let(:output) { %Q|aabbccddeeff| }
+
+        it_behaves_like 'escape conf expectedly'
+      end
+
+      context 'when input is nil' do
+        let(:input) { nil }
+        it { is_expected.to eq("") }
+      end
+
+      context 'when input is empty' do
+        let(:input) { "" }
+        it { is_expected.to eq("") }
+      end
+    end
+
+    describe '.unescape_conf' do
+
+      let(:input) { "" }
+      let(:output) { "" }
+      subject { ConfigHelper.unescape_conf(input) }
+
+      shared_examples 'unescape conf expectedly' do
+        it { is_expected.to eq(output) }
+      end
+
+      context 'when input contains escaped characters' do
+        let(:input) { %Q|aa\\nbb\\r\\ncc\\tdd\\"ee\\\\ff| }
+        let(:output) { %Q|aa\nbb\r\ncc\tdd"ee\\ff| }
+
+        it_behaves_like 'unescape conf expectedly'
+      end
+
+      context 'when input does not contain escape characters' do
+        let(:input) { %Q|aabbccddeeff| }
+        let(:output) { %Q|aabbccddeeff| }
+
+        it_behaves_like 'unescape conf expectedly'
+      end
+
+      context 'when input is nil' do
+        let(:input) { nil }
+        it { is_expected.to eq("") }
+      end
+
+      context 'when input is empty' do
+        let(:input) { "" }
+        it { is_expected.to eq("") }
+      end
+    end
+  end
+end
+
+

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.13 ruby lib
+# stub: flydata 0.3.14 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.13"
+  s.version = "0.3.14"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-04-09"
+  s.date = "2015-04-19"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -45,12 +45,14 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/core_ext/object/prepend.rb",
     "flydata-core/lib/flydata-core/errors.rb",
     "flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb",
+    "flydata-core/lib/flydata-core/fluent/config_helper.rb",
     "flydata-core/lib/flydata-core/logger.rb",
     "flydata-core/lib/flydata-core/table_def.rb",
     "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
     "flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
     "flydata-core/lib/flydata-core/thread_context.rb",
     "flydata-core/spec/config/user_maintenance_spec.rb",
+    "flydata-core/spec/fluent/config_helper_spec.rb",
     "flydata-core/spec/logger_spec.rb",
     "flydata-core/spec/spec_helper.rb",
     "flydata-core/spec/table_def/mysql_table_def_spec.rb",
@@ -172,7 +174,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.3"
+  s.rubygems_version = "2.4.6"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then
@@ -185,14 +187,13 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<json>, [">= 1.8.0", "~> 1.8"])
       s.add_runtime_dependency(%q<highline>, [">= 1.6.19", "~> 1.6"])
       s.add_runtime_dependency(%q<fluentd>, ["= 0.10.46"])
-      s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.2", "~> 1.0"])
       s.add_runtime_dependency(%q<fluent-plugin-mysql-binlog>, [">= 0.0.2", "~> 0.0"])
-      s.add_runtime_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
+      s.add_runtime_dependency(%q<mysql2>, [">= 0.3.17", "~> 0.3"])
       s.add_runtime_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
       s.add_runtime_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_runtime_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_runtime_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-      s.add_runtime_dependency(%q<kodama>, [">= 0.1.2", "~> 0.1.2"])
+      s.add_runtime_dependency(%q<kodama>, [">= 0.1.3", "~> 0.1.2"])
       s.add_development_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_development_dependency(%q<rspec>, ["~> 3.0"])
       s.add_development_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -209,14 +210,13 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<json>, [">= 1.8.0", "~> 1.8"])
       s.add_dependency(%q<highline>, [">= 1.6.19", "~> 1.6"])
       s.add_dependency(%q<fluentd>, ["= 0.10.46"])
-      s.add_dependency(%q<ruby-binlog>, [">= 1.0.2", "~> 1.0"])
       s.add_dependency(%q<fluent-plugin-mysql-binlog>, [">= 0.0.2", "~> 0.0"])
-      s.add_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
+      s.add_dependency(%q<mysql2>, [">= 0.3.17", "~> 0.3"])
       s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
       s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-      s.add_dependency(%q<kodama>, [">= 0.1.2", "~> 0.1.2"])
+      s.add_dependency(%q<kodama>, [">= 0.1.3", "~> 0.1.2"])
       s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_dependency(%q<rspec>, ["~> 3.0"])
       s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -234,14 +234,13 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<json>, [">= 1.8.0", "~> 1.8"])
     s.add_dependency(%q<highline>, [">= 1.6.19", "~> 1.6"])
     s.add_dependency(%q<fluentd>, ["= 0.10.46"])
-    s.add_dependency(%q<ruby-binlog>, [">= 1.0.2", "~> 1.0"])
     s.add_dependency(%q<fluent-plugin-mysql-binlog>, [">= 0.0.2", "~> 0.0"])
-    s.add_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
+    s.add_dependency(%q<mysql2>, [">= 0.3.17", "~> 0.3"])
     s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
     s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
     s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
     s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-    s.add_dependency(%q<kodama>, [">= 0.1.2", "~> 0.1.2"])
+    s.add_dependency(%q<kodama>, [">= 0.1.3", "~> 0.1.2"])
     s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
     s.add_dependency(%q<rspec>, ["~> 3.0"])
     s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])

data/lib/flydata/command/base.rb

@@ -30,6 +30,18 @@ module Flydata
           end
         end
       end
+
+      def show_purpose_name
+        de = data_entry
+        log_info_stdout("Your current application name is '#{de['purpose_name']}'")
+      end
+
+      def data_entry
+        @de ||= retrieve_data_entries.first
+        raise "There are no data entries." unless @de
+        @de
+      end
+
       def register_crontab
         data_entries = retrieve_data_entries
         if data_entries.any?{|e| e['log_deletion']}

data/lib/flydata/command/status.rb

@@ -5,6 +5,7 @@ module Flydata
   module Command
     class Status < Base
       def run
+        show_purpose_name
         sender = Flydata::Command::Sender.new
         sender.status
       end

data/lib/flydata/command/sync.rb

@@ -137,6 +137,7 @@ EOS
       # Command: flydata sync:reset
       #  - Entry method
       def reset(*tables)
+        show_purpose_name
         # Flush client buffer
         msg = tables.empty? ? '' : " for these tables : #{tables.join(" ")}"
         return unless ask_yes_no("This resets the current sync#{msg}.  Are you sure?")
@@ -285,9 +286,10 @@ EOS
         de = data_entry
 
         unless @new_tables.empty?
+          show_purpose_name
           unsynced_table_message = "We've noticed that these tables have not been synced yet: #{@new_tables.join(", ")}\n"
           unless @ddl_tables.empty?
-            unsynced_table_message << 
+            unsynced_table_message <<
             "  WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.\n" +
             "    We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'\n" +
             "    to generate SQL to run on Redshift to create the correct tables\n" +
@@ -385,6 +387,10 @@ FlyData Sync will start synchronizing the following database tables
   database:     #{de['mysql_data_entry_preference']['database']}
   tables:       #{tables.join(", ")}#{data_servers}
 EOM
+
+        confirmation_text << <<-EOM if de['mysql_data_entry_preference']['ssl_ca']
+  ssl:          Yes
+EOM
         confirmation_text << <<-EOM if file_dump
   dump file:    #{fp}
 
@@ -884,11 +890,19 @@ Thank you for using FlyData!
         case de['type']
         when 'RedshiftMysqlDataEntry'
           mp = de['mysql_data_entry_preference']
+
           if mp['tables_append_only']
             mp['tables'] = (mp['tables'].split(",") + mp['tables_append_only'].split(",")).uniq
           else
             mp['tables'] = mp['tables'].split(",").uniq
           end
+
+          unless mp['ssl_ca_content'].to_s.strip.empty?
+            sync_fm = create_sync_file_manager(de)
+            sync_fm.save_ssl_ca(mp['ssl_ca_content'])
+            mp['ssl_ca'] = sync_fm.ssl_ca_path
+            mp['sslca'] = mp['ssl_ca']
+          end
         else
           raise SyncDataEntryError, "No supported data entry. Only mysql-redshift sync is supported."
         end

data/lib/flydata/compatibility_check.rb

@@ -78,13 +78,16 @@ module Flydata
 
     SELECT_QUERY_TMPLT = "SELECT %s"
     BINLOG_RETENTION_HOURS = 24
+    SELECT_TABLE_INFO_TMPLT = "SELECT table_name, table_type, engine FROM information_schema.tables WHERE table_schema = '%s' and table_name in(%s)"
 
     #def initialize(de_hash, dump_dir=nil)
     def initialize(dp_hash, de_hash, options={})
       super
-      @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
+      @db_opts = [:host, :port, :username, :password, :database, :ssl_ca].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
+      @db_opts[:sslca] = @db_opts[:ssl_ca] # for mysql2 gem
       @dump_dir = options[:dump_dir] || nil
       @backup_dir = options[:backup_dir] || nil
+      @tables = de_hash['tables']
     end
 
     def print_errors
@@ -202,6 +205,22 @@ module Flydata
       end
     end
 
+    # If table_type='VIEW' or engine='MEMORY', raise error.
+    def check_mysql_table_types
+      return if @tables.empty?
+      client = Mysql2::Client.new(@db_opts)
+      sel_query = SELECT_TABLE_INFO_TMPLT % [client.escape(@db_opts[:database]), @tables.collect{|t| "'#{client.escape(t)}'"}.join(", ")]
+      begin
+        invalid_tables = []
+        client.query(sel_query).each do |r|
+          invalid_tables.push(r['table_name']) if r['table_type'] == 'VIEW' || r['engine'] == 'MEMORY'
+        end
+        raise MysqlCompatibilityError, "FlyData does not support VIEW and MEMORY ENGINE table. Remove following tables from data entry: #{invalid_tables.join(", ")}" unless invalid_tables.empty?
+      ensure
+        client.close
+      end
+    end
+
     def run_mysql_retention_check(mysql_client)
       expire_logs_days_limit = BINLOG_RETENTION_HOURS / 24
       sel_query = SELECT_QUERY_TMPLT % '@@expire_logs_days'

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -16,6 +16,7 @@ require 'flydata/fluent-plugins/mysql/binlog_record_dispatcher'
 require 'flydata/fluent-plugins/mysql/context'
 require 'flydata/fluent-plugins/idle_event_detector'
 require 'flydata/fluent-plugins/mysql/table_meta'
+require 'flydata-core/fluent/config_helper'
 
 #Monkey-patch fluentd class (EngineClass) to support shutdown for input plugin.
 #This will be called when USR1 signal is received
@@ -53,6 +54,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
   config_param :initial_idle_interval, :integer, :default => 30
   config_param :continuous_idle_interval, :integer, :default => 600
   config_param :check_interval, :integer, :default => 5
+  config_param :ssl_ca_content, :string, :default => ''
 
   def configure(conf)
     super
@@ -65,9 +67,19 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     @sent_position_file_path = @sync_fm.sent_binlog_path(@position_file)
 
     load_custom_conf
+
+    # SSL configuration
+    unless @ssl_ca_content.to_s.strip.empty?
+      @ssl_ca_path = @sync_fm.ssl_ca_path(@position_file)
+      @sync_fm.save_ssl_ca(FlydataCore::Fluent::ConfigHelper.unescape_conf(@ssl_ca_content), @ssl_ca_path)
+    end
+
+    # Db access opts
+    db_opts = { host: @host, port: @port, username: @username, password: @password, database: @database, ssl_ca: @ssl_ca_path }
+
     $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
     $log.info "mysql client version: #{`mysql -V`}"
-    server_version = `echo 'select version();' | #{Flydata::Util::MysqlUtil.generate_mysql_cmd(host: @host, port: @port, username: @username, password: @password)} 2>/dev/null`
+    server_version = `echo 'select version();' | #{Flydata::Util::MysqlUtil.generate_mysql_cmd(db_opts)} 2>/dev/null`
     $log.info "mysql server version: #{server_version}"
 
     @tables = @tables.split(/,\s*/)
@@ -77,19 +89,20 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       @omit_events[table] = [:delete, :truncate_table]
     end
 
-    #Remove tables that do not have pos files
+    # Remove tables that do not have pos files
     new_tables = @sync_fm.get_new_table_list(@tables, "pos")
     @tables -= new_tables
     $log.info "Not watching these tables: #{new_tables.join(", ")}"
 
-    table_meta = Mysql::TableMeta.new(
-      mysql_url: mysql_url, database: @database, tables: @tables)
+    table_meta = Mysql::TableMeta.new(db_opts.merge(tables: @tables))
 
     table_revs = tables.inject({}) do |h, table_name|
       h[table_name] = @sync_fm.table_rev(table_name)
       h
     end
 
+
+    # Set context
     @context = Mysql::Context.new(
       database: @database, tables: @tables,
       tag: @tag, sync_fm: @sync_fm, omit_events: @omit_events,
@@ -134,6 +147,11 @@ EOS
     @context.table_meta.update
     start_kodama(mysql_url) do |c|
       c.binlog_position_file = @position_file
+      if @ssl_ca_path.to_s != '' && c.respond_to?(:ssl_ca=)
+        $log.info "SSL is enabled. (ssl_ca: #{@ssl_ca_path})"
+        c.ssl_ca = @ssl_ca_path
+      end
+
       if c.respond_to?(:sent_binlog_position_file=)
         $log.info "Sent position feature is enabled. sent_position_file:#{@sent_position_file_path}"
         c.sent_binlog_position_file = @sent_position_file_path

data/lib/flydata/fluent-plugins/mysql/table_meta.rb

@@ -3,9 +3,11 @@ require 'mysql2'
 module Mysql
   class TableMeta
     MANDATORY_OPTS = [
-      :mysql_url, :database, :tables
+      :host, :port, :username, :password,
+      :database, :tables,
     ]
     OPTIONAL_OPTS = [
+      :ssl_ca
     ]
 
     GET_TABLE_META_SQL = <<EOT
@@ -24,14 +26,17 @@ EOT
     def initialize(opts)
       missing_opts = MANDATORY_OPTS - opts.keys
       raise "Mandatory option(s) are missing: #{missing_opts.join(', ')}" unless (missing_opts.empty?)
-      @mysql_url = opts[:mysql_url]
+
+      @db_opts = [:host, :port, :username, :password, :database, :ssl_ca].inject({}) {|h, sym| h[sym] = opts[sym]; h}
+      @db_opts[:sslca] = @db_opts[:ssl_ca] # for mysql2 gem
+
       @database = opts[:database]
       @tables = opts[:tables]
       @table_meta = Hash.new{|h, k| h[k] = {}}
     end
 
     def update
-      conn = Mysql2::Client.new(@mysql_url)
+      conn = Mysql2::Client.new(@db_opts)
       sql = GET_TABLE_META_SQL % {
         database: @database, tables: @tables.collect{|t| "'#{t}'"}.join(',') }
       columns = conn.query(sql)

data/lib/flydata/fluent-plugins/preference.rb

@@ -76,6 +76,7 @@ module Fluent
         host: {},
         username: {},
         password: {encrypted: true},
+        ssl_ca_content: {},
       },
     }
 

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -7,7 +7,7 @@ module Flydata
 
       module MysqlAccessible
         def mysql_conf(conf)
-          @mysql_conf = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
+          @mysql_conf = [:host, :port, :username, :password, :database, :ssl_ca, :sslca].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
         end
 
         def mysql_cli(conf = nil)
@@ -42,7 +42,7 @@ module Flydata
       class MysqlDumpGenerator
         def initialize(conf)
           @conf = conf
-          @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
+          @db_opts = [:host, :port, :username, :password, :database, :ssl_ca, :sslca].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
         end
 
         def dump(file_path)

data/lib/flydata/parser/mysql/mysql_alter_table.treetop

@@ -1062,6 +1062,13 @@ grammar MysqlAlterTable
         }
       end
     }
+    / auto_increment_opt opt_equal nsp ulong_num {
+      def action
+        { action: :auto_increment,
+          support_level: :nonbreaking
+        }
+      end
+    }
     #TODO - There are other rules which need to be implemented when required
   end
 

data/lib/flydata/sync_file_manager.rb

@@ -105,6 +105,21 @@ module Flydata
       "#{master_binlog_path[0..-5]}.sent.pos"
     end
 
+    # ssl_ca file path
+    def ssl_ca_path(master_binlog_path = binlog_path)
+      unless master_binlog_path && master_binlog_path.end_with?('binlog.pos')
+        raise ArgumentError.new("Invalid binlog path. binlog path needs to end with 'binlog.pos'")
+      end
+      # <data-entry-name>.ssl_ca.pem
+      "#{master_binlog_path[0..-12]}.ssl_ca.pem"
+    end
+
+    def save_ssl_ca(ssl_ca_content, path = ssl_ca_path)
+      File.open(path, 'w') do |f|
+        f.write(ssl_ca_content)
+      end
+    end
+
     # table files
     def reset_table_position_files(tables)
       tables.each do |table_name|

data/lib/flydata/util/mysql_util.rb

@@ -12,6 +12,7 @@ module Flydata
       # - password
       # - database
       # - tables  # array
+      # - ssl_ca
       # - custom_option   # string
       def self.generate_mysql_cmd(option)
         raise ArgumentError.new("option must be hash.") unless option.kind_of?(Hash)
@@ -21,13 +22,18 @@ module Flydata
         port = option[:port] ? "-P #{option[:port]}" : nil
         username = option[:username] ? "-u#{option[:username]}" : nil
         password = if !(option[:password].to_s.empty?)
-                     "-p\"#{option[:password].gsub('`', '\\\`')}\""
+                     "-p\"#{option[:password].gsub('$','\\$').gsub('"','\\"').gsub('`', '\\\`')}\""
                    else
                      nil
                    end
         database = option[:database]
         tables = option[:tables] ? option[:tables].join(' ') : nil
-        default_option = option[:no_default_option] ? nil : DEFAULT_MYSQL_CMD_OPTION
+        ssl_ca = option[:ssl_ca] ? option[:ssl_ca] : nil
+
+        default_option = option[:no_default_option] ? "" : DEFAULT_MYSQL_CMD_OPTION
+        default_option += " --ssl-ca=#{ssl_ca}" if ssl_ca
+        default_option = nil if default_option == ''
+
         custom_option = option[:custom_option]
 
         [command, host, port, username, password, default_option,

data/spec/flydata/command/base_spec.rb

@@ -7,13 +7,15 @@ module Command
 describe Base do
   subject { described_class.new }
   let(:data_entries) {
-[{"id"=>4, "data_port_id"=>1, "name"=>"flydata_sync_mysql_2", "log_path"=>nil, "created_at"=>"2015-03-02T03:06:25.000Z", "updated_at"=>"2015-03-02T03:08:02.000Z", "log_deletion"=>nil, "display_name"=>"synctest", "heroku_resource_id"=>nil, "heroku_log_type"=>nil, "log_file_type"=>nil, "log_file_delimiter"=>nil, "enabled"=>true, "type"=>"RedshiftMysqlDataEntry", "tag_name"=>"flydata.c5c0eb3d_dp1.flydata_sync_mysql_2", "tag_name_dev"=>"flydata.c5c0eb3d_dp1.flydata_sync_mysql_2.dev", "data_port_key"=>"c5c0eb3d", "schema_name"=>"", "table_name"=>"", "redshift_schema_name"=>"", "redshift_table_name"=>"", "mysql_data_entry_preference"=>{"host"=>"ubertas.flydata.co", "port"=>3306, "username"=>"mak", "password"=>password, "database"=>"mak_development", "tables"=>"items,orders", "tables_append_only"=>""}}]
+[{"id"=>4, "data_port_id"=>1, "name"=>"flydata_sync_mysql_2", "log_path"=>nil, "created_at"=>"2015-03-02T03:06:25.000Z", "updated_at"=>"2015-03-02T03:08:02.000Z", "log_deletion"=>nil, "display_name"=>"synctest", "heroku_resource_id"=>nil, "heroku_log_type"=>nil, "log_file_type"=>nil, "log_file_delimiter"=>nil, "enabled"=>true, "type"=>"RedshiftMysqlDataEntry", "tag_name"=>"flydata.c5c0eb3d_dp1.flydata_sync_mysql_2", "tag_name_dev"=>"flydata.c5c0eb3d_dp1.flydata_sync_mysql_2.dev", "data_port_key"=>"c5c0eb3d", "purpose_name" => "Test app", "schema_name"=>"", "table_name"=>"", "redshift_schema_name"=>"", "redshift_table_name"=>"", "mysql_data_entry_preference"=>{"host"=>"ubertas.flydata.co", "port"=>3306, "username"=>"mak", "password"=>password, "database"=>"mak_development", "tables"=>"items,orders", "tables_append_only"=>""}}]
   }
   let(:flydata) { double('flydata') }
   let(:path) { '/data_entries' }
   let(:response_body) { data_entries }
   let(:response) { double('response') }
   let(:response_code) { 200 }
+  let(:password) { '' }
+
   before do
     allow(response).to receive(:code).and_return(response_code)
     allow(flydata).to receive(:get).with(path).and_return(response_body)
@@ -37,8 +39,13 @@ describe Base do
     end
   end
 
+  describe "#show_purpose_name" do
+    it "calls log_info_stdout with appropriate message" do
+      expect(subject).to receive(:log_info_stdout).with("Your current application name is 'Test app'")
+      subject.show_purpose_name
+    end
+  end
 end
 
 end
 end
-

data/spec/flydata/command/status_spec.rb

@@ -8,6 +8,7 @@ module Flydata
       let(:sender) { double("sender") }
 
       it do
+        expect(subject).to receive(:show_purpose_name)
         expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
         expect(sender).to receive(:status)
         subject.run

data/spec/flydata/compatibility_check_spec.rb

@@ -138,6 +138,49 @@ module Flydata
         end
       end
     end
-  end
 
+    describe "#check_mysql_table_types" do
+      let(:test_data_entry) do
+        { "host" => "test",
+          "port" => 1234,
+          "username" => "test",
+          "password" => "password",
+          "database" => "test_db",
+          "tables"=>["normal_table", "engine_table", "view_table"] }
+      end
+      let(:normal_table) { {"table_name"=>"normal_table", "table_type"=>"BASE TABLE", "engine"=>"InnoDB"} }
+      let(:engine_table) { {"table_name"=>"engine_table", "table_type"=>"BASE TABLE", "engine"=>"MEMORY"} }
+      let(:view) { {"table_name"=>"view_table", "table_type"=>"VIEW", "engine"=>nil} }
+      let(:client) { double('client') }
+      let(:subject_object) { MysqlCompatibilityCheck.new(:default_data_port,test_data_entry, {}) }
+      let(:error) { Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError } 
+      let(:base_error_msg) { "FlyData does not support VIEW and MEMORY ENGINE table. Remove following tables from data entry: %s" }
+      subject { subject_object.check_mysql_table_types }
+      before do
+        allow(Mysql2::Client).to receive(:new).and_return(client)
+        allow(client).to receive(:query).and_return(table_list)
+        allow(client).to receive(:escape).and_return("aaa")
+        allow(client).to receive(:close)
+      end
+      context "where data entry has VIEW and MEMORY engine table" do
+        let(:error_msg) { base_error_msg %  engine_table['table_name'] + ', ' + view['table_name'] }
+        let(:table_list) { [ engine_table, view ] }
+        it { expect{subject}.to raise_error(error, /#{error_msg}/) }
+      end
+      context "where data entry has MEMORY engine table" do
+        let(:error_msg) { base_error_msg %  engine_table['table_name'] } 
+        let(:table_list) { [ engine_table ] }
+        it { expect{subject}.to raise_error(error, /#{error_msg}/) }
+      end
+      context "where data entry has the VIEW" do
+        let(:error_msg) { base_error_msg % view['table_name'] }
+        let(:table_list) { [ view ] }
+        it { expect{subject}.to raise_error(error, /#{error_msg}/) }
+      end
+      context "where data entry does not have either VIEW and ENGINE table" do
+        let(:table_list) { [ normal_table ] }
+        it { expect{subject}.to_not raise_error }
+      end
+    end
+  end
 end

data/spec/flydata/fluent-plugins/mysql/table_meta_spec.rb

@@ -3,13 +3,12 @@ require 'flydata/fluent-plugins/mysql/table_meta'
 
 module Mysql
   describe TableMeta do
-    let(:mysql_url) { double('mysql_url') }
+    let(:db_opts) { { host: 'test-host', port: 3306, username: 'test-user', password: 'test-pswd', database: 'test-db' } }
     let(:database) { 'test-db' }
     let(:tables) { %w(a_table b_table c_table) }
 
     let(:conn) { double('conn') }
-    let(:table_meta) { TableMeta.new(
-      mysql_url: mysql_url, database: database, tables: tables) }
+    let(:table_meta) { TableMeta.new(db_opts.merge(database: database, tables: tables)) }
 
     before do
       allow(conn).to receive(:close)

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -1365,6 +1365,17 @@ describe 'MysqlAlterTableParser' do
         let(:alter_table_action) { "engine#{equal} #{storage_engine}"}
         it_behaves_like "test optional equal", "test storage engines", "a parser parsing a nonbreaking query"
       end
+      context "set auto increment" do
+        let(:action) { :auto_increment }
+        context "single digit number" do
+          let(:alter_table_action) { "auto_increment#{equal} 1"}
+          it_behaves_like "test optional equal", "a parser parsing a nonbreaking query"
+        end
+        context "multi digit number" do
+          let(:alter_table_action) { "AUTO_INCREMENT#{equal} 1234"}
+          it_behaves_like "test optional equal", "a parser parsing a nonbreaking query"
+        end
+      end
     end
     context "force" do
       let(:alter_table_action) { "force" }

data/spec/flydata/sync_file_manager_spec.rb

@@ -194,6 +194,36 @@ module Flydata
       end
     end
 
+    describe '#ssl_ca_path' do
+      context 'with no args' do
+        subject { default_sfm.ssl_ca_path }
+        it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.ssl_ca.pem") }
+      end
+      context 'with invalid args' do
+        subject { default_sfm.ssl_ca_path('/home/ec2-user/.flydata/flydata_sync.pos') }
+        it { expect{subject}.to raise_error(ArgumentError) }
+      end
+      context 'with valid args' do
+        subject { default_sfm.ssl_ca_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
+        it { is_expected.to eq('/home/ec2-user/.flydata/flydata_sync.ssl_ca.pem') }
+      end
+    end
+
+    describe '#save_ssl_ca_path' do
+      let(:ssl_ca_path) { default_sfm.ssl_ca_path }
+      let(:ssl_ca_content) { 'aaaaabbbbbccccc' }
+      subject { default_sfm.save_ssl_ca(ssl_ca_content) }
+
+      after do
+        File.delete(ssl_ca_path)
+      end
+
+      it 'write a ssl_ca_content to ssl_ca_path' do
+        subject
+        expect(IO.read(ssl_ca_path)).to eq(ssl_ca_content)
+      end
+    end
+
     describe '#increment_and_save_table_position' do
       let(:test_table) { 'test_table' }
       before do

data/spec/flydata/util/mysql_util_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'flydata/util/mysql_util'
 
 module Flydata
   module Util
@@ -13,6 +14,7 @@ module Flydata
         database: 'testdb',
       } }
       let(:option) { default_option }
+      let(:cmd_pswd_check) { 'mysqldump -h test-host -P 3306 -utestuser -p%s --default-character-set=utf8 --protocol=tcp testdb' }
 
       describe '.generate_mysql_cmd' do
         subject { described_class.generate_mysql_cmd(option) }
@@ -50,11 +52,29 @@ module Flydata
           ) }
         end
 
-        context 'when password is empty' do
+        context 'when password is not empty' do
           before { option[:password] = "abc`def" }
-          it { is_expected.to eq(
-            'mysqldump -h test-host -P 3306 -utestuser -p"abc\\`def" --default-character-set=utf8 --protocol=tcp testdb'
-          ) }
+          it { is_expected.to eq(cmd_pswd_check % '"abc\\`def"' ) }
+        end
+
+        context 'when password includes double quate' do
+          before { option[:password] = "abc\"def" }
+          it { is_expected.to eq(cmd_pswd_check % '"abc\\"def"') }
+        end
+
+        context 'when password includes dollars sign' do
+          before { option[:password] = "abc$def" }
+          it { is_expected.to eq(cmd_pswd_check % '"abc\\$def"' ) }
+        end
+
+        context 'when password includes multiple special characters' do
+          before { option[:password] = "ab\"\"c$$def" }
+          it { is_expected.to eq(cmd_pswd_check % '"ab\\"\\"c\\$\\$def"' ) }
+        end
+
+        context 'when password does not includes special characters' do
+          before { option[:password] = "abcdef" }
+          it { is_expected.to eq(cmd_pswd_check % '"abcdef"' ) }
         end
 
         context 'when tables are specified' do
@@ -64,6 +84,13 @@ module Flydata
           ) }
         end
 
+        context 'when ssl_ca is specified' do
+          before { option[:ssl_ca] = 'mysql_sync.ssl_ca.pem' }
+          it { is_expected.to eq(
+            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp --ssl-ca=mysql_sync.ssl_ca.pem testdb'
+          ) }
+        end
+
         context 'when no_default_option is turned on' do
           before { option[:no_default_option] = true }
           it { is_expected.to eq(
@@ -71,7 +98,6 @@ module Flydata
           ) }
         end
 
-
         context 'with options for mysql_protocol_tcp_compat' do
           before {
             option[:command] = 'mysql'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.14
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-09 00:00:00.000000000 Z
+date: 2015-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -122,26 +122,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.10.46
-- !ruby/object:Gem::Dependency
-  name: ruby-binlog
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.0.2
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.0.2
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: fluent-plugin-mysql-binlog
   requirement: !ruby/object:Gem::Requirement
@@ -168,7 +148,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.3.11
+        version: 0.3.17
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.3'
@@ -178,7 +158,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.3.11
+        version: 0.3.17
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.3'
@@ -268,7 +248,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 0.1.3
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.2
@@ -278,7 +258,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 0.1.3
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.2
@@ -488,12 +468,14 @@ files:
 - flydata-core/lib/flydata-core/core_ext/object/prepend.rb
 - flydata-core/lib/flydata-core/errors.rb
 - flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb
+- flydata-core/lib/flydata-core/fluent/config_helper.rb
 - flydata-core/lib/flydata-core/logger.rb
 - flydata-core/lib/flydata-core/table_def.rb
 - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
 - flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
 - flydata-core/lib/flydata-core/thread_context.rb
 - flydata-core/spec/config/user_maintenance_spec.rb
+- flydata-core/spec/fluent/config_helper_spec.rb
 - flydata-core/spec/logger_spec.rb
 - flydata-core/spec/spec_helper.rb
 - flydata-core/spec/table_def/mysql_table_def_spec.rb
@@ -632,7 +614,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: FlyData Agent