flydata 0.2.20 → 0.2.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4d29cd6b59f9acb406d62017d559296f0fa27b7
-  data.tar.gz: ab8d2cdcebbaab49345f62da3aef259e5a977e02
+  metadata.gz: b2bb40962e8bd93a3daf20cee2de578c4abc1788
+  data.tar.gz: 266ca371867bd725c6721bae7d23fbea9fd4a0e6
 SHA512:
-  metadata.gz: 6f590f2046f0c016a2beae67555a0544b919b9ff5873376ee190f6a2065ae30e6e961178926127537a97baab5140dc1e1c9c5e5051895a5661d4cf93b90ccc0a
-  data.tar.gz: 6af1ee2c5597ed08e133650a5882ca6a4531ff47b0a25bf82566a3c3e0de4ecb3172c6c6da62449a853af49785197c15ae012ce59a4303e4a11d1826c02e7cd6
+  metadata.gz: daaa7f5468f4f334ef0b7d0cea6cfd672c0a816fd119ca4b54c4f6fa5619987ff37c5df587b11c734d4a29fd2518072277c98a810755d0f0db8008fc719a0ec1
+  data.tar.gz: 3140aea09a5fad9977f1a0bf9d60f317f71637129e0b7ac423198e43e84159534f1715d49d28db28ddc135e522bcbdc32f3ed1415904d2f6cffeffc1f46066c3

data/VERSION

@@ -1 +1 @@
-0.2.20
+0.2.21

data/flydata.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: flydata 0.2.21 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.20"
+  s.version = "0.2.21"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2014-12-05"
+  s.date = "2014-12-12"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -100,6 +102,7 @@ Gem::Specification.new do |s|
     "lib/flydata/fluent-plugins/mysql/context.rb",
     "lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb",
     "lib/flydata/fluent-plugins/mysql/dml_record_handler.rb",
+    "lib/flydata/fluent-plugins/mysql/table_meta.rb",
     "lib/flydata/fluent-plugins/out_forward_ssl.rb",
     "lib/flydata/fluent-plugins/preference.rb",
     "lib/flydata/flydata_crontab.sh",
@@ -127,6 +130,7 @@ Gem::Specification.new do |s|
     "spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb",
     "spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
     "spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb",
+    "spec/flydata/fluent-plugins/mysql/table_meta_spec.rb",
     "spec/flydata/heroku_spec.rb",
     "spec/flydata/output/forwarder_spec.rb",
     "spec/flydata/parser/mysql/alter_table_parser_spec.rb",
@@ -139,8 +143,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.14"
+  s.rubygems_version = "2.2.2"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/base.rb

@@ -1,3 +1,6 @@
+require 'slop'
+require 'flydata/api_client'
+
 module Flydata
   module Command
     class Base

data/lib/flydata/command_logger.rb

@@ -19,7 +19,7 @@ module Flydata
       log_warn(message, log_params, {stderr: true})
     end
 
-    def log_error_stderr(message)
+    def log_error_stderr(message, log_params = {})
       log_error(message, log_params, {stderr: true})
     end
   end

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -14,6 +14,7 @@ require_relative 'preference'
 require_relative 'mysql/binlog_position_file'
 require_relative 'mysql/binlog_record_dispatcher'
 require_relative 'mysql/context'
+require_relative 'mysql/table_meta'
 
 #Monkey-patch fluentd class (EngineClass) to support shutdown for input plugin.
 #This will be called when USR1 signal is received
@@ -69,9 +70,14 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     end
     sync_fm = Flydata::FileUtil::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
 
+    table_meta = Mysql::TableMeta.new(
+      mysql_url: mysql_url, database: @database, tables: @tables)
+    table_meta.update
+
     @context = Mysql::Context.new(
       database: @database, tables: @tables,
-      tag: @tag, sync_fm: sync_fm, omit_events: @omit_events
+      tag: @tag, sync_fm: sync_fm, omit_events: @omit_events,
+      table_meta: table_meta,
     )
     @record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
   end

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -14,6 +14,7 @@ module Mysql
     def initialize(context)
       @context = context
       @first_empty_binlog = true
+      @table_meta = @context.table_meta
 
       # Load per-table binlog position
       @table_binlog_pos = {}

data/lib/flydata/fluent-plugins/mysql/context.rb

@@ -1,7 +1,7 @@
 module Mysql
   class Context
     MANDATORY_OPTS = [
-      :database, :tables, :tag, :sync_fm, :omit_events
+      :database, :tables, :tag, :sync_fm, :omit_events, :table_meta
     ]
     OPTIONAL_OPTS = [
       :current_binlog_file

data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb

@@ -42,25 +42,34 @@ module Mysql
 
     def emit_rows(type, record)
       emit_record(type, record) do |opt|
+        table_name = record['table_name']
         records = record["rows"].collect do |row|
           row = yield(row) if block_given?  # Give the caller a chance to generate the correct row
-          { ROW => convert_to_flydata_row_format(row) }
+          { ROW => convert_to_flydata_row_format(table_name, row) }
         end
         encode_signless_integer(records, record["columns"])
         records
       end
     end
 
-    def convert_to_flydata_row_format(row)
+    def convert_to_flydata_row_format(table_name, row)
       row.each.with_index(1).inject({}) do |h, (v, i)|
         if v.kind_of?(String)
-          v = v.encode('utf-16', :undef => :replace, :invalid => :replace).encode('utf-8')
+          v = encode_row_value(table_name, v)
         end
         h[i.to_s] = v
         h
       end
     end
 
+    def encode_row_value(table_name, value)
+      if src_encoding = @table_meta[table_name][:encoding]
+        value.encode('utf-8', src_encoding, :undef => :replace, :invalid => :replace)
+      else
+        value.encode('utf-16', :undef => :replace, :invalid => :replace).encode('utf-8')
+      end
+    end
+
     def encode_signless_integer(records, column_types)
       records.each do |record|
         record[ROW].keys.each do |position|

data/lib/flydata/fluent-plugins/mysql/table_meta.rb

@@ -0,0 +1,83 @@
+require 'mysql2'
+
+module Mysql
+  class TableMeta
+    MANDATORY_OPTS = [
+      :mysql_url, :database, :tables
+    ]
+    OPTIONAL_OPTS = [
+    ]
+
+    GET_TABLE_META_SQL = <<EOT
+SELECT
+  T.table_name as table_name,
+  CCSA.character_set_name as character_set_name
+FROM
+  information_schema.`TABLES` T,
+  information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY` CCSA
+WHERE
+  CCSA.collation_name = T.table_collation
+  AND T.table_schema = '%{database}'
+  AND T.table_name in (%{tables});
+EOT
+
+    def initialize(opts)
+      missing_opts = MANDATORY_OPTS - opts.keys
+      raise "Mandatory option(s) are missing: #{missing_opts.join(', ')}" unless (missing_opts.empty?)
+      @mysql_url = opts[:mysql_url]
+      @database = opts[:database]
+      @tables = opts[:tables]
+      @table_meta = Hash.new{|h, k| h[k] = {}}
+    end
+
+    def update
+      conn = Mysql2::Client.new(@mysql_url)
+      sql = GET_TABLE_META_SQL % {
+        database: @database, tables: @tables.collect{|t| "'#{t}'"}.join(',') }
+      columns = conn.query(sql)
+      columns.collect do |col|
+        @table_meta[col['table_name'].to_sym][:encoding] = get_ruby_charset_name(col['character_set_name'])
+      end
+    ensure
+      conn.close rescue nil if conn
+    end
+
+    # Charset naming conversion rule. mysql => ruby
+    #
+    # mysql
+    #   http://dev.mysql.com/doc/refman/5.6/en/charset-charsets.html
+    # mysql(supported CJK character sets)
+    #   http://dev.mysql.com/doc/refman/5.6/en/faqs-cjk.html#qandaitem-A-11-1-1
+    # For ruby, you can see encoding list with "Encoding.list"
+    CHARSET_ENCODE_RULE = {
+      'ascii' => nil, # 'ASCII-8BIT', (not need to be encoded)
+      'utf8' => nil, # 'UTF-8', (not need to be encoded)
+      'utf16' => 'UTF-16',
+      'utf32' => 'UTF-32',
+      'latin1' => 'ISO-8859-1',
+      'latin2' => 'ISO-8859-2',
+      'big5' => 'Big5',
+      'cp932' => 'CP932',
+      'eucjpms' => 'eucJP-ms',
+      'euckr' => 'EUC-KR',
+      'gb18030' => 'GB18030',
+      'gb2312' => 'GB2312',
+      'gbk' => 'GBK',
+      'sjis' => 'Shift_JIS',
+      'ujis' => 'EUC-JP',
+    }
+
+    def get_ruby_charset_name(mysql_charset)
+      return nil if mysql_charset.to_s.empty?
+      raise "Unsupported charset:#{mysql_charset}." unless CHARSET_ENCODE_RULE.has_key?(mysql_charset)
+      charset = CHARSET_ENCODE_RULE[mysql_charset]
+      return charset
+    end
+
+    # Return table meta
+    #   :character_set_name
+    def [](table_name)
+      @table_meta[table_name.to_sym]
+    end
+  end
+end

data/spec/fluent_plugins_spec_helper.rb

@@ -1,5 +1,7 @@
+require 'spec_helper'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib', 'flydata', 'fluent-plugins'))
-require 'rspec'
+$log = nil
+
 require 'fluent/test'
 
 module Fluent

data/spec/flydata/cli_spec.rb

@@ -40,7 +40,7 @@ module Flydata
                 expect(a.to_hash).to be_empty
               }.and_return(command_obj)
               expect(command_obj).to receive(:run).with(*options)
-  
+
               subject.run
             end
           end

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -225,6 +225,7 @@ EOT
       setup_initial_flydata_files
       allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
       allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
+      allow(Mysql::TableMeta).to receive(:update)
       allow(plugin).to receive(:`).with("mysql -V").and_return("mysql  Ver 14.14 Distrib 5.5.40")
       allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
       Timecop.freeze(now)

data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb

@@ -1,4 +1,4 @@
-require_relative '../../../spec_helper'
+require 'fluent_plugins_spec_helper'
 require 'flydata/fluent-plugins/mysql/alter_table_query_handler'
 
 module Mysql
@@ -10,11 +10,13 @@ module Mysql
     end
     let(:database) { "testdb" }
     let(:table) { "foo" }
+    let(:table_meta) { double'table_meta' }
     let(:context) do
       r = double('context')
       allow(r).to receive(:sync_fm).and_return(sync_fm)
       allow(r).to receive(:database).and_return([database])
       allow(r).to receive(:tables).and_return([table])
+      allow(r).to receive(:table_meta).and_return(table_meta)
       r
     end
     subject { described_class.new(context) }
@@ -38,7 +40,8 @@ module Mysql
         r
       end
       before do
-        allow(MysqlAlterTableParser).to receive(:new).and_return(parser)
+        parser_class = ParserProvider.parser(:mysql, :mysql_alter_table)
+        allow(parser_class).to receive(:new).and_return(parser)
         allow(subject).to receive(:acceptable_table?).and_return(true)
         allow(subject).to receive(:acceptable_db?).and_return(true)
         allow(subject).to receive(:check_empty_binlog)

data/spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb

@@ -1,4 +1,5 @@
-require_relative '../../../spec_helper'
+require 'fluent_plugins_spec_helper'
+require 'mysql/binlog_position'
 
 module Mysql
   describe BinLogPosition do

data/spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb

@@ -1,4 +1,4 @@
-require_relative '../../../spec_helper'
+require 'fluent_plugins_spec_helper'
 require 'flydata/fluent-plugins/mysql/binlog_query_dispatcher'
 
 module Mysql

data/spec/flydata/fluent-plugins/mysql/table_meta_spec.rb

@@ -0,0 +1,61 @@
+require 'fluent_plugins_spec_helper'
+require 'flydata/fluent-plugins/mysql/table_meta'
+
+module Mysql
+  describe TableMeta do
+    let(:mysql_url) { double('mysql_url') }
+    let(:database) { 'test-db' }
+    let(:tables) { %w(a_table b_table c_table) }
+
+    let(:conn) { double('conn') }
+    let(:table_meta) { TableMeta.new(
+      mysql_url: mysql_url, database: database, tables: tables) }
+
+    before do
+      allow(conn).to receive(:close)
+      allow(Mysql2::Client).to receive(:new).and_return(conn)
+    end
+
+    describe '.update' do
+      let(:query_result) {[]}
+      before do
+        allow(conn).to receive(:query).and_return(query_result)
+      end
+
+      context 'when character_set_name is supported' do
+        context 'when encoding is not necessary' do
+          let(:query_result) {[
+            {'table_name' => 'a_table', 'character_set_name' => 'utf8' }
+          ]}
+          it 'set nil for encoding' do
+            table_meta.update
+            expect(table_meta['a_table'][:encoding]).to be_nil
+          end
+        end
+
+        context 'when encoding is needed' do
+          let(:query_result) {[
+            {'table_name' => 'a_table', 'character_set_name' => 'latin1' },
+            {'table_name' => 'b_table', 'character_set_name' => 'cp932' }
+          ]}
+          it 'set ruby encoding encoding' do
+            table_meta.update
+            expect(table_meta['a_table'][:encoding]).to eq('ISO-8859-1')
+            expect(table_meta['b_table'][:encoding]).to eq('CP932')
+          end
+        end
+      end
+
+      context 'when character_set_name is not supported' do
+        let(:query_result) {[
+          {'table_name' => 'a_table', 'character_set_name' => 'xxxx' }
+        ]}
+        it 'raise an error' do
+          expect{table_meta.update}.to raise_error('Unsupported charset:xxxx.')
+        end
+      end
+    end
+  end
+end
+
+

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'flydata/parser_provider'
 
 describe 'MysqlAlterTableParser' do
   before do

data/spec/flydata/sync_file_manager_spec.rb

@@ -1,5 +1,7 @@
 # coding: utf-8
 require 'spec_helper'
+require 'flydata/sync_file_manager'
+require 'flydata/parser/mysql/dump_parser'
 
 module Flydata
   module FileUtil

data/spec/spec_helper.rb

@@ -12,6 +12,9 @@ File.open(File.join(temp_dir_path, 'flydata_api_host'), 'w') do |f|
   f.write('http://0.0.0.0:3000')
 end
 
+require 'logger'
+$log = Logger.new(STDOUT)
+
 require 'flydata'
 
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.20
+  version: 0.2.21
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-05 00:00:00.000000000 Z
+date: 2014-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -485,6 +485,7 @@ files:
 - lib/flydata/fluent-plugins/mysql/context.rb
 - lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb
 - lib/flydata/fluent-plugins/mysql/dml_record_handler.rb
+- lib/flydata/fluent-plugins/mysql/table_meta.rb
 - lib/flydata/fluent-plugins/out_forward_ssl.rb
 - lib/flydata/fluent-plugins/preference.rb
 - lib/flydata/flydata_crontab.sh
@@ -512,6 +513,7 @@ files:
 - spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
 - spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb
 - spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb
+- spec/flydata/fluent-plugins/mysql/table_meta_spec.rb
 - spec/flydata/heroku_spec.rb
 - spec/flydata/output/forwarder_spec.rb
 - spec/flydata/parser/mysql/alter_table_parser_spec.rb
@@ -541,7 +543,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: FlyData Agent