flydata 0.3.10 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 478d29c0767a534808dea15674528d5b5442bbed
-  data.tar.gz: c6536fb66fedffbdcc52069cb28f32f74fe3465d
+  metadata.gz: c81a2a245fd400366aa535123cd4bcfa009aa29f
+  data.tar.gz: 6762cefccdabfd09f5e7034bbf32aa9fb6d6262b
 SHA512:
-  metadata.gz: 6b97f3acc0e763fcff4f2d4a18dcc4d9625d736db229b00554aee36da0ff8e68a189324a46abb3d985dfd1e2d4964ec6963a9299d475170e759485cb1e07faec
-  data.tar.gz: fdee03e1c1bf32699f8c5157aa51f48ecf6f1c023d2160a63b996ca7ac0599fa952de02b55045715b4a66283083b204860201a31a157b9cbf92d2413aebc77d5
+  metadata.gz: cd9983b914ef12e36f39248dfe4fde7c49e321b17df7ad476155061d84da62dde5a0cd78ed94c17b23c55bebd5b3b591adccafb45c5c4036305702e8cd5bab44
+  data.tar.gz: 18e24eb120bc7d0aa61d7e9819c4d59593a07645703ffcb627d8ede58deb6e43953a20318006a5c4fcf562cc4decac586d34ec4b697373cbc7dde0faea63a7f4

data/Gemfile

@@ -14,6 +14,7 @@ gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
 gem "io-console", '~> 0.4.2', '>= 0.4.2'
+gem "kodama", '~> 0.1.2', '>= 0.1.2'
 
 group :development do
   gem "jeweler", '~> 1.8', '>= 1.8.8'

data/Gemfile.lock

@@ -61,7 +61,7 @@ GEM
       rdoc
     json (1.8.1)
     jwt (1.0.0)
-    kodama (0.1.1)
+    kodama (0.1.2)
       ruby-binlog (>= 0.1.9)
     method_source (0.8.2)
     mime-types (2.3)
@@ -133,6 +133,7 @@ DEPENDENCIES
   io-console (~> 0.4.2, >= 0.4.2)
   jeweler (~> 1.8, >= 1.8.8)
   json (~> 1.8, >= 1.8.0)
+  kodama (~> 0.1.2, >= 0.1.2)
   mysql2 (~> 0.3, >= 0.3.11)
   protected_attributes (~> 1.0, >= 1.0.8)
   pry

data/VERSION

@@ -1 +1 @@
-0.3.10
+0.3.11

data/flydata.gemspec

@@ -2,16 +2,14 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.10 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.10"
+  s.version = "0.3.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-03-31"
+  s.date = "2015-04-03"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -168,7 +166,8 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.3"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.0.14"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then
@@ -188,6 +187,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_runtime_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_runtime_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
+      s.add_runtime_dependency(%q<kodama>, [">= 0.1.2", "~> 0.1.2"])
       s.add_development_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_development_dependency(%q<rspec>, ["~> 3.0"])
       s.add_development_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -211,6 +211,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
+      s.add_dependency(%q<kodama>, [">= 0.1.2", "~> 0.1.2"])
       s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_dependency(%q<rspec>, ["~> 3.0"])
       s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -235,6 +236,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
     s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
     s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
+    s.add_dependency(%q<kodama>, [">= 0.1.2", "~> 0.1.2"])
     s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
     s.add_dependency(%q<rspec>, ["~> 3.0"])
     s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])

data/lib/flydata/command/sync.rb

@@ -182,6 +182,7 @@ EOS
         new_tables_after_reset = @new_tables + @input_tables
         if @input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
           delete_files << sync_fm.binlog_path
+          delete_files << sync_fm.sent_binlog_path
         end
         delete_files.flatten.each do |path|
           FileUtils.rm(path) if File.exists?(path)
@@ -228,6 +229,54 @@ EOS
         do_generate_table_ddl(de)
       end
 
+      # Command: flydata sync:fix_binlogpos
+      #  - Arguments
+      def self.slop_fix_binlogpos
+        Slop.new do
+          on 'f', 'force', 'update sent binlog position file forcibly'
+        end
+      end
+
+      # Command: flydata sync:fix_binlogpos
+      # Set binlog path
+      #  - Entry method
+      def fix_binlogpos
+        de = data_entry
+        sync_fm = create_sync_file_manager(de)
+
+        if File.exists?(sync_fm.sent_binlog_path) && !opts.force?
+          log_info_stdout("Skip creating sent binlogpos because sent position file is exist already. (#{sync_fm.sent_binlog_path})")
+          return
+        end
+
+        if Flydata::Command::Sender.new.process_exist?
+          log_warn_stderr("flydata is running. flydata process needs to be stopped with 'flydata stop'.")
+          return
+        end
+
+        binlog_info = sync_fm.load_binlog
+        if binlog_info.nil?
+          log_info_stdout("Skip creating sent binlogpos because binlog position file is empty or invalid. (#{sync_fm.sent_binlog_path})")
+          return
+        end
+
+        say("Updating binlog position files...")
+        log_info("Updating binlog position files... Original binlog_info:#{binlog_info}")
+
+        # Update binlog.sent.pos file
+        #   -1 is because the position in binlog.pos is the next event's position.
+        #   on the other hand the position in sent position indicates already processed.
+        binlog_info[:pos] -= 1
+        log_info("Updating sent position file. #{binlog_info} -> #{sync_fm.sent_binlog_path}")
+        sync_fm.save_sent_binlog(binlog_info)
+
+        # Update binlog.pos file to start from head of the current binlog file
+        new_binlog_info = binlog_info.dup.tap{|h| h[:pos] = 4}  # 4 is the first position of binlog file.
+        log_info("Updating original position file. #{new_binlog_info} -> #{sync_fm.binlog_path}")
+        sync_fm.save_binlog(new_binlog_info)
+        log_info_stdout("Done!")
+      end
+
       private
 
       # Initial sync

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -60,6 +60,10 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     unless @binlog_position_file.exists?
       raise "No position file(#{@position_file}). Initial synchronization is required before starting."
     end
+
+    @sync_fm = Flydata::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
+    @sent_position_file_path = @sync_fm.sent_binlog_path(@position_file)
+
     load_custom_conf
     $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
     $log.info "mysql client version: #{`mysql -V`}"
@@ -72,7 +76,6 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       @tables << table unless @tables.include?(table)
       @omit_events[table] = [:delete]
     end
-    @sync_fm = Flydata::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
 
     #Remove tables that do not have pos files
     new_tables = @sync_fm.get_new_table_list(@tables, "pos")
@@ -131,6 +134,10 @@ EOS
     @context.table_meta.update
     start_kodama(mysql_url) do |c|
       c.binlog_position_file = @position_file
+      if c.respond_to?(:sent_binlog_position_file=)
+        $log.info "Sent position feature is enabled. sent_position_file:#{@sent_position_file_path}"
+        c.sent_binlog_position_file = @sent_position_file_path
+      end
       c.connection_retry_limit = @retry_limit
       c.connection_retry_wait = @retry_wait
       c.log_level = @log_level.to_sym

data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb

@@ -13,13 +13,13 @@ module Mysql
       PATTERN
     end
 
-    def process(record, normalized_query)
+    def process(record)
       emit_record(:alter_table, record) do |opt|
         ret = nil
         begin
-          result = ParserProvider.parser(:mysql, :mysql_alter_table).new.parse(normalized_query)
+          result = ParserProvider.parser(:mysql, :mysql_alter_table).new.parse(record["normalized_query"])
           if result.nil?
-            $log.error("Received unsupported alter table query. normalized query:'#{normalized_query}', raw query: '#{record['query']}'")
+            $log.error("Received unsupported alter table query. normalized query:'#{record['normalized_query']}', raw query: '#{record['query']}'")
           else
             ret = result.tree
             breaking_query = ret[:actions].any?{|action| !action.has_key?(:support_level) || action[:support_level] != :nonbreaking}
@@ -27,7 +27,7 @@ module Mysql
           end
         rescue => e
           msg = <<EOS
-Received unsupported alter table query. normalized query:'#{normalized_query}', raw query: '#{record['query']}' Caused by error '#{e.to_s}'
+Received unsupported alter table query. normalized query:'#{record['normalized_query']}', raw query: '#{record['query']}' Caused by error '#{e.to_s}'
   Stacktrace :
 #{e.backtrace.join("\n")}
 EOS

data/lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb

@@ -10,7 +10,7 @@ module Mysql
       normalize_query(record["query"]) do |query|
         @handlers.each do |handler|
           if (handler.pattern.match(query))
-            handler.process(record, query)
+            handler.process(record.merge({"normalized_query" => query}))
             break
           end
         end

data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb

@@ -7,7 +7,7 @@ class DdlQueryHandler < BinlogQueryHandler
 
   def acceptable_db?(record)
     db_name = record["db_name"]
-    if DDL_TABLE_QUERY =~ record["query"]
+    if DDL_TABLE_QUERY =~ record["normalized_query"]
       table_name_in_query = $1 ? $1 : $2
 
       if (idx = table_name_in_query.index("."))

data/lib/flydata/sync_file_manager.rb

@@ -1,3 +1,5 @@
+require 'mysql2'
+
 module Flydata
   class SyncFileManager
     DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
@@ -70,7 +72,7 @@ module Flydata
       end
     end
 
-    # binlog.pos file
+    # master binlog.pos file
     def save_binlog(binlog_pos)
       path = binlog_path
       File.open(path, 'w') do |f|
@@ -78,10 +80,32 @@ module Flydata
       end
     end
 
+    def load_binlog(file_path = binlog_path)
+      return nil unless File.exists?(file_path)
+      f, pos = IO.read(file_path).strip.split("\t")
+      return nil if f.nil? || f.empty? || pos.nil?
+      { binfile: f, pos: pos.to_i }
+    end
+
     def binlog_path
       File.join(FLYDATA_HOME, @data_entry['name'] + ".binlog.pos")
     end
 
+    # sent binlog.pos file
+    def save_sent_binlog(binlog_pos)
+      File.open(sent_binlog_path, 'w') do |f|
+        f.write(binlog_content(binlog_pos))
+      end
+    end
+
+    def sent_binlog_path(master_binlog_path = binlog_path)
+      unless master_binlog_path && master_binlog_path.end_with?('binlog.pos')
+        raise ArgumentError.new("Invalid binlog path. binlog path needs to end with 'binlog.pos'")
+      end
+      "#{master_binlog_path[0..-5]}.sent.pos"
+    end
+
+    # table files
     def reset_table_position_files(tables)
       tables.each do |table_name|
         file = File.join(table_positions_dir_path, table_name + ".pos")

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -15,7 +15,7 @@ module Fluent
   TEST_SEQUENCE_NUM = 1
   TEST_TABLE_BINLOG_POS = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.binlog.pos")
   TEST_TABLES = "#{TEST_TABLE},test_table_1,test_table_2,test_table_3"
-  TEST_POSITION_FILE = "test_position.log"
+  TEST_POSITION_FILE = "test_position.binlog.pos"
   TEST_REVISION_FILE = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.rev")
   TEST_TIMESTAMP = 1389214083
   TEST_TABLE_APPEND_ONLY = "test_table_4"

data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb

@@ -22,14 +22,15 @@ module Mysql
     subject { described_class.new(context) }
     describe '#process' do
       let(:query) { "a_query" }
+      let(:normalized_query) { double('normalized_query') }
       let(:record) do
         r = double('record')
         allow(r).to receive(:[]).with("db_name").and_return(database)
         allow(r).to receive(:[]).with("query").and_return(query)
         allow(r).to receive(:[]).with("table_name").and_return(table)
+        allow(r).to receive(:[]).with("normalized_query").and_return(normalized_query)
         r
       end
-      let(:normalized_query) { double('normalized_query') }
       let(:parse_result) do
         r = double('parse_result')
         r
@@ -51,7 +52,7 @@ module Mysql
         it "returns nil with an error log" do
           expect($log).to receive(:error).with(/Received unsupported alter table query.*Caused by error '#{an_error}'/m)
 
-          expect(subject.process(record, normalized_query)).to eq(nil)
+          expect(subject.process(record)).to eq(nil)
         end
       end
 
@@ -74,7 +75,7 @@ module Mysql
         it "returns nil with a warn log" do
           expect($log).to receive(:error).with(/Received unsupported alter table query\./)
 
-          expect(subject.process(record, normalized_query)).to eq(nil)
+          expect(subject.process(record)).to eq(nil)
         end
       end
     end

data/spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb

@@ -46,12 +46,13 @@ EOT
     let(:record) do
       r = double('record')
       allow(r).to receive(:[]).with("query").and_return(query)
+      expect(r).to receive(:merge).with({"normalized_query" => normalized_query}).and_return(r)
       r
     end
     let(:query_handler) do
       r = double('query_handler')
       allow(r).to receive(:pattern).and_return(/^ALTER TABLE/i)
-      expect(r).to receive(:process).with(record, normalized_query)
+      expect(r).to receive(:process).with(record)
       r
     end
     let(:context) { double('context') }

data/spec/flydata/sync_file_manager_spec.rb

@@ -153,10 +153,47 @@ module Flydata
       end
     end
 
+    describe '#load_binlog' do
+      let(:binfile) { 'mysqlbinlog.000001' }
+      let(:pos) { 107 }
+      let(:binlog_pos) { {binfile: binfile, pos: pos} }
+
+      subject { default_sfm.load_binlog }
+
+      context 'when binlog pos does not exist' do
+        before do
+          File.delete(default_sfm.binlog_path) if File.exist?(default_sfm.binlog_path)
+        end
+        it { is_expected.to be_nil }
+      end
+
+      context 'when binlog pos exists' do
+        before do
+          default_sfm.save_binlog(binlog_pos)
+        end
+        it { is_expected.to eq(binlog_pos) }
+      end
+    end
+
     describe '#binlog_path' do
       it { expect(default_sfm.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
     end
 
+    describe '#sent_binlog_path' do
+      context 'with no args' do
+        subject { default_sfm.sent_binlog_path }
+        it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.sent.pos") }
+      end
+      context 'with invalid args' do
+        subject { default_sfm.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.pos') }
+        it { expect{subject}.to raise_error(ArgumentError) }
+      end
+      context 'with valid args' do
+        subject { default_sfm.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
+        it { is_expected.to eq('/home/ec2-user/.flydata/flydata_sync.binlog.sent.pos') }
+      end
+    end
+
     describe '#increment_and_save_table_position' do
       let(:test_table) { 'test_table' }
       before do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.11
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-31 00:00:00.000000000 Z
+date: 2015-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -262,6 +262,26 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.4.2
+- !ruby/object:Gem::Dependency
+  name: kodama
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: jeweler
   requirement: !ruby/object:Gem::Requirement
@@ -608,7 +628,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: FlyData Agent