flydata 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b34f397692efd9d9bc9575588d19263b9c0df2c
-  data.tar.gz: 21b42d4c19267a6656eb8f8d8c21607cf54501f1
+  metadata.gz: 8b8bbf89988f5ae28562da185290028c00f02304
+  data.tar.gz: e443c6fb75c571b1c1fc95ef40c3a877a7bb31fa
 SHA512:
-  metadata.gz: 147ed1f61574abacfc850460e4745953c37bba5827985bd82adc51c9ef6750cb6188f3e56f52a3e855f73096c9fb6c05e97039ed68ac0982029a259f8a9a8c78
-  data.tar.gz: a73489d7abcfb93db8720819143ad751d78493389d911407ede925a8f2417ba9161d7a742365b97de56595ca168139832ef179f37a025ce47e015da97b03829d
+  metadata.gz: aef7d1b13401d4881f020406ed6cfd32aca7193a01570cfac081556ce24a9a67b63678ce7da1762036dfbddeb02cbf67f283e4ce3ff4e7d9c756580cb786960c
+  data.tar.gz: cb4a80a2c7ce7fc1a507c57b6ad6ea51f23ae3299c562e3165799764d03eb9c8a021895209b1bb7c4708a54ebd609ae29b8dd437cf85d495e166169624dab0f6

data/Gemfile

@@ -13,6 +13,7 @@ gem "mysql2", '~> 0.3', '>= 0.3.11'
 gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
+gem "io-console", '~> 0.4.2', '>= 0.4.2'
 
 group :development do
   gem "jeweler", '~> 1.8', '>= 1.8.8'

data/Gemfile.lock

@@ -49,6 +49,7 @@ GEM
     highline (1.6.21)
     http_parser.rb (0.6.0)
     i18n (0.6.11)
+    io-console (0.4.2)
     jeweler (1.8.8)
       builder
       bundler (~> 1.0)
@@ -129,6 +130,7 @@ DEPENDENCIES
   fluentd (= 0.10.46)
   highline (~> 1.6, >= 1.6.19)
   i18n (~> 0.6, >= 0.6.5)
+  io-console (~> 0.4.2, >= 0.4.2)
   jeweler (~> 1.8, >= 1.8.8)
   json (~> 1.8, >= 1.8.0)
   mysql2 (~> 0.3, >= 0.3.11)

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.3.2

data/flydata.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: flydata 0.3.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.1"
+  s.version = "0.3.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-02-24"
+  s.date = "2015-02-25"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -150,8 +152,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.14"
+  s.rubygems_version = "2.2.2"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then
@@ -170,6 +171,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
       s.add_runtime_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_runtime_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
+      s.add_runtime_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
       s.add_development_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_development_dependency(%q<rspec>, ["~> 3.0"])
       s.add_development_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -192,6 +194,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
       s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
+      s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
       s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_dependency(%q<rspec>, ["~> 3.0"])
       s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -215,6 +218,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
     s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
     s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
+    s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
     s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
     s.add_dependency(%q<rspec>, ["~> 3.0"])
     s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])

data/lib/flydata/command/sender.rb

@@ -34,12 +34,7 @@ module Flydata
 
         # Start sender(fluentd) process
         log_info_stdout("Starting sender process.") unless options[:quiet]
-        Dir.chdir(FLYDATA_HOME){
-          Kernel.system("bash #{FLYDATA_SERVERINFO}", :out => [FLYDATA_LOG,'a'], :err => [FLYDATA_LOG,'a'])
-          daemon_opt = opts.no_daemon? ? "" : daemon_option
-          Kernel.system("ruby `which fluentd` #{daemon_opt} -l #{FLYDATA_LOG} -c #{FLYDATA_CONF} -p #{FLYDATA_FLUENT_PLUGIN_DIR}")
-        }
-        Kernel.sleep 5
+        raw_start(options)
 
         wait_until_client_ready(options)
         #wait_until_logs_uploaded
@@ -68,7 +63,7 @@ EOF
         unless process_exist?
           return true if client_buffer_empty?
           log_info_stdout("Process doesn't exist. But, the client buffer is not empty!!") unless options[:quiet]
-          start(options)
+          raw_start(options)
         end
 
         log_info_stdout('Stopping input plugins and flushing the client buffer.') unless options[:quiet]
@@ -122,6 +117,14 @@ EOF
       # Return a list of fluentd parent processes run by the same user for the
       # same flydata.pid file but not the process managed by the file itself.
       CMD = %Q{test -f %s && ps -u `whoami` -o ppid,pid,args | grep '^ *1 ' | grep '\\%s' | egrep -v "\\b`cat %s`\\b"}
+      def raw_start(options)
+        Dir.chdir(FLYDATA_HOME){
+          Kernel.system("bash #{FLYDATA_SERVERINFO}", :out => [FLYDATA_LOG,'a'], :err => [FLYDATA_LOG,'a'])
+          daemon_opt = opts.no_daemon? ? "" : daemon_option
+          Kernel.system("ruby `which fluentd` #{daemon_opt} -l #{FLYDATA_LOG} -c #{FLYDATA_CONF} -p #{FLYDATA_FLUENT_PLUGIN_DIR}")
+        }
+        Kernel.sleep 5
+      end
       def orphan_processes
         cmd =  CMD % [ pid_file, daemon_option, pid_file ]
         result = `#{cmd}`

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -75,6 +75,11 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     end
     @sync_fm = Flydata::FileUtil::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
 
+    #Remove tables that do not have pos files
+    new_tables = @sync_fm.get_new_table_list(@tables, "pos")
+    @tables -= new_tables
+    $log.info "Not watching these tables: #{new_tables.join(", ")}"
+
     table_meta = Mysql::TableMeta.new(
       mysql_url: mysql_url, database: @database, tables: @tables)
     table_meta.update

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -520,6 +520,23 @@ EOT
           end
           include_examples 'emits records correctly'
         end
+
+        context 'do not watch table if no position file is found' do
+          # Stub SyncFileManager.new(nil).get_new_table_list() and return [TEST_TABLE]
+          # Need to make sure no event is sent... how do I do that
+          let(:sync_fm) { double('sync_fm') }
+          before do
+            Flydata::FileUtil::SyncFileManager.any_instance.should_receive(:get_new_table_list).with(TEST_TABLES.split(","), "pos").and_return([TEST_TABLE])
+
+            Test.configure_plugin(plugin, TEST_CONFIG)
+            plugin.event_listener(rotate_event)
+          end
+          it do
+            event = insert_event
+            event['table_name'] = TEST_TABLE
+            expect_no_emitted_record(event)
+          end
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-24 00:00:00.000000000 Z
+date: 2015-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -242,6 +242,26 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: io-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.2
 - !ruby/object:Gem::Dependency
   name: jeweler
   requirement: !ruby/object:Gem::Requirement
@@ -572,7 +592,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: FlyData Agent