flydata 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3aabc1598aa6770f9f4238bae95ef42b1577c0aa
-  data.tar.gz: 5ba322b53b34ffaf0986542a8071c103c4d2d881
+  metadata.gz: 2f947592ebedab5272b94129436953e9fac61c46
+  data.tar.gz: b676c7dd72b765244ef0b99d9cd44c584c0126c1
 SHA512:
-  metadata.gz: a1b6059f259512e9d7f190f9bc37c51905979adab22310f4782e92b3ebbc7156673ce48259d7f7ba6a43ae2d40755b2836b59adffd3b36d035a8ceb0ff1b9c4b
-  data.tar.gz: 129df374f7bbba7ed5dc70eb23bd52b46ee683521cd391c7fa0c9440f9c56649d2761210be0eca31cb8d1c05c5ed55a9ba8aadacec84085d73460af1b950303f
+  metadata.gz: 9ed77e0b82c8d0c2dbe3e2be510a0fe0b9f3d9bfef08788443b26085989cfaa566348e7537c393786b8d451a5d687dc3f5c9b32c7d8a9c45508f0f7d014099b3
+  data.tar.gz: d3073aba50dff2fbf45a613c5e64c15297bc65f380c284511acddd3af775cd33f8dbaf7acdaed023b2426c9459922b321482a2121968a53afb43ed060d680f57

data/VERSION

@@ -1 +1 @@
-0.3.5
+0.3.6

data/bin/flydata

@@ -5,6 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'rubygems'
 require 'flydata'
+require 'flydata/cli'
 
 args = ARGV.dup
 ARGV.clear # for 'gets'

data/flydata-core/lib/flydata-core/core_ext/module.rb

@@ -1 +1 @@
-require_relative 'module/include'
+require 'flydata-core/core_ext/module/include'

data/flydata-core/lib/flydata-core/core_ext/object.rb

@@ -1 +1 @@
-require_relative 'object/prepend'
+require 'flydata-core/core_ext/object/prepend'

data/flydata.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: flydata 0.3.6 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.5"
+  s.version = "0.3.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-02-28"
+  s.date = "2015-03-04"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -89,7 +91,7 @@ Gem::Specification.new do |s|
     "lib/flydata/command/stop.rb",
     "lib/flydata/command/sync.rb",
     "lib/flydata/command/version.rb",
-    "lib/flydata/command_logger.rb",
+    "lib/flydata/command_loggable.rb",
     "lib/flydata/compatibility_check.rb",
     "lib/flydata/credentials.rb",
     "lib/flydata/cron.rb",
@@ -127,9 +129,24 @@ Gem::Specification.new do |s|
     "spec/fluent_plugins_spec_helper.rb",
     "spec/fly_data_model_spec.rb",
     "spec/flydata/api/data_entry_spec.rb",
+    "spec/flydata/api_client_spec.rb",
     "spec/flydata/cli_spec.rb",
+    "spec/flydata/command/base_spec.rb",
+    "spec/flydata/command/conf_spec.rb",
+    "spec/flydata/command/crontab_spec.rb",
+    "spec/flydata/command/encrypt_spec.rb",
+    "spec/flydata/command/kill_all_spec.rb",
+    "spec/flydata/command/login_spec.rb",
+    "spec/flydata/command/restart_spec.rb",
+    "spec/flydata/command/routine_spec.rb",
     "spec/flydata/command/sender_spec.rb",
+    "spec/flydata/command/setlogdel_spec.rb",
+    "spec/flydata/command/setup_spec.rb",
+    "spec/flydata/command/start_spec.rb",
+    "spec/flydata/command/status_spec.rb",
+    "spec/flydata/command/stop_spec.rb",
     "spec/flydata/command/sync_spec.rb",
+    "spec/flydata/command/version_spec.rb",
     "spec/flydata/compatibility_check_spec.rb",
     "spec/flydata/fluent-plugins/idle_event_detector_spec.rb",
     "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
@@ -150,8 +167,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.14"
+  s.rubygems_version = "2.4.3"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata.rb

@@ -1,25 +1,22 @@
 # load_path settings
 def append_load_path_if_not_exist(new_lib_path)
-  absolute_path = File.realpath(new_lib_path)
-  $LOAD_PATH.unshift absolute_path unless $LOAD_PATH.include? absolute_path
+  expanded_path = File.expand_path(new_lib_path)
+  $LOAD_PATH.unshift expanded_path unless $LOAD_PATH.include? expanded_path
 end
-flydata_top_dir = File.realpath('../../', __FILE__)
+flydata_top_dir = File.expand_path('../../', __FILE__)
 flydata_core_dir = File.join(flydata_top_dir, 'flydata-core')
-flydata_core_lib_dir = File.realpath('lib', flydata_core_dir)
+flydata_core_lib_dir = File.expand_path('lib', flydata_core_dir)
 append_load_path_if_not_exist(flydata_core_lib_dir)
 
 # require common libs
 require 'active_support/core_ext/string'
-require 'active_support/dependencies'
 require 'active_support/time'
 require 'highline/import'
 require 'readline'
 require 'json'
 require 'pp'
 
-# require all flydata libs
-lib_dir = File.absolute_path(File.dirname(__FILE__))
-ActiveSupport::Dependencies.autoload_paths << lib_dir
+require 'flydata/heroku'
 
 module Flydata
   FLYDATA_DEBUG = !!(ENV['FLYDATA_DEBUG'])

data/lib/flydata/api/data_entry.rb

@@ -1,3 +1,5 @@
+require 'flydata/api/base'
+
 module Flydata
   module Api
     class DataEntry < Base

data/lib/flydata/api/data_port.rb

@@ -1,3 +1,5 @@
+require 'flydata/api/base'
+
 module Flydata
   module Api
     class DataPort < Base

data/lib/flydata/api/redshift_cluster.rb

@@ -1,3 +1,5 @@
+require 'flydata/api/base'
+
 module Flydata
   module Api
     class RedshiftCluster < Base

data/lib/flydata/api_client.rb

@@ -1,5 +1,7 @@
 require 'rest_client'
 require 'singleton'
+require 'flydata/credentials'
+require 'flydata/helpers'
 
 module Flydata
   FLYDATA_API_HOST = ENV['FLYDATA_API_HOST'] || 'https://flydata.co'
@@ -39,6 +41,7 @@ module Flydata
     # high level api
     def method_missing(cls_name, *args, &block)
       method_name = args.shift.to_s
+      require File.join("flydata/api", cls_name.to_s)
       api_cls = "Flydata::Api::#{cls_name.to_s.camelize}".constantize
       api_cls.new(self)
     end

data/lib/flydata/cli.rb

@@ -1,6 +1,7 @@
 require 'slop'
-require_relative 'command_logger'
-require_relative 'errors'
+require 'flydata/command_loggable'
+require 'flydata/errors'
+require 'flydata/helpers'
 
 module Flydata
   class Cli
@@ -29,6 +30,7 @@ module Flydata
           first_arg = @args.shift
           cmd, sub_cmd = parse_command(first_arg)
           begin
+            require_command_source(cmd)
             cmd_cls = "Flydata::Command::#{cmd.capitalize}".constantize
           rescue NameError 
             # NameError is raised when the constant is not defined in its target
@@ -66,5 +68,14 @@ module Flydata
     def parse_command(cmd)
       cmd.split(':')
     end
+    COMMAND_DIR = 'flydata/command'
+    def require_command_source(cmd)
+      source_path = File.join(COMMAND_DIR, cmd)
+      begin
+        require source_path
+      rescue LoadError
+        # ignore
+      end
+    end
   end
 end

data/lib/flydata/command/base.rb

@@ -1,5 +1,8 @@
 require 'slop'
 require 'flydata/api_client'
+require 'flydata/command_loggable'
+require 'flydata/preference/data_entry_preference'
+
 
 module Flydata
   module Command
@@ -22,6 +25,7 @@ module Flydata
           if Flydata::Preference::DataEntryPreference.conf_exists?(de)
             Flydata::Preference::DataEntryPreference.load_conf(de)
           else
+            Flydata::Preference::DataEntryPreference.filter_data_entry(de)
             de
           end
         end
@@ -29,6 +33,8 @@ module Flydata
       def register_crontab
         data_entries = retrieve_data_entries
         if data_entries.any?{|e| e['log_deletion']}
+          # require on demand to avoid mutual require issue
+          require 'flydata/command/crontab'
           Flydata::Command::Crontab.new.run
         end
       end

data/lib/flydata/command/conf.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/preference/data_entry_preference'
+
 module Flydata
   module Command
     class Conf < Base

data/lib/flydata/command/crontab.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/cron'
+
 module Flydata
   module Command
     class Crontab < Base

data/lib/flydata/command/encrypt.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/util/encryptor'
+
 module Flydata
   module Command
     class Encrypt < Base

data/lib/flydata/command/kill_all.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/command/sender'
+
 module Flydata
   module Command
     class Kill_all < Base

data/lib/flydata/command/login.rb

@@ -1,3 +1,5 @@
+require 'flydata/command/base'
+
 module Flydata
   module Command
     class Login < Base

data/lib/flydata/command/restart.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/command/sender'
+
 module Flydata
   module Command
     class Restart < Base

data/lib/flydata/command/routine.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/log_monitor'
+
 module Flydata
   module Command
     class Routine < Base

data/lib/flydata/command/sender.rb

@@ -1,4 +1,6 @@
 require 'flydata/compatibility_check'
+require 'flydata/command/base'
+require 'flydata/command/sync'
 
 module Flydata
   module Command

data/lib/flydata/command/setlogdel.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/command/login'
+
 module Flydata
   module Command
     class Setlogdel < Base
@@ -16,7 +19,7 @@ module Flydata
         data_entries = retrieve_data_entries
         if data_entries.size < 1
           puts("There are no registered entries.")
-          puts("You need to create a data entry by flydata setup command.")
+          puts("You need to create a data entry with flydata setup command.")
           return nil
         end
         choices = data_entries.map {|e|

data/lib/flydata/command/setup.rb

@@ -1,4 +1,9 @@
-require_relative 'sync'
+require 'flydata/command/base'
+require 'flydata/command/conf'
+require 'flydata/command/login'
+require 'flydata/command/sender'
+require 'flydata/helpers'
+require 'flydata/sync_file_manager'
 
 module Flydata
   module Command
@@ -92,7 +97,7 @@ What's next?
             true
           elsif has_registered_redshift_mysql_data_entries?
             de = retrieve_data_entries.first
-            if File.exists?(Flydata::FileUtil::SyncFileManager.new(de).binlog_path)
+            if File.exists?(Flydata::SyncFileManager.new(de).binlog_path)
               sender = Flydata::Command::Sender.new
               if sender.process_exist?
                 sender.stop(quiet: true)

data/lib/flydata/command/start.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/command/sender'
+
 module Flydata
   module Command
     class Start < Base

data/lib/flydata/command/status.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/command/sender'
+
 module Flydata
   module Command
     class Status < Base

data/lib/flydata/command/stop.rb

@@ -1,3 +1,6 @@
+require 'flydata/command/base'
+require 'flydata/command/sender'
+
 module Flydata
   module Command
     class Stop < Base

data/lib/flydata/command/sync.rb

@@ -2,11 +2,18 @@ require 'fiber'
 require 'msgpack'
 require 'open3'
 require 'mysql2'
+require 'rest_client'
 require 'sys/filesystem'
-require 'flydata/sync_file_manager'
+require 'flydata/command/base'
+require 'flydata/command/conf'
+require 'flydata/command/sender'
 require 'flydata/compatibility_check'
+require 'flydata/errors'
+require 'flydata/helpers'
 require 'flydata/output/forwarder'
 require 'flydata/parser/mysql/dump_parser'
+require 'flydata/preference/data_entry_preference'
+require 'flydata/sync_file_manager'
 require 'flydata/util/mysql_util'
 require 'flydata-core/table_def'
 #require 'ruby-prof'
@@ -603,7 +610,7 @@ EOM
         start_time = Time.now
         log_info_stdout("Waiting for the server buffer to get empty.")
         prev_message =nil
-        while (status = check_server_status(tables)) && (status['state'] == 'processing')
+        while (status = check_server_status(tables)) && (status['state'].nil? || status['state'] == 'processing')
           prev_message = status['message']
           if timeout > 0 && Time.now - start_time > timeout
             raise ServerDataProcessingTimeout.new
@@ -815,7 +822,7 @@ Thank you for using FlyData!
       end
 
       def create_sync_file_manager(de = data_entry)
-        Flydata::FileUtil::SyncFileManager.new(de)
+        SyncFileManager.new(de)
       end
 
       def verify_input_tables(input_tables, all_tables)

data/lib/flydata/command/version.rb

@@ -1,3 +1,5 @@
+require 'flydata/command/base'
+
 module Flydata
   module Command
     class Version < Base

data/lib/flydata/compatibility_check.rb

@@ -1,5 +1,5 @@
 require 'mysql2'
-require 'flydata/command_logger'
+require 'flydata/command_loggable'
 require 'flydata/util/mysql_util'
 
 module Flydata

data/lib/flydata/credentials.rb

@@ -1,3 +1,5 @@
+require 'flydata/helpers'
+
 module Flydata
   class Credentials
     include Helpers

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -5,18 +5,17 @@ require 'binlog'
 require 'kodama'
 
 # Load client library(flydata/cli/lib)
-lib = File.absolute_path(File.dirname(__FILE__) + '/../..')
+lib = File.expand_path(File.dirname(__FILE__), '/../..')
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'flydata'
 require 'flydata/sync_file_manager'
 require 'flydata/util/mysql_util'
-
-require_relative 'preference'
-require_relative 'mysql/binlog_position_file'
-require_relative 'mysql/binlog_record_dispatcher'
-require_relative 'mysql/context'
-require_relative 'idle_event_detector'
-require_relative 'mysql/table_meta'
+require 'flydata/fluent-plugins/preference'
+require 'flydata/fluent-plugins/mysql/binlog_position_file'
+require 'flydata/fluent-plugins/mysql/binlog_record_dispatcher'
+require 'flydata/fluent-plugins/mysql/context'
+require 'flydata/fluent-plugins/idle_event_detector'
+require 'flydata/fluent-plugins/mysql/table_meta'
 
 #Monkey-patch fluentd class (EngineClass) to support shutdown for input plugin.
 #This will be called when USR1 signal is received
@@ -73,7 +72,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       @tables << table unless @tables.include?(table)
       @omit_events[table] = [:delete]
     end
-    @sync_fm = Flydata::FileUtil::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
+    @sync_fm = Flydata::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
 
     #Remove tables that do not have pos files
     new_tables = @sync_fm.get_new_table_list(@tables, "pos")