flydata 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a3fbdc5a0e42708b1dd6f173c366810e4f58892
-  data.tar.gz: 85a647490961e9b85baea5524b9c0f4f395a9d64
+  metadata.gz: e0c1397fca1de7c4cb3382fcf4278e03de645693
+  data.tar.gz: 533850a6c296ca0df8ba934ec277e99b5b155bd2
 SHA512:
-  metadata.gz: 200c29e2a64610cd4561ae127365f934b9cef0ef9dad264fed54d191bc3ec779eb22fe507d4d55c6373928a560c807dafa0cc0883243b0865723a62649349d40
-  data.tar.gz: 2a5b4a1e246b448e6fffe76c6fff8416298d22c4e46719109d6d9c2fca99c3f2ddb1cc3e70cb5b5bf9afbac388c3048b1b840c05952bcc0ab9e14a53c89b901f
+  metadata.gz: 15180ce5c9fd332023fbaefb28dc290a144885d23e8e1bc1d12a542dadece366da16c8f6c2078f3c3b405db25010c67b2ad372ba8a4f8cc97ee59703828f351f
+  data.tar.gz: 4a1c28339d84a87d19304ca7a4e85239dbadbbdda4009a08a1d14add1af58c38238626042996524310a2f994891cacf195aaed06edc59c7ac6b38a80d336f143

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.2.3

data/bin/serverinfo

@@ -48,6 +48,10 @@ log_server_details()
 
   echo "Logging release details"
   cat /etc/*-release
+
+  echo "Logging flydata versions"
+  gem list flydata
+  flydata version
 }
 
 log_server_details

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.2.2 ruby lib
+# stub: flydata 0.2.3 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.2"
+  s.version = "0.2.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-09-12"
+  s.date = "2014-09-16"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -87,6 +87,7 @@ Gem::Specification.new do |s|
     "spec/flydata/command/sender_spec.rb",
     "spec/flydata/command/sync_spec.rb",
     "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
+    "spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb",
     "spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
     "spec/flydata/heroku_spec.rb",
     "spec/flydata/parser/mysql/alter_table_parser_spec.rb",

data/lib/flydata/cli.rb

@@ -16,7 +16,11 @@ module Flydata
         if @args.size > 0
           first_arg = @args.shift
           cmd, sub_cmd = parse_command(first_arg)
-          cmd_cls = "Flydata::Command::#{cmd.capitalize}".constantize
+          begin
+            cmd_cls = "Flydata::Command::#{cmd.capitalize}".constantize
+          rescue
+            raise "Command not found: #{cmd}"
+          end
           # Command class can define options for each subcommand by defining method "slop_subcommandname"
           slop_method = sub_cmd ? "slop_#{sub_cmd}".to_sym : :slop
           options = cmd_cls.respond_to?(slop_method) ? cmd_cls.send(slop_method) : Slop.new(strict: true)

data/lib/flydata/command/setup.rb

@@ -27,6 +27,7 @@ following from the command line.
 
     $ source ~/.bashrc
 
+
 What's next?
 
     - Check data on Redshift (%s)
@@ -43,6 +44,11 @@ Thank you for using FlyData!
       INITIAL_SYNC_MESSAGE_TEMPLATE = <<-EOM
 FlyData Agent has been installed on your server successfully.
 
+To complete your installation and to add the `flydata` command, please run the
+following from the command line.
+
+    $ source ~/.bashrc
+
 
 What's next?
 
@@ -63,6 +69,11 @@ What's next?
       NO_DE_SYNC_CANCEL_MESSAGE_TEMPLATE = <<-EOM
 FlyData Agent has been installed on your server successfully.
 
+To complete your installation and to add the `flydata` command, please run the
+following from the command line.
+
+    $ source ~/.bashrc
+
 
 What's next?
 

data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb

@@ -15,13 +15,24 @@ module Mysql
 
     def process(record, normalized_query)
       emit_record(:alter_table, record, increment_table_rev: true) do
-        result = ParserProvider.parser(:mysql, :mysql_alter_table).new.parse(record['query'])
-        if result.nil?
-          $log.warn("Recieved not supported alter table query. query:'#{record['query']}'")
-          nil
-        else
-          result.tree
+        ret = nil
+        begin
+          result = ParserProvider.parser(:mysql, :mysql_alter_table).new.parse(record['query'])
+          if result.nil?
+            $log.warn("Received unsupported alter table query. query:'#{record['query']}'")
+          else
+            ret = result.tree
+          end
+        rescue => e
+          msg = <<EOS
+Received unsupported alter table query. query:'#{record['query']}'
+  Caused by error '#{e.to_s}'
+  Stacktrace :
+#{e.backtrace.join("\n")}
+EOS
+          $log.error(msg)
         end
+        ret
       end
     end
   end

data/lib/flydata/helpers.rb

@@ -18,25 +18,30 @@ module Flydata
 
     def usage_text
       text = ""
+      text += <<-EOM
+Fix the issue and try again.  If the problem continues, please contact support@flydata.com
+
+EOM
+      flydata_log = File.join(FLYDATA_HOME, 'flydata.log')
+      if File.exists?(flydata_log)
+        text += <<-EOM
+Also check the Agent log.
+   Log path: #{flydata_log}
+EOM
+      end
+      text += ""
       text += '-' * 64
       text += "\n"
       text += <<-EOM
 Usage: flydata COMMAND
-  setup                    # setup initially
   start                    # start flydata process
   stop                     # stop flydata process
   restart                  # restart flydata process
   status                   # check flydata process
   conf                     # show configuration
-  sync                     # initial sync(only for mysql database)
-
-If you encountered login or any other errors during setup,
-please setup flydata again by following commands.
-   source ~/.bashrc
-   flydata setup
-
-You can check the logs of sender(flydata) process.
-   Log path: #{File.join(FLYDATA_HOME, 'flydata.log')}
+  sync [tables]            # initial sync (only for mysql database)
+  sync:generate_table_ddl [tables] # Generate CREATE TABLE script for Redshift
+  sync:reset [tables]      # reset sync
       EOM
       text += "\n"
       text += '-' * 64

data/lib/flydata/parser/mysql/mysql_alter_table.treetop

@@ -196,7 +196,7 @@ grammar MysqlAlterTable
 
   rule null_opt
     'not'i sp 'null'i { def null_opt_option; { not_null: true }; end }
-    / sp 'null'i { def null_opt_option; { }; end }
+    / 'null'i { def null_opt_option; { }; end }
   end
   rule default_opt
     'default'i sp default_value { def default_opt_option; { default: default_value.default_value }; end }

data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb

@@ -0,0 +1,79 @@
+require_relative '../../../spec_helper'
+require 'flydata/fluent-plugins/mysql/alter_table_query_handler'
+
+module Mysql
+  describe AlterTableQueryHandler do
+    let(:sync_fm) do
+      r = double('sync_fm')
+      allow(r).to receive(:get_table_binlog_pos).and_return("mysql-bin.000065\t120")
+      r
+    end
+    let(:database) { "testdb" }
+    let(:table) { "foo" }
+    let(:context) do
+      r = double('context')
+      allow(r).to receive(:sync_fm).and_return(sync_fm)
+      allow(r).to receive(:database).and_return([database])
+      allow(r).to receive(:tables).and_return([table])
+      r
+    end
+    subject { described_class.new(context) }
+    describe '#process' do
+      let(:query) { "a_query" }
+      let(:record) do
+        r = double('record')
+        allow(r).to receive(:[]).with("db_name").and_return(database)
+        allow(r).to receive(:[]).with("query").and_return(query)
+        allow(r).to receive(:[]).with("table_name").and_return(table)
+        r
+      end
+      let(:normalized_query) { double('normalized_query') }
+      let(:parse_result) do
+        r = double('parse_result')
+        r
+      end
+      let(:parser) do
+        r = double('parser')
+        allow(r).to receive(:parse).and_return(parse_result)
+        r
+      end
+      before do
+        allow(MysqlAlterTableParser).to receive(:new).and_return(parser)
+        allow(subject).to receive(:acceptable_table?).and_return(true)
+        allow(subject).to receive(:acceptable_db?).and_return(true)
+        allow(subject).to receive(:check_empty_binlog)
+      end
+      shared_examples "a process method receiving an exception" do
+        let(:an_error) { "an error" }
+        it "returns nil with an error log" do
+          expect($log).to receive(:error).with(/Received unsupported alter table query.*Caused by error '#{an_error}'/m)
+
+          expect(subject.process(record, normalized_query)).to eq(nil)
+        end
+      end
+
+      context "when the parser#parse throws an exception" do
+        before do
+          expect(parser).to receive(:parse).and_raise(RuntimeError.new(an_error))
+        end
+        it_behaves_like "a process method receiving an exception"
+      end
+      context "when the parse result throws an exception" do
+        before do
+          expect(parse_result).to receive(:tree).and_raise(RuntimeError.new(an_error))
+        end
+        it_behaves_like "a process method receiving an exception"
+      end
+      context "when the parse result is nil" do
+        before do
+          expect(parser).to receive(:parse).and_return(nil)
+        end
+        it "returns nil with a warn log" do
+          expect($log).to receive(:warn).with(/Received unsupported alter table query\. query:'#{query}'/)
+
+          expect(subject.process(record, normalized_query)).to eq(nil)
+        end
+      end
+    end
+  end
+end

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -225,8 +225,23 @@ describe 'MysqlAlterTableParser' do
           })
         end
       end
-    end
 
+      context 'with null option' do
+        let(:length) { 45 }
+        let(:query) { "alter table test_table add column extra varchar(#{length}) null" }
+        it do
+          expect(subject).to eq({
+            type: :alter_table,
+            table_name: "test_table",
+            actions: [{
+              action: :add_column,
+              column: "extra",
+              type: "varchar(#{length * 3})",
+            }]
+          })
+        end
+      end
+    end
 
     context 'with alter table drop column syntax' do
       let(:query) { "alter table test_table drop column value" }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Koichi Fujikawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-12 00:00:00.000000000 Z
+date: 2014-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -452,6 +452,7 @@ files:
 - spec/flydata/command/sender_spec.rb
 - spec/flydata/command/sync_spec.rb
 - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
+- spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
 - spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb
 - spec/flydata/heroku_spec.rb
 - spec/flydata/parser/mysql/alter_table_parser_spec.rb