flydata 0.6.3 → 0.6.4

data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_AUTO_INCREMENT_keyword.dump

@@ -0,0 +1,43 @@
+-- MySQL dump 10.13  Distrib 5.5.37, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: flydata_sync
+-- ------------------------------------------------------
+-- Server version	5.5.37-0ubuntu0.12.04.1-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `sample1`
+--
+
+DROP TABLE IF EXISTS `sample1`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `sample1` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `title` varchar(256) DEFAULT NULL COMMENT 'This column is not AUTO_INCREMENT column.',
+  `name` text,
+  `num` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-07-15 10:07:45

data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_not_null_keyword.dump

@@ -0,0 +1,43 @@
+-- MySQL dump 10.13  Distrib 5.5.37, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: flydata_sync
+-- ------------------------------------------------------
+-- Server version	5.5.37-0ubuntu0.12.04.1-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `sample1`
+--
+
+DROP TABLE IF EXISTS `sample1`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `sample1` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `title` varchar(256) DEFAULT NULL COMMENT 'This column is not null column.',
+  `name` text,
+  `num` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-07-15 10:07:45

data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unique_keyword.dump

@@ -0,0 +1,43 @@
+-- MySQL dump 10.13  Distrib 5.5.37, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: flydata_sync
+-- ------------------------------------------------------
+-- Server version	5.5.37-0ubuntu0.12.04.1-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `sample1`
+--
+
+DROP TABLE IF EXISTS `sample1`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `sample1` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `title` varchar(256) DEFAULT NULL COMMENT 'This column is not unique column.',
+  `name` text,
+  `num` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-07-15 10:07:45

data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unsigned_keyword.dump

@@ -0,0 +1,43 @@
+-- MySQL dump 10.13  Distrib 5.5.37, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: flydata_sync
+-- ------------------------------------------------------
+-- Server version	5.5.37-0ubuntu0.12.04.1-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `sample1`
+--
+
+DROP TABLE IF EXISTS `sample1`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `sample1` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `title` varchar(256) DEFAULT NULL COMMENT 'This column is not unsigned column.',
+  `name` text,
+  `num` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-07-15 10:07:45

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'flydata-core/table_def'
+require 'timecop'
 
 module FlydataCore
 module TableDef
@@ -28,9 +29,23 @@ describe RedshiftTableDef do
 
     let(:schema_prefix) { "" }
 
+    let(:drop_table_subquery) { <<EOT }
+DROP TABLE IF EXISTS #{schema_prefix}"test_table_flydata20160125232857";
+CREATE TABLE IF NOT EXISTS #{schema_prefix}"test_table" (
+  "id" int4,
+  "age" int8,
+  "value" varchar(max),
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
+ALTER TABLE #{schema_prefix}"test_table" RENAME TO "test_table_flydata20160125232857";
+EOT
+
+    let(:drop_backup_table_subquery) { <<EOT.strip }
+DROP TABLE IF EXISTS #{schema_prefix}"test_table_flydata20160125232857";
+EOT
+
     let(:create_table_queries) { <<EOT.strip }
-DROP TABLE IF EXISTS #{schema_prefix}"test_table";
-CREATE TABLE #{schema_prefix}"test_table" (
+#{drop_table_subquery}CREATE TABLE #{schema_prefix}"test_table" (
   "id" int4,
   "age" int8,
   "value" varchar(max),
@@ -79,9 +94,13 @@ EOT
 CREATE SCHEMA IF NOT EXISTS "test_schema";
 EOT
 
+    before do
+      Timecop.freeze(Time.local(2016, 1, 25, 23, 28, 57))
+    end
+
     context 'with simple flydatadef' do
       it 'should return ddl' do
-        expect(subject).to eq "#{create_table_queries}\n#{flydata_ctl_update}"
+        expect(subject).to eq "BEGIN;\n#{create_table_queries}\n#{flydata_ctl_update}\nCOMMIT;\n#{drop_backup_table_subquery}\n"
       end
     end
 
@@ -89,7 +108,18 @@ EOT
       let(:option) { { flydata_ctl_columns: true } }
 
       it 'should return ddl including flydata_ctl_columns creation' do
-        expect(subject).to eq "#{flydata_ctl_create}\n#{create_table_queries}\n#{flydata_ctl_update}"
+        expect(subject).to eq "#{flydata_ctl_create}\nBEGIN;\n#{create_table_queries}\n#{flydata_ctl_update}\nCOMMIT;\n#{drop_backup_table_subquery}\n"
+      end
+    end
+
+    context 'when skip_drop_table option is true' do
+      before do
+        option.merge!(skip_drop_table: true)
+      end
+
+      let(:drop_table_subquery) { ""}
+      it 'should return ddl' do
+        expect(subject).to eq "BEGIN;\n#{create_table_queries}\n#{flydata_ctl_update}\nCOMMIT;\n"
       end
     end
 
@@ -97,7 +127,7 @@ EOT
       let(:value_column) { value_column_hash.merge(comment: "helloworld") }
 
       it 'should add comment creation ddl' do
-        expect(subject).to eq "#{create_table_queries}\n#{comment_subquery}\n#{flydata_ctl_update}"
+        expect(subject).to eq "BEGIN;\n#{create_table_queries}\n#{comment_subquery}\n#{flydata_ctl_update}\nCOMMIT;\n#{drop_backup_table_subquery}\n"
       end
     end
 
@@ -107,12 +137,15 @@ EOT
       let(:schema_prefix) { '"test_schema".' }
 
       it 'should preappend schema name to table name' do
-        expect(subject).to eq <<EOT.strip
+        expect(subject).to eq <<EOT
 #{create_schema_query}
 #{flydata_ctl_create}
+BEGIN;
 #{create_table_queries}
 #{comment_subquery}
 #{flydata_ctl_update}
+COMMIT;
+#{drop_backup_table_subquery}
 EOT
       end
     end
@@ -121,7 +154,7 @@ EOT
       let(:value_column) { value_column_hash.merge(charset: "ISO_8859_1") }
       let(:value_type) { "#{value_type_body} cs:ISO_8859_1" }
       it 'should return ddl' do
-        expect(subject).to eq "#{create_table_queries}\n#{flydata_ctl_update}"
+        expect(subject).to eq "BEGIN;\n#{create_table_queries}\n#{flydata_ctl_update}\nCOMMIT;\n#{drop_backup_table_subquery}\n"
       end
     end
   end
@@ -540,7 +573,7 @@ EOT
         default_charset: "UTF_8"} }
     let(:schema_name) { nil }
 
-    let(:expected_sql) { <<EOS.strip }
+    let(:expected_sql) { <<EOS }
 DELETE FROM "flydata_ctl_columns" WHERE table_name = '#{table_name}';
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('#{table_name}', '#{expected_column_name}', 'varchar(12)', 1);

data/lib/flydata/cli.rb

@@ -2,6 +2,7 @@ require 'slop'
 require 'flydata/command_loggable'
 require 'flydata/errors'
 require 'flydata/helpers'
+require 'flydata/source'
 
 module Flydata
   class Cli
@@ -70,11 +71,16 @@ module Flydata
     end
     COMMAND_DIR = 'flydata/command'
     def require_command_source(cmd)
-      source_path = File.join(COMMAND_DIR, cmd)
-      begin
-        require source_path
-      rescue LoadError
-        # ignore
+      command_dirs = [COMMAND_DIR] + Source.command_dir_paths
+      command_dirs.each do |command_dir|
+        source_path = File.join(command_dir, cmd)
+        result = begin
+                   require source_path
+                   :success
+                 rescue LoadError
+                   :fail
+                 end
+        break if result == :success
       end
     end
   end

data/lib/flydata/command/base.rb

@@ -1,6 +1,7 @@
 require 'slop'
 require 'flydata/api_client'
 require 'flydata/command_loggable'
+require 'flydata/source'
 require 'flydata/command/exclusive_runnable'
 require 'flydata/preference/data_entry_preference'
 
@@ -42,10 +43,14 @@ module Flydata
 
       def data_entry
         @de ||= retrieve_data_entries.first
-        raise "There are no data entries." unless @de
+        raise "No data entry exists.  Please set one up on the FlyData Console (#{dashboard_url})" unless @de
         @de
       end
 
+      def source
+        @source ||= Source.create(data_entry)
+      end
+
       def register_crontab
         data_entries = retrieve_data_entries
         if data_entries.any?{|e| e['log_deletion']}
@@ -55,6 +60,14 @@ module Flydata
         end
       end
 
+      def dashboard_url
+        "#{flydata.flydata_api_host}/dashboard"
+      end
+
+      def redshift_console_url
+        "#{flydata.flydata_api_host}/redshift_clusters/query/new"
+      end
+
       # print console
       def newline; puts end
       def separator(str="=")

data/lib/flydata/command/exclusive_runnable.rb

@@ -11,6 +11,21 @@ module ExclusiveRunnable
   end
 
   module ClassMethods
+    # messages
+    COMMAND_IS_ALREADY_RUNNING = <<EOT
+Command `%s` is already running or terminated abnormally. (pid:%s)
+Wait until the command completes or stop the command.
+EOT
+    DATA_INCONSISTENCY_MAY_OCCUR = <<EOT
+
+!! Command `%s` is still running. (pid:%s)
+We highly recommend to *stop the process* before run a new command.
+If two commands run concurrently, DATA INCONSISTENCY MAY OCCUR.
+
+EOT
+          CONCURRENT_RUN = <<EOT
+Run command:"%s" while a process for the previous command:"%s" still exists (pid:%s)
+EOT
     def run_exclusive(method, options = {})
       saved_method = :"__#{method}"
       alias_method saved_method, method
@@ -19,18 +34,27 @@ module ExclusiveRunnable
         result = nil
         exclusive_run_info = nil
         begin
-          info = self.class.load_exclusive_run_info
-          if info
-            #TODO If the previous command terminates abnormally, 
-            # 'exclusive_run.info' file won't be deleted even the pid 
-            # doesn't exist anymore. 
-            # This prevents execution of any commands afterwards, and 
-            # the user will see this misleading error message. 
-            # Introducing "--force-run" option can be a solution. 
-
-            raise "Command `#{info['command']}` is already running.  Wait until the command completes or stop the command.  (pid:#{info['pid']})"
+          new_command = self.class.command(method, options)
+          new_pid = Process.pid
+          old_exclusive_run_info = self.class.load_exclusive_run_info
+          if old_exclusive_run_info
+            old_command = old_exclusive_run_info['command']
+            old_pid = old_exclusive_run_info['pid']
+
+            raise COMMAND_IS_ALREADY_RUNNING % [ old_command, old_pid ] if !opts.force_run?
+
+            #`force-run` option is specified.
+            #Override old `exclusive_run.info` file and run a command.
+            if self.class.process_exist?(old_pid)
+              log_warn_stderr DATA_INCONSISTENCY_MAY_OCCUR % [ old_command, old_pid ]
+              exit unless ask_yes_no("Do you proceed?", false)
+            end
+            $log.info CONCURRENT_RUN % [ new_command, old_command, old_pid ]
+            self.class.delete_exclusive_run_info
           end
-          exclusive_run_info = { command: self.class.command(method, options), pid: Process.pid }
+
+          #Run a command
+          exclusive_run_info = { command: new_command, pid: new_pid }
           self.class.save_exclusive_run_info(exclusive_run_info)
           result = send(saved_method, *args)
         ensure
@@ -65,7 +89,13 @@ module ExclusiveRunnable
       path = exclusive_run_info_path
       File.delete(path) if File.exists?(path)
     end
-
+    def process_exist?(pid)
+      # Returns true if the process is running.
+      Process.getpgid(pid.to_i)
+      true
+    rescue Errno::ESRCH
+      false
+    end
     def command(subcommand, options = {})
       return options[:command] if options[:command]
 

data/lib/flydata/command/helper.rb

@@ -49,7 +49,7 @@ module Flydata
 
       def stop(options = {})
         if running?
-          run_command(stop_cmd)
+          run_command(stop_cmd, options)
         else
           log_info_stdout("Helper is not running.", {}, options)
         end
@@ -66,7 +66,7 @@ module Flydata
       def restart(options = {})
         if running?
           log_info_stdout("Restarting Helper.", {}, options)
-          run_command(stop_cmd)
+          run_command(stop_cmd, options)
         end
         raw_start
       end
@@ -74,7 +74,7 @@ module Flydata
        def reload(options = {})
          if running?
            log_info_stdout("Reloading Helper.", {}, options)
-           run_command(kill_hup_cmd)
+           run_command(kill_hup_cmd, options)
          end
        end
 
@@ -95,14 +95,14 @@ module Flydata
         end
       end
 
-      def run_command(cmd)
+      def run_command(cmd, options)
         o, e, s = Util::Shell.run_cmd(cmd)
         log_error_stderr(e) if not e.to_s.empty?
         success = (s.to_i == 0)
         if success
-          log_info_stdout("Done")
+          log_info_stdout("Done", {}, options)
         else
-          log_error_stderr("Failed")
+          log_error_stderr("Failed", {}, options)
         end
         success
       end