flydata 0.2.25 → 0.2.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 950ac222e77c45b7dfb9a65f2921d612d78d0a56
-  data.tar.gz: 50e414cced803f9a3ebf279ebef5c5b0409607b6
+  metadata.gz: e9ac346de420a785776ae309674640dd0afe3ff3
+  data.tar.gz: b9dcd9d82316b6906517ab008cc02e1024e8851e
 SHA512:
-  metadata.gz: c9ead00fb45d5f022632e7e59d9b128e9c44b7878e85c6a9796aae0b77e9f5d41bcb83812ef5c32058a1372d5c87be5dc6e9febe4d6416d7e81e2ba8564c2dc3
-  data.tar.gz: f1f36e0bb8354c5580e8be6df19f7f2bf4c23714420b28d9a0a74cd84d9d83dbc692c371503db66376dd42825d9ddab9ff505490f63169d11af818cb59f3b613
+  metadata.gz: f2c936dab2a85f0b2b575900218b03472874be5061c2dc72f89753f37ef7f166f31bfd4e4ffb1f47f367b12b1aac3367cc6d10f766a219a45831787456c512c8
+  data.tar.gz: 138786e380f022dd2ad70a3a96e20a88a379a36fb4ba1ed4636605d81a98c4f65ae86f4f143c87cc0ae09758c5a5a00487d2d60ff78327ceb1e8cfd58a6d8eb8

data/VERSION

@@ -1 +1 @@
-0.2.25
+0.2.26

data/flydata-core/circle.yml

@@ -0,0 +1,3 @@
+machine:
+  ruby:
+    version: 2.0.0-p481

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -40,6 +40,7 @@ class RedshiftTableDef
 
     begin
       tabledef = ""
+      tabledef += create_schema_sql(schema_name) if options[:flydata_ctl_table] && !schema_name.to_s.empty?
       tabledef += create_flydata_ctl_table_sql(schema_name) if options[:flydata_ctl_table]
       tabledef += create_table_sql(flydata_tabledef, schema_name) unless options[:ctl_only]
       tabledef += comment_sql(flydata_tabledef, schema_name) unless options[:ctl_only]
@@ -51,6 +52,9 @@ class RedshiftTableDef
   end
 
   FLYDATA_CTL_COLUMNS_TABLE = "flydata_ctl_columns"
+  CREATE_SCHEMA_SQL = <<EOS
+CREATE SCHEMA "%s";
+EOS
   CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
 CREATE TABLE %s(
   id integer NOT NULL IDENTITY(1,1),
@@ -71,6 +75,10 @@ EOS
     table_name_for_ddl(FLYDATA_CTL_COLUMNS_TABLE, schema_name)
   end
 
+  def self.create_schema_sql(schema_name)
+    CREATE_SCHEMA_SQL % schema_name
+  end
+
   def self.create_flydata_ctl_table_sql(schema_name)
     # No drop table here intentionally because losing the data is fatal.
     tbl = flydata_ctl_table_for_ddl(schema_name)

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -111,6 +111,7 @@ EOT
 
       it 'should preappend schema name to table name' do
         expect(subject).to eq( <<EOT.strip )
+CREATE SCHEMA "test_schema";
 CREATE TABLE "test_schema"."flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.2.25 ruby lib
+# stub: flydata 0.2.26 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.25"
+  s.version = "0.2.26"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2014-12-21"
+  s.date = "2014-12-31"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -35,6 +35,7 @@ Gem::Specification.new do |s|
     "flydata-core/.travis.yml",
     "flydata-core/Gemfile",
     "flydata-core/Gemfile.lock",
+    "flydata-core/circle.yml",
     "flydata-core/lib/flydata-core.rb",
     "flydata-core/lib/flydata-core/core_ext.rb",
     "flydata-core/lib/flydata-core/core_ext/module.rb",

data/lib/flydata/cli.rb

@@ -29,7 +29,10 @@ module Flydata
           cmd, sub_cmd = parse_command(first_arg)
           begin
             cmd_cls = "Flydata::Command::#{cmd.capitalize}".constantize
-          rescue
+          rescue NameError 
+            # NameError is raised when the constant is not defined in its target
+            # file, that is, command not found.
+            # Other exceptions mean that some other error happended.
             raise "Command not found: #{cmd}"
           end
           # Command class can define options for each subcommand by defining method "slop_subcommandname"

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -78,10 +78,15 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       mysql_url: mysql_url, database: @database, tables: @tables)
     table_meta.update
 
+    table_revs = tables.inject({}) do |h, table_name|
+      h[table_name] = @sync_fm.table_rev(table_name)
+      h
+    end
+
     @context = Mysql::Context.new(
       database: @database, tables: @tables,
       tag: @tag, sync_fm: @sync_fm, omit_events: @omit_events,
-      table_meta: table_meta,
+      table_meta: table_meta, table_revs: table_revs,
     )
     @record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
     @idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval)

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -70,13 +70,14 @@ module Mysql
       raise "Missing table name. #{record}" if table.to_s.empty?
       return unless acceptable_table?(record, table) && acceptable_event?(type, table)
 
-      table_rev = @context.sync_fm.table_rev(table)
+      table_rev = @context.table_revs[table]
       position = record['next_position'] - record['event_length']
 
       # Add common information to each record
       records.each do |r|
         if opt[:increment_table_rev]
           table_rev = @context.sync_fm.increment_table_rev(table, table_rev)
+          @context.table_revs[table] = table_rev
         end
         r[TYPE] = type
         r[RESPECT_ORDER] = true

data/lib/flydata/fluent-plugins/mysql/context.rb

@@ -1,7 +1,7 @@
 module Mysql
   class Context
     MANDATORY_OPTS = [
-      :database, :tables, :tag, :sync_fm, :omit_events, :table_meta
+      :database, :tables, :tag, :sync_fm, :omit_events, :table_meta, :table_revs,
     ]
     OPTIONAL_OPTS = [
       :current_binlog_file

data/lib/flydata/sync_file_manager.rb

@@ -98,8 +98,14 @@ module Flydata
         f = @table_position_files[table_name]
         seq = f.read
         seq = seq.to_i + 1
-        yield(seq)
-        f.rewind
+        begin
+          yield(seq)
+        ensure
+          # when an error happened in yield, the sequence number should remain
+          # as is.  For the next call to read the value correctly, the position
+          # must be rewound.
+          f.rewind
+        end
         f.truncate(0)
         f.write(seq)
         f.flush

data/spec/flydata/sync_file_manager_spec.rb

@@ -156,6 +156,37 @@ module Flydata
       describe '#binlog_path' do
         it { expect(default_sfm.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
       end
+
+      describe '#increment_and_save_table_position' do
+        let(:test_table) { 'test_table' }
+        before do
+          table_pos_dir = SyncFileManager::TABLE_POSITIONS_DIR
+          FileUtils.mkdir_p table_pos_dir
+          table_pos_file = "#{table_pos_dir}/#{test_table}.pos"
+          FileUtils.rm(table_pos_file) if File.exist?(table_pos_file)
+        end
+        subject { default_sfm }
+        context 'when an exception happens in the block' do
+          let(:first_pos) { 1 }
+          let(:last_pos) { first_pos + 1 }
+          before do
+            subject.increment_and_save_table_position(test_table) do |seq|
+              expect(seq).to eq first_pos
+            end
+            begin
+              subject.increment_and_save_table_position(test_table) do |seq|
+                expect(seq).to eq last_pos
+                raise "error"
+              end
+            rescue
+              # intentionally left blank
+            end
+          end
+          it 'yields with the previous value' do
+            expect{|b| subject.increment_and_save_table_position(test_table, &b)}.to yield_with_args(last_pos)
+          end
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.25
+  version: 0.2.26
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-21 00:00:00.000000000 Z
+date: 2014-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -438,6 +438,7 @@ files:
 - flydata-core/.travis.yml
 - flydata-core/Gemfile
 - flydata-core/Gemfile.lock
+- flydata-core/circle.yml
 - flydata-core/lib/flydata-core.rb
 - flydata-core/lib/flydata-core/core_ext.rb
 - flydata-core/lib/flydata-core/core_ext/module.rb