fluent-plugin-mysql-appender 0.3.4 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b084608ea2ad992303897931c4c4383ca24d0f1b
4
- data.tar.gz: e5f1ca6447a7e27125509397811ab01eb8a76e1f
3
+ metadata.gz: b9f6408d1a7b224768ddef9852ce5f68344b15c7
4
+ data.tar.gz: 628c15bf44d0d25a1495652f4bc5aa291dad8749
5
5
  SHA512:
6
- metadata.gz: 5e1ea6d7d009424fde9f0fbe89c9c9c782bc4494a54b0ea8ed919415568a66be2b36f8dcc1882920a8c1134db881a4490f3ede993ea8cb4bf25477399632ce96
7
- data.tar.gz: e74dda81a5b493b057fe72ef4ad1188ccd02067fc3386b4f12a0c68e60a6d1c124c1464bb1e16a754e0cd82bb124ff665a128d0795ce33a43f5cc33f22b4cbeb
6
+ metadata.gz: 0414ae7c3bb515fc587e1e950da5e543787c98721f33a950eb086ddfd8b6056724bdb157c9ff666dbb2632ed48ef8e8af337d9c16614415610972ecce2f92a6e
7
+ data.tar.gz: a4813bb12472e8c40134dc46d8ae9e9e3983dda2c410ebac99277c4f2a8c847e20e17543bf0a5f8a0178a7e4f09ac77d17c0c2cab51f19c450429801b22a443e
@@ -50,7 +50,6 @@ Sample "in_tables.yml" is below.
50
50
  - column1
51
51
  - column2
52
52
  last_id: -1
53
- buffer: 10 # last 10 records don't append (default 0).
54
53
  td_database: sample_datasets
55
54
 
56
55
  - table_name: test_tbl2
@@ -62,6 +61,5 @@ Sample "in_tables.yml" is below.
62
61
  - column1
63
62
  - column2
64
63
  last_id: -1
65
- buffer: 10 # last 10 records don't append (default 0).
66
64
  td_database: sample_datasets
67
65
  ```
@@ -25,7 +25,6 @@ It is a guide to replicate single mysql table to treasure data.
25
25
  time_column created_at # specify TIME column.
26
26
  limit 1000 # query limit
27
27
  last_id -1 # specify primary_key start
28
- buffer 10 # last 10 records don't append (default 0).
29
28
  </source>
30
29
 
31
30
  <match appender.*.*>
@@ -1,7 +1,7 @@
1
1
  # coding: utf-8
2
2
  Gem::Specification.new do |spec|
3
3
  spec.name = "fluent-plugin-mysql-appender"
4
- spec.version = "0.3.4"
4
+ spec.version = "0.3.5"
5
5
  spec.authors = ["TERASAKI Tsuyoshi"]
6
6
  spec.email = ["tsuyoshi_terasaki@realworld.jp"]
7
7
 
@@ -26,7 +26,6 @@ module Fluent
26
26
  config_param :primary_key, :string, :default => 'id'
27
27
  config_param :interval, :string, :default => '1m'
28
28
  config_param :tag, :string, :default => nil
29
- config_param :buffer, :integer, :default => 0
30
29
 
31
30
  def configure(conf)
32
31
  super
@@ -64,27 +63,22 @@ module Fluent
64
63
  start_time = Time.now
65
64
  select_query = @query.gsub(/"/,'') + " where #{primary_key} > #{last_id} order by #{primary_key} asc limit #{limit}"
66
65
  rows, con = query(select_query, con)
67
- if @buffer < rows.size then
68
- if index == (rows.size - @buffer) then
69
- break
66
+ rows.each_with_index do |row, index|
67
+ tag = format_tag(@tag, {:event => :insert})
68
+ if @time_column.nil? then
69
+ td_time = Engine.now
70
+ else
71
+ if row[@time_column].kind_of?(Time) then
72
+ td_time = row[@time_column].to_i
73
+ else
74
+ td_time = Time.parse(row[@time_column].to_s).to_i
75
+ end
70
76
  end
71
- rows.each_with_index do |row, index|
72
- tag = format_tag(@tag, {:event => :insert})
73
- if @time_column.nil? then
74
- td_time = Engine.now
75
- else
76
- if row[@time_column].kind_of?(Time) then
77
- td_time = row[@time_column].to_i
78
- else
79
- td_time = Time.parse(row[@time_column].to_s).to_i
80
- end
81
- end
82
- row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
83
- router.emit(tag, td_time, row)
84
- rows_count += 1
85
- if index == rows.size - @buffer - 1
86
- @last_id = row[@primary_key]
87
- end
77
+ row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
78
+ router.emit(tag, td_time, row)
79
+ rows_count += 1
80
+ if index == rows.size - 1
81
+ @last_id = row[@primary_key]
88
82
  end
89
83
  end
90
84
  con.close
@@ -29,11 +29,11 @@ module Fluent
29
29
  @interval = Config.time_value(@interval)
30
30
 
31
31
  if @yaml_path.nil?
32
- raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter."
32
+ raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter or file not found."
33
33
  end
34
34
 
35
35
  if !File.exist?(@yaml_path)
36
- raise Fluent::ConfigError, "mysql_appender_multi: No such file in 'yaml_path'."
36
+ raise Fluent::ConfigError, "mysql_appender_multi: 'yaml_path' No such file."
37
37
  end
38
38
 
39
39
  if @tag.nil?
@@ -66,7 +66,6 @@ module Fluent
66
66
  def poll(config)
67
67
  begin
68
68
  tag = format_tag(config)
69
- buffer = config['buffer'] || 0
70
69
  @mutex.synchronize {
71
70
  $log.info "mysql_replicator_multi: polling start. :tag=>#{tag}"
72
71
  }
@@ -76,26 +75,21 @@ module Fluent
76
75
  rows_count = 0
77
76
  start_time = Time.now
78
77
  rows, con = query(get_query(config, last_id), con)
79
- if buffer < rows.size then
80
- rows.each_with_index do |row, index|
81
- if index == (rows.size - buffer) then
82
- break
83
- end
84
- if config['time_column'].nil? then
85
- td_time = Engine.now
86
- else
87
- if row[config['time_column']].kind_of?(Time) then
88
- td_time = row[config['time_column']].to_i
89
- else
90
- td_time = Time.parse(row[config['time_column']].to_s).to_i
91
- end
92
- end
93
- row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
94
- router.emit(tag, td_time, row)
95
- rows_count += 1
96
- if index == (rows.size - buffer - 1)
97
- last_id = row[config['primary_key']]
98
- end
78
+ rows.each_with_index do |row, index|
79
+ if config['time_column'].nil? then
80
+ td_time = Engine.now
81
+ else
82
+ if row[config['time_column']].kind_of?(Time) then
83
+ td_time = row[config['time_column']].to_i
84
+ else
85
+ td_time = Time.parse(row[config['time_column']].to_s).to_i
86
+ end
87
+ end
88
+ row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
89
+ router.emit(tag, td_time, row)
90
+ rows_count += 1
91
+ if index == rows.size - 1
92
+ last_id = row[config['primary_key']]
99
93
  end
100
94
  end
101
95
  con.close
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mysql-appender
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.4
4
+ version: 0.3.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - TERASAKI Tsuyoshi