fluent-plugin-mysql-appender 0.4.9 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/example/mysql_multi_table_to_tresure_data.md +30 -13
  3. data/fluent-plugin-mysql-appender.gemspec +1 -1
  4. data/lib/fluent/plugin/in_mysql_appender_multi.rb +12 -15
  5. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 424553e560bb98d0602e2f00c13e6301d72967b5
4
- data.tar.gz: 189980b5c302be2016f59cc4834d92ce016b31d8
3
+ metadata.gz: 3fa294a4d025bbf2cde13a939dff99c60c589b32
4
+ data.tar.gz: a59ab31f182c72ce384a8183ca83d688f76af8e0
5
5
  SHA512:
6
- metadata.gz: b53d691fb0fd6c040b91d2d2f6dd49220beb381efcb4007e71507e70d40788143ffc49d7d52db06aa517d8fd0d8c597e3f4bf9d144472b23212b108a149cbde2
7
- data.tar.gz: dd33875581bd3784ff933f87f9f602d4da29087411b4c164739ee768bf51f398e9c3036d41bcd6e9e89302571ea98e665106232937faa4fe7b2fcf4ec0401e51
6
+ metadata.gz: a7f11ae093e4761e925112da22eca72f8668f124db5cf9c386eb3dea240f09f20211fd7c828a7e2942e96c04baa4f66289012faa74ba6048fe6f59218d5b024e
7
+ data.tar.gz: e1dc84259238aabe3629652c2d2884c284f9d596c65d6709c9216c027c4244b7837c72bc4f1009b795bc2f727688953e3c19464ac3b95ac1ad9c47aef3d4c732
data/example/mysql_multi_table_to_tresure_data.md CHANGED
@@ -2,6 +2,16 @@
2
2
 
3
3
  It is a guide to replicate multiple mysql table to treasure data.
4
4
 
5
+ ## environment variables
6
+
7
+ Please set environment variables.
8
+
9
+ ```
10
+ TD_APIKEY xxxxxxxx # Treasure data API key.
11
+ TD_ENDPOINT  api.treasuredata.com # Treasure data API endpoint. e.g. "api.treasuredata.com".
12
+ TD_DATABASE sample_db # Treasure data database name.
13
+ ```
14
+
5
15
  ## configuration
6
16
 
7
17
  ```
@@ -20,9 +30,6 @@ It is a guide to replicate multiple mysql table to treasure data.
20
30
 
21
31
  <match appender_multi.*.*>
22
32
  type tdlog
23
- endpoint your_td_endpoint
24
- apikey your_td_apikey
25
-
26
33
  auto_create_table
27
34
  buffer_type file
28
35
  buffer_path /var/log/td-agent/buffer/td
@@ -42,26 +49,36 @@ Sample "in_tables.yml" is below.
42
49
 
43
50
  ```
44
51
  - table_name: test_tbl1
45
- primary_key: id
46
- time_column: created_at
52
+ primary_key: id # incremental id
53
+ time_column: created_at # assigned to td's time column
47
54
  limit: 1000
48
55
  columns:
49
56
  - id
50
57
  - column1
51
58
  - column2
52
- last_id: -1
53
- td_database: sample_datasets
54
- entry_time: created_at # if this column is greater (now - delay), wait insert.
59
+ - created_at
55
60
  delay: 10s
61
+ entry_time: created_at # if this column is greater (now - delay), wait insert.
56
62
 
57
63
  - table_name: test_tbl2
58
- primary_key: id
59
- time_column: created_at
64
+ primary_key: id # incremental id
65
+ time_column: created_at # assigned to td's time column
60
66
  limit: 1000
61
67
  columns:
62
68
  - id
63
69
  - column1
64
70
  - column2
65
- last_id: -1
66
- td_database: sample_datasets
67
- ```
71
+ - created_at
72
+ delay: 10s
73
+ entry_time: created_at # if this column is greater (now - delay), wait insert.
74
+ ```
75
+
76
+ ```
77
+ select id, column1, column2, created_ad from test_tbl1 where id > {last_id} limit 1000
78
+ ```
79
+
80
+ ```
81
+ select id, column1, column2, created_ad from test_tbl2 where id > {last_id} limit 1000
82
+ ```
83
+
84
+ run query in each syncronize loops.
data/fluent-plugin-mysql-appender.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  # coding: utf-8
2
2
  Gem::Specification.new do |spec|
3
3
  spec.name = "fluent-plugin-mysql-appender"
4
- spec.version = "0.4.9"
4
+ spec.version = "0.5.0"
5
5
  spec.authors = ["TERASAKI Tsuyoshi"]
6
6
  spec.email = ["tsuyoshi_terasaki@realworld.jp"]
7
7
 
data/lib/fluent/plugin/in_mysql_appender_multi.rb CHANGED
@@ -68,9 +68,8 @@ module Fluent
68
68
  def poll(config)
69
69
  begin
70
70
  tag = format_tag(config)
71
- delay = Config.time_value(config['delay'] || 0)
72
71
  @mutex.synchronize {
73
- $log.info "mysql_appender_multi: polling start. :tag=>#{tag} :delay=>#{delay}"
72
+ $log.info "mysql_appender_multi: polling start. :tag=>#{tag}"
74
73
  }
75
74
  last_id = get_lastid(config)
76
75
  loop do
@@ -78,20 +77,18 @@ module Fluent
78
77
  start_time = Time.now
79
78
  db = get_connection
80
79
  db.query(get_query(config, last_id)).each do |row|
81
- if !config['entry_time'].nil? then
82
- entry_time = get_time(row[config['entry_time']])
83
- if (start_time - delay) > entry_time then
84
- if config['time_column'].nil? then
85
- td_time = Engine.now
86
- else
87
- td_time = get_time(row[config['time_column']]).to_i
88
- end
89
- row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
90
- router.emit(tag, td_time, row)
91
- rows_count += 1
92
- last_id = row[config['primary_key']]
93
- end
80
+ if rows_count > 0 && (last_id.to_i + 1) != row[config['primary_key']] then
81
+ break
94
82
  end
83
+ if config['time_column'].nil? then
84
+ td_time = Engine.now
85
+ else
86
+ td_time = get_time(row[config['time_column']]).to_i
87
+ end
88
+ row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
89
+ router.emit(tag, td_time, row)
90
+ rows_count += 1
91
+ last_id = row[config['primary_key']]
95
92
  end
96
93
  db.close
97
94
  elapsed_time = sprintf("%0.02f", Time.now - start_time)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mysql-appender
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.9
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - TERASAKI Tsuyoshi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-05 00:00:00.000000000 Z
11
+ date: 2017-03-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd