fluent-plugin-pgjson 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a690b321412f2b0fe7dd2537e38c284b3d6753ee
4
- data.tar.gz: abed844423a52e625fc1798231224df083b7e8ba
3
+ metadata.gz: 4dfa8ba1bbd21e1ffa84476064f2cb0eced7e5c8
4
+ data.tar.gz: 7522f3e3b4639c1022f58f41ee9b568fea58827a
5
5
  SHA512:
6
- metadata.gz: 933ee51523a63e67430316e12b0c22119d51d1e20a659e6b909bfdd63e8e6852c38f56cf85f6737b7f436e4915570617db6bcba9dc3e44a88962b669410b2e8a
7
- data.tar.gz: 9f9b7c82bcc74e005b03abe6e98b6e496821407a4adbc8a9314702cc090ac373b93ec7b03bd2195d88f17db8bbefe60c9c3edb1e997d18f66ab0eed0d3929df2
6
+ metadata.gz: d3ff0bb506bac3398d9f02e5c3023babc3a7534261de3e05cb0c06738c6cc586f1f94578c3fa9e4fba91468682d1cb7aca129fefff2fed23dea84543003862ab
7
+ data.tar.gz: d9acbeca7d5d942ec7f1454716e3b69750b25bdc83416253f86ca1c04e921774b58f968c0cedadee06ad82a2be11cb7951551d95523dd27f830e373f21b73490
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-pgjson"
6
- s.version = "0.0.5"
6
+ s.version = "0.0.6"
7
7
  s.authors = ["OKUNO Akihiro"]
8
8
  s.email = ["choplin.choplin@gmail.com"]
9
9
  s.homepage = "https://github.com/choplin/fluent-plugin-pgjson"
@@ -13,7 +13,7 @@ class PgJsonOutput < Fluent::BufferedOutput
13
13
  config_param :time_col , :string , :default => 'time'
14
14
  config_param :tag_col , :string , :default => 'tag'
15
15
  config_param :record_col , :string , :default => 'record'
16
- config_param :messagepack, :bool , :default => false
16
+ config_param :msgpack , :bool , :default => false
17
17
 
18
18
  def initialize
19
19
  super
@@ -38,16 +38,18 @@ class PgJsonOutput < Fluent::BufferedOutput
38
38
  end
39
39
 
40
40
  def write(chunk)
41
+ init_connection
41
42
  begin
42
- init_connection
43
43
  @conn.exec("COPY #{@table} (#{@tag_col}, #{@time_col}, #{@record_col}) FROM STDIN WITH DELIMITER E'\\x01'")
44
44
  chunk.msgpack_each do |tag, time, record|
45
45
  @conn.put_copy_data "#{tag}\x01#{Time.at(time).to_s}\x01#{record_value(record)}\n"
46
46
  end
47
47
  @conn.put_copy_end
48
48
  rescue
49
- @conn.close()
50
- @conn = nil
49
+ if ! @conn.nil?
50
+ @conn.close()
51
+ @conn = nil
52
+ end
51
53
  raise "failed to send data to postgres: #$!"
52
54
  end
53
55
  end
@@ -74,7 +76,9 @@ class PgJsonOutput < Fluent::BufferedOutput
74
76
  if @msgpack
75
77
  "\\#{@conn.escape_bytea(record.to_msgpack)}"
76
78
  else
77
- record.to_json
79
+ json = record.to_json
80
+ json.gsub!(/\\/){ '\\\\' }
81
+ json
78
82
  end
79
83
  end
80
84
  end
@@ -1,46 +1,133 @@
1
+ require 'pg'
2
+ require 'securerandom'
1
3
  require 'helper'
2
4
 
3
5
  class PgJsonOutputTest < Test::Unit::TestCase
4
- def setup
5
- Fluent::Test.setup
6
- end
6
+ HOST = "localhost"
7
+ PORT = 5432
8
+ DATABASE = "postgres"
9
+ TABLE = "test_fluentd_#{SecureRandom.hex}"
10
+ USER = "postgres"
11
+ PASSWORD = "postgres"
12
+
13
+ TIME_COL = "time"
14
+ TAG_COL = "tag"
15
+ RECORD_COL = "record"
7
16
 
8
17
  CONFIG = %[
9
18
  type pgjson
10
- host localhost
11
- port 5432
12
- database fluentd
13
- table fluentd
14
- user postgres
15
- password postgres
16
- time_col time
17
- tag_col tag
18
- record_col record
19
+ host #{HOST}
20
+ port #{PORT}
21
+ database #{DATABASE}
22
+ table #{TABLE}
23
+ user #{USER}
24
+ password #{PASSWORD}
25
+ time_col #{TIME_COL}
26
+ tag_col #{TAG_COL}
27
+ record_col #{RECORD_COL}
19
28
  ]
20
29
 
21
- def create_driver(conf = CONFIG, tag='test')
22
- Fluent::Test::BufferedOutputTestDriver.new(Fluent::PgJsonOutput, tag).configure(conf)
30
+ def setup
31
+ Fluent::Test.setup
32
+ end
33
+
34
+ def create_driver(conf = CONFIG, tag = 'test')
35
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::PgJsonOutput).configure(conf)
23
36
  end
24
37
 
25
38
  def test_configure
26
39
  d = create_driver
27
40
 
28
- assert_equal "localhost", d.instance.host
29
- assert_equal 5432, d.instance.port
30
- assert_equal "fluentd", d.instance.database
31
- assert_equal "fluentd", d.instance.table
32
- assert_equal "postgres", d.instance.user
33
- assert_equal "postgres", d.instance.password
34
- assert_equal "time", d.instance.time_col
35
- assert_equal "tag", d.instance.tag_col
36
- assert_equal "record", d.instance.record_col
41
+ assert_equal HOST, d.instance.host
42
+ assert_equal PORT, d.instance.port
43
+ assert_equal DATABASE, d.instance.database
44
+ assert_equal TABLE, d.instance.table
45
+ assert_equal USER, d.instance.user
46
+ assert_equal PASSWORD, d.instance.password
47
+ assert_equal TIME_COL, d.instance.time_col
48
+ assert_equal TAG_COL, d.instance.tag_col
49
+ assert_equal RECORD_COL, d.instance.record_col
37
50
  end
38
51
 
39
- def test_format
40
- d = create_driver
52
+ def test_write
53
+ with_connection do |conn|
54
+ tag = 'test'
55
+ time = Time.parse("2014-12-26 07:58:37 UTC")
56
+ record = {"a"=>1}
57
+
58
+ d = create_driver(CONFIG, tag)
59
+ d.emit(record, time.to_i)
60
+ d.run
61
+ wait_for_data(conn)
62
+
63
+ res = conn.exec("select * from #{TABLE}")[0]
64
+ assert_equal res[TAG_COL], tag
65
+ assert_equal Time.parse(res[TIME_COL]), time
66
+ assert_equal res[RECORD_COL], record.to_json
67
+ end
41
68
  end
42
69
 
43
- def test_write
44
- d = create_driver
70
+ def test_escape_of_backslash
71
+ with_connection do |conn|
72
+ tag = 'test'
73
+ time = Time.parse("2014-12-26 07:58:37 UTC")
74
+ record = {"a"=>"\"foo\""}
75
+
76
+ d = create_driver(CONFIG, tag)
77
+ d.emit(record, time.to_i)
78
+ d.run
79
+ wait_for_data(conn)
80
+
81
+ res = conn.exec("select * from #{TABLE}")[0]
82
+ assert_equal res[TAG_COL], tag
83
+ assert_equal Time.parse(res[TIME_COL]), time
84
+ assert_equal res[RECORD_COL], record.to_json
85
+ end
86
+ end
87
+
88
+
89
+ def ensure_connection
90
+ conn = nil
91
+
92
+ assert_nothing_raised do
93
+ conn = PGconn.new(:dbname => DATABASE, :host => HOST, :port => PORT, :user => USER, :password => PASSWORD)
94
+ end
95
+
96
+ conn
97
+ end
98
+
99
+ def create_test_table(conn)
100
+ conn.exec(<<-SQL)
101
+ CREATE TABLE #{TABLE} (
102
+ #{TAG_COL} Text
103
+ ,#{TIME_COL} Timestamptz
104
+ ,#{RECORD_COL} Json
105
+ );
106
+ SQL
45
107
  end
108
+
109
+ def drop_test_table(conn)
110
+ conn.exec("DROP TABLE #{TABLE}")
111
+ end
112
+
113
+ def with_connection(&block)
114
+ conn = ensure_connection
115
+ create_test_table(conn)
116
+ begin
117
+ block.call(conn)
118
+ ensure
119
+ drop_test_table(conn)
120
+ conn.close
121
+ end
122
+ end
123
+
124
+ def wait_for_data(conn)
125
+ 10.times do
126
+ res = conn.exec "select count(*) from #{TABLE}"
127
+ return if res.getvalue(0,0).to_i > 0
128
+ sleep 0.5
129
+ end
130
+ raise "COPY has not been finished correctly"
131
+ end
132
+
46
133
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-pgjson
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - OKUNO Akihiro
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-11-11 00:00:00.000000000 Z
11
+ date: 2014-12-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd