fluent-plugin-pgjson 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/fluent-plugin-pgjson.gemspec +1 -1
- data/lib/fluent/plugin/out_pgjson.rb +9 -5
- data/test/plugin/test_out.rb +114 -27
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4dfa8ba1bbd21e1ffa84476064f2cb0eced7e5c8
|
4
|
+
data.tar.gz: 7522f3e3b4639c1022f58f41ee9b568fea58827a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d3ff0bb506bac3398d9f02e5c3023babc3a7534261de3e05cb0c06738c6cc586f1f94578c3fa9e4fba91468682d1cb7aca129fefff2fed23dea84543003862ab
|
7
|
+
data.tar.gz: d9acbeca7d5d942ec7f1454716e3b69750b25bdc83416253f86ca1c04e921774b58f968c0cedadee06ad82a2be11cb7951551d95523dd27f830e373f21b73490
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-pgjson"
|
6
|
-
s.version = "0.0.
|
6
|
+
s.version = "0.0.6"
|
7
7
|
s.authors = ["OKUNO Akihiro"]
|
8
8
|
s.email = ["choplin.choplin@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/choplin/fluent-plugin-pgjson"
|
@@ -13,7 +13,7 @@ class PgJsonOutput < Fluent::BufferedOutput
|
|
13
13
|
config_param :time_col , :string , :default => 'time'
|
14
14
|
config_param :tag_col , :string , :default => 'tag'
|
15
15
|
config_param :record_col , :string , :default => 'record'
|
16
|
-
config_param :
|
16
|
+
config_param :msgpack , :bool , :default => false
|
17
17
|
|
18
18
|
def initialize
|
19
19
|
super
|
@@ -38,16 +38,18 @@ class PgJsonOutput < Fluent::BufferedOutput
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def write(chunk)
|
41
|
+
init_connection
|
41
42
|
begin
|
42
|
-
init_connection
|
43
43
|
@conn.exec("COPY #{@table} (#{@tag_col}, #{@time_col}, #{@record_col}) FROM STDIN WITH DELIMITER E'\\x01'")
|
44
44
|
chunk.msgpack_each do |tag, time, record|
|
45
45
|
@conn.put_copy_data "#{tag}\x01#{Time.at(time).to_s}\x01#{record_value(record)}\n"
|
46
46
|
end
|
47
47
|
@conn.put_copy_end
|
48
48
|
rescue
|
49
|
-
@conn.
|
50
|
-
|
49
|
+
if ! @conn.nil?
|
50
|
+
@conn.close()
|
51
|
+
@conn = nil
|
52
|
+
end
|
51
53
|
raise "failed to send data to postgres: #$!"
|
52
54
|
end
|
53
55
|
end
|
@@ -74,7 +76,9 @@ class PgJsonOutput < Fluent::BufferedOutput
|
|
74
76
|
if @msgpack
|
75
77
|
"\\#{@conn.escape_bytea(record.to_msgpack)}"
|
76
78
|
else
|
77
|
-
record.to_json
|
79
|
+
json = record.to_json
|
80
|
+
json.gsub!(/\\/){ '\\\\' }
|
81
|
+
json
|
78
82
|
end
|
79
83
|
end
|
80
84
|
end
|
data/test/plugin/test_out.rb
CHANGED
@@ -1,46 +1,133 @@
|
|
1
|
+
require 'pg'
|
2
|
+
require 'securerandom'
|
1
3
|
require 'helper'
|
2
4
|
|
3
5
|
class PgJsonOutputTest < Test::Unit::TestCase
|
4
|
-
|
5
|
-
|
6
|
-
|
6
|
+
HOST = "localhost"
|
7
|
+
PORT = 5432
|
8
|
+
DATABASE = "postgres"
|
9
|
+
TABLE = "test_fluentd_#{SecureRandom.hex}"
|
10
|
+
USER = "postgres"
|
11
|
+
PASSWORD = "postgres"
|
12
|
+
|
13
|
+
TIME_COL = "time"
|
14
|
+
TAG_COL = "tag"
|
15
|
+
RECORD_COL = "record"
|
7
16
|
|
8
17
|
CONFIG = %[
|
9
18
|
type pgjson
|
10
|
-
host
|
11
|
-
port
|
12
|
-
database
|
13
|
-
table
|
14
|
-
user
|
15
|
-
password
|
16
|
-
time_col
|
17
|
-
tag_col
|
18
|
-
record_col
|
19
|
+
host #{HOST}
|
20
|
+
port #{PORT}
|
21
|
+
database #{DATABASE}
|
22
|
+
table #{TABLE}
|
23
|
+
user #{USER}
|
24
|
+
password #{PASSWORD}
|
25
|
+
time_col #{TIME_COL}
|
26
|
+
tag_col #{TAG_COL}
|
27
|
+
record_col #{RECORD_COL}
|
19
28
|
]
|
20
29
|
|
21
|
-
def
|
22
|
-
Fluent::Test
|
30
|
+
def setup
|
31
|
+
Fluent::Test.setup
|
32
|
+
end
|
33
|
+
|
34
|
+
def create_driver(conf = CONFIG, tag = 'test')
|
35
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::PgJsonOutput).configure(conf)
|
23
36
|
end
|
24
37
|
|
25
38
|
def test_configure
|
26
39
|
d = create_driver
|
27
40
|
|
28
|
-
assert_equal
|
29
|
-
assert_equal
|
30
|
-
assert_equal
|
31
|
-
assert_equal
|
32
|
-
assert_equal
|
33
|
-
assert_equal
|
34
|
-
assert_equal
|
35
|
-
assert_equal
|
36
|
-
assert_equal
|
41
|
+
assert_equal HOST, d.instance.host
|
42
|
+
assert_equal PORT, d.instance.port
|
43
|
+
assert_equal DATABASE, d.instance.database
|
44
|
+
assert_equal TABLE, d.instance.table
|
45
|
+
assert_equal USER, d.instance.user
|
46
|
+
assert_equal PASSWORD, d.instance.password
|
47
|
+
assert_equal TIME_COL, d.instance.time_col
|
48
|
+
assert_equal TAG_COL, d.instance.tag_col
|
49
|
+
assert_equal RECORD_COL, d.instance.record_col
|
37
50
|
end
|
38
51
|
|
39
|
-
def
|
40
|
-
|
52
|
+
def test_write
|
53
|
+
with_connection do |conn|
|
54
|
+
tag = 'test'
|
55
|
+
time = Time.parse("2014-12-26 07:58:37 UTC")
|
56
|
+
record = {"a"=>1}
|
57
|
+
|
58
|
+
d = create_driver(CONFIG, tag)
|
59
|
+
d.emit(record, time.to_i)
|
60
|
+
d.run
|
61
|
+
wait_for_data(conn)
|
62
|
+
|
63
|
+
res = conn.exec("select * from #{TABLE}")[0]
|
64
|
+
assert_equal res[TAG_COL], tag
|
65
|
+
assert_equal Time.parse(res[TIME_COL]), time
|
66
|
+
assert_equal res[RECORD_COL], record.to_json
|
67
|
+
end
|
41
68
|
end
|
42
69
|
|
43
|
-
def
|
44
|
-
|
70
|
+
def test_escape_of_backslash
|
71
|
+
with_connection do |conn|
|
72
|
+
tag = 'test'
|
73
|
+
time = Time.parse("2014-12-26 07:58:37 UTC")
|
74
|
+
record = {"a"=>"\"foo\""}
|
75
|
+
|
76
|
+
d = create_driver(CONFIG, tag)
|
77
|
+
d.emit(record, time.to_i)
|
78
|
+
d.run
|
79
|
+
wait_for_data(conn)
|
80
|
+
|
81
|
+
res = conn.exec("select * from #{TABLE}")[0]
|
82
|
+
assert_equal res[TAG_COL], tag
|
83
|
+
assert_equal Time.parse(res[TIME_COL]), time
|
84
|
+
assert_equal res[RECORD_COL], record.to_json
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
|
89
|
+
def ensure_connection
|
90
|
+
conn = nil
|
91
|
+
|
92
|
+
assert_nothing_raised do
|
93
|
+
conn = PGconn.new(:dbname => DATABASE, :host => HOST, :port => PORT, :user => USER, :password => PASSWORD)
|
94
|
+
end
|
95
|
+
|
96
|
+
conn
|
97
|
+
end
|
98
|
+
|
99
|
+
def create_test_table(conn)
|
100
|
+
conn.exec(<<-SQL)
|
101
|
+
CREATE TABLE #{TABLE} (
|
102
|
+
#{TAG_COL} Text
|
103
|
+
,#{TIME_COL} Timestamptz
|
104
|
+
,#{RECORD_COL} Json
|
105
|
+
);
|
106
|
+
SQL
|
45
107
|
end
|
108
|
+
|
109
|
+
def drop_test_table(conn)
|
110
|
+
conn.exec("DROP TABLE #{TABLE}")
|
111
|
+
end
|
112
|
+
|
113
|
+
def with_connection(&block)
|
114
|
+
conn = ensure_connection
|
115
|
+
create_test_table(conn)
|
116
|
+
begin
|
117
|
+
block.call(conn)
|
118
|
+
ensure
|
119
|
+
drop_test_table(conn)
|
120
|
+
conn.close
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def wait_for_data(conn)
|
125
|
+
10.times do
|
126
|
+
res = conn.exec "select count(*) from #{TABLE}"
|
127
|
+
return if res.getvalue(0,0).to_i > 0
|
128
|
+
sleep 0.5
|
129
|
+
end
|
130
|
+
raise "COPY has not been finished correctly"
|
131
|
+
end
|
132
|
+
|
46
133
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-pgjson
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- OKUNO Akihiro
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-12-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|