fluent-plugin-datahub 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +3 -0
- data/README.md +38 -0
- data/Rakefile +14 -0
- data/VERSION +1 -0
- data/build.sh +11 -0
- data/fluent-plugin-datahub.gemspec +22 -0
- data/lib/fluent/plugin/datahub/datahub-client.rb +27 -0
- data/lib/fluent/plugin/datahub/datahub-http-client-test.rb +343 -0
- data/lib/fluent/plugin/datahub/datahub-http-client.rb +229 -0
- data/lib/fluent/plugin/datahub/datahub-project.rb +59 -0
- data/lib/fluent/plugin/datahub/datahub-put-record-result.rb +23 -0
- data/lib/fluent/plugin/datahub/datahub-record-entity.rb +136 -0
- data/lib/fluent/plugin/datahub/datahub-record-schema.rb +73 -0
- data/lib/fluent/plugin/datahub/datahub-shard.rb +13 -0
- data/lib/fluent/plugin/datahub/datahub-topic.rb +73 -0
- data/lib/fluent/plugin/out_datahub.rb +402 -0
- data/sample/csv_sample.conf +22 -0
- data/sample/csv_sample.csv +14 -0
- data/sample/log_sample.conf +17 -0
- data/sample/log_sample.log +1 -0
- metadata +97 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
<source>
|
|
2
|
+
@type tail
|
|
3
|
+
path ${DIR}/csv_sample.data
|
|
4
|
+
tag test1
|
|
5
|
+
format csv
|
|
6
|
+
keys id,name,gender,salary,my_time
|
|
7
|
+
</source>
|
|
8
|
+
|
|
9
|
+
<match test1>
|
|
10
|
+
@type datahub
|
|
11
|
+
access_id yourAccessId
|
|
12
|
+
access_key yourAccessKey
|
|
13
|
+
endpoint yourEndpoint
|
|
14
|
+
project_name test_project
|
|
15
|
+
topic_name fluentd_out_7
|
|
16
|
+
# shard_id 6
|
|
17
|
+
column_names ["id", "name", "gender", "salary", "my_time"]
|
|
18
|
+
flush_interval 10s
|
|
19
|
+
dirty_data_continue true
|
|
20
|
+
dirty_data_file ${DIR}/dirty.data
|
|
21
|
+
retry_times 3
|
|
22
|
+
</match>
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
1000,hj,true,100.1,14321111111
|
|
2
|
+
2000,hj,true,100.1,14321111111
|
|
3
|
+
3000,333,true,100.1,14321111111
|
|
4
|
+
4000,hj,true,100.1,14321111111
|
|
5
|
+
5000,hj,true,100.1,14321111111
|
|
6
|
+
6000,hj,true,100.1,14321111111
|
|
7
|
+
7000,hj,true,100.1,14321111111
|
|
8
|
+
8000,hj,true,100.1,14321111111
|
|
9
|
+
9000,hj,true,100.1,14321111111
|
|
10
|
+
10001,hj,true,100.1,14321111111
|
|
11
|
+
10002,hj,true,100.1,14321111111
|
|
12
|
+
10003,hj,true,100.1,14321111111
|
|
13
|
+
10004,hj,true,100.1,14321111111
|
|
14
|
+
10005,hj,true,100.1,14321111111
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
source>
|
|
2
|
+
@type tail
|
|
3
|
+
path ${DIR}/log_sample.log
|
|
4
|
+
tag test
|
|
5
|
+
format /(?<request_time>\d\d:\d\d:\d\d.\d+)\s+\[(?<thread_id>[\w\-]+)\]\s+(?<log_level>\w+)\s+(?<class>\w+)\s+-\s+\[(?<request_id>\w+)\]\s+(?<detail>.+)/
|
|
6
|
+
</source>
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
<match test>
|
|
10
|
+
@type datahub
|
|
11
|
+
access_id yourAccessId
|
|
12
|
+
access_key yourAccessKey
|
|
13
|
+
endpoint yourEndpoint
|
|
14
|
+
project_name test_project
|
|
15
|
+
topic_name datahub_fluentd_out_1
|
|
16
|
+
column_names ["thread_id", "log_level", "class"]
|
|
17
|
+
</match>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
14:48:15.661 [qtp1075479366-20] INFO AuditInterceptor - [8jxn9mkknu02dokvifbuwzt9] begin /web/v1/project/8o5fzzsfxscrga52obirgyuq/dataSource?pageIndex=1&pageSize=12&type=6 GET, 068295, null
|
metadata
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: fluent-plugin-datahub
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.0.1
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Aliyun
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2016-09-19 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: fluentd
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: 0.12.0
|
|
20
|
+
- - "<"
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: '2'
|
|
23
|
+
type: :runtime
|
|
24
|
+
prerelease: false
|
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
26
|
+
requirements:
|
|
27
|
+
- - ">="
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: 0.12.0
|
|
30
|
+
- - "<"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '2'
|
|
33
|
+
- !ruby/object:Gem::Dependency
|
|
34
|
+
name: yajl-ruby
|
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - "~>"
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '1.0'
|
|
40
|
+
type: :runtime
|
|
41
|
+
prerelease: false
|
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - "~>"
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: '1.0'
|
|
47
|
+
description: Aliyun Datahub output plugin for Fluentd event collector
|
|
48
|
+
email: stream@service.aliyun.com
|
|
49
|
+
executables: []
|
|
50
|
+
extensions: []
|
|
51
|
+
extra_rdoc_files: []
|
|
52
|
+
files:
|
|
53
|
+
- Gemfile
|
|
54
|
+
- README.md
|
|
55
|
+
- Rakefile
|
|
56
|
+
- VERSION
|
|
57
|
+
- build.sh
|
|
58
|
+
- fluent-plugin-datahub.gemspec
|
|
59
|
+
- lib/fluent/plugin/datahub/datahub-client.rb
|
|
60
|
+
- lib/fluent/plugin/datahub/datahub-http-client-test.rb
|
|
61
|
+
- lib/fluent/plugin/datahub/datahub-http-client.rb
|
|
62
|
+
- lib/fluent/plugin/datahub/datahub-project.rb
|
|
63
|
+
- lib/fluent/plugin/datahub/datahub-put-record-result.rb
|
|
64
|
+
- lib/fluent/plugin/datahub/datahub-record-entity.rb
|
|
65
|
+
- lib/fluent/plugin/datahub/datahub-record-schema.rb
|
|
66
|
+
- lib/fluent/plugin/datahub/datahub-shard.rb
|
|
67
|
+
- lib/fluent/plugin/datahub/datahub-topic.rb
|
|
68
|
+
- lib/fluent/plugin/out_datahub.rb
|
|
69
|
+
- sample/csv_sample.conf
|
|
70
|
+
- sample/csv_sample.csv
|
|
71
|
+
- sample/log_sample.conf
|
|
72
|
+
- sample/log_sample.log
|
|
73
|
+
homepage: https://datahub.console.aliyun.com/datahub
|
|
74
|
+
licenses:
|
|
75
|
+
- Apache-2.0
|
|
76
|
+
metadata: {}
|
|
77
|
+
post_install_message:
|
|
78
|
+
rdoc_options: []
|
|
79
|
+
require_paths:
|
|
80
|
+
- lib
|
|
81
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
82
|
+
requirements:
|
|
83
|
+
- - ">="
|
|
84
|
+
- !ruby/object:Gem::Version
|
|
85
|
+
version: '0'
|
|
86
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
87
|
+
requirements:
|
|
88
|
+
- - ">="
|
|
89
|
+
- !ruby/object:Gem::Version
|
|
90
|
+
version: '0'
|
|
91
|
+
requirements: []
|
|
92
|
+
rubyforge_project:
|
|
93
|
+
rubygems_version: 2.4.5.1
|
|
94
|
+
signing_key:
|
|
95
|
+
specification_version: 4
|
|
96
|
+
summary: Aliyun Datahub output plugin for Fluentd event collector
|
|
97
|
+
test_files: []
|