fluent-plugin-datahub 0.0.1

data/sample/csv_sample.conf

@@ -0,0 +1,22 @@
+<source>
+  @type tail
+  path ${DIR}/csv_sample.data
+  tag test1
+  format csv
+  keys id,name,gender,salary,my_time
+</source>
+
+<match test1>
+  @type datahub
+  access_id yourAccessId
+  access_key yourAccessKey
+  endpoint yourEndpoint
+  project_name test_project
+  topic_name fluentd_out_7
+  # shard_id 6
+  column_names ["id", "name", "gender", "salary", "my_time"]
+  flush_interval 10s
+  dirty_data_continue true
+  dirty_data_file ${DIR}/dirty.data
+  retry_times 3
+</match>

data/sample/csv_sample.csv

@@ -0,0 +1,14 @@
+1000,hj,true,100.1,14321111111
+2000,hj,true,100.1,14321111111
+3000,333,true,100.1,14321111111
+4000,hj,true,100.1,14321111111
+5000,hj,true,100.1,14321111111
+6000,hj,true,100.1,14321111111
+7000,hj,true,100.1,14321111111
+8000,hj,true,100.1,14321111111
+9000,hj,true,100.1,14321111111
+10001,hj,true,100.1,14321111111
+10002,hj,true,100.1,14321111111
+10003,hj,true,100.1,14321111111
+10004,hj,true,100.1,14321111111
+10005,hj,true,100.1,14321111111

data/sample/log_sample.conf

@@ -0,0 +1,17 @@
+source>
+  @type tail
+  path ${DIR}/log_sample.log
+  tag test
+  format /(?<request_time>\d\d:\d\d:\d\d.\d+)\s+\[(?<thread_id>[\w\-]+)\]\s+(?<log_level>\w+)\s+(?<class>\w+)\s+-\s+\[(?<request_id>\w+)\]\s+(?<detail>.+)/
+</source>
+
+
+<match test>
+  @type datahub
+  access_id yourAccessId
+  access_key yourAccessKey
+  endpoint yourEndpoint
+  project_name test_project
+  topic_name datahub_fluentd_out_1
+  column_names ["thread_id", "log_level", "class"]
+</match>

data/sample/log_sample.log

@@ -0,0 +1 @@
+14:48:15.661 [qtp1075479366-20] INFO  AuditInterceptor - [8jxn9mkknu02dokvifbuwzt9] begin /web/v1/project/8o5fzzsfxscrga52obirgyuq/dataSource?pageIndex=1&pageSize=12&type=6 GET, 068295, null

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-datahub
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Aliyun
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Aliyun Datahub output plugin for Fluentd event collector
+email: stream@service.aliyun.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- build.sh
+- fluent-plugin-datahub.gemspec
+- lib/fluent/plugin/datahub/datahub-client.rb
+- lib/fluent/plugin/datahub/datahub-http-client-test.rb
+- lib/fluent/plugin/datahub/datahub-http-client.rb
+- lib/fluent/plugin/datahub/datahub-project.rb
+- lib/fluent/plugin/datahub/datahub-put-record-result.rb
+- lib/fluent/plugin/datahub/datahub-record-entity.rb
+- lib/fluent/plugin/datahub/datahub-record-schema.rb
+- lib/fluent/plugin/datahub/datahub-shard.rb
+- lib/fluent/plugin/datahub/datahub-topic.rb
+- lib/fluent/plugin/out_datahub.rb
+- sample/csv_sample.conf
+- sample/csv_sample.csv
+- sample/log_sample.conf
+- sample/log_sample.log
+homepage: https://datahub.console.aliyun.com/datahub
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Aliyun Datahub output plugin for Fluentd event collector
+test_files: []