fluent-plugin-twitter-enchanced 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.travis.yml +15 -0
- data/Appraisals +3 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +14 -0
- data/README.md +154 -0
- data/Rakefile +10 -0
- data/fluent-plugin-twitter.gemspec +25 -0
- data/gemfiles/fluentd_v0.14.gemfile +7 -0
- data/lib/fluent/plugin/in_twitter.rb +168 -0
- data/lib/fluent/plugin/out_twitter.rb +48 -0
- data/test/helper.rb +6 -0
- data/test/plugin/test_in_twitter.rb +127 -0
- data/test/plugin/test_out_twitter.rb +88 -0
- metadata +135 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 12ae0331f9ea7e2a19a23bdbbb7118a42d6dc202
|
4
|
+
data.tar.gz: fae48822ad3705aaa6f7e1a756d2758899b9920b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 59e503ed5d3f130c97952b91f5bcabc805d5f5b40ffa362d31027649b3021736294612b3435793dcd856ebe095af35959c6fe385c87e37e0d9bd4c16bb7ccf9f
|
7
|
+
data.tar.gz: 515628a4b9c8b36c12bb64ef826a5412cb6098ec7da13e90c011164a877fef1a9aa9f3e28c9d3ec0970bc69a416305f518e2629d244655f076c8ff20d6242211
|
data/.travis.yml
ADDED
data/Appraisals
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Copyright (c) 2012- Kentaro Yoshida
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
14
|
+
|
data/README.md
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
fluent-plugin-twitter [![Build Status](https://travis-ci.org/y-ken/fluent-plugin-twitter.png?branch=master)](https://travis-ci.org/y-ken/fluent-plugin-twitter)
|
2
|
+
=====================
|
3
|
+
|
4
|
+
## Component
|
5
|
+
Fluentd Input/Output plugin to process tweets with Twitter Streaming API.
|
6
|
+
|
7
|
+
## Dependency
|
8
|
+
|
9
|
+
before use, install dependent library as:
|
10
|
+
|
11
|
+
```
|
12
|
+
# for RHEL/CentOS (eventmachine requires build dependency)
|
13
|
+
$ sudo yum -y install gcc gcc-c++ openssl-devel libcurl libcurl-devel
|
14
|
+
|
15
|
+
# for Ubuntu/Debian (eventmachine requires build dependency)
|
16
|
+
$ sudo apt-get install build-essential libssl-dev
|
17
|
+
```
|
18
|
+
|
19
|
+
## Requirements
|
20
|
+
|
21
|
+
| fluent-plugin-twitter | fluentd | ruby |
|
22
|
+
|--------------------|------------|--------|
|
23
|
+
| 0.6.1 | v0.14.x | >= 2.1 |
|
24
|
+
| 0.5.4 | v0.12.x | >= 1.9 |
|
25
|
+
|
26
|
+
## Installation
|
27
|
+
|
28
|
+
install with `gem` or `td-agent-gem` command as:
|
29
|
+
|
30
|
+
```
|
31
|
+
# for fluentd
|
32
|
+
$ gem install eventmachine
|
33
|
+
$ gem install fluent-plugin-twitter
|
34
|
+
|
35
|
+
# for td-agent2
|
36
|
+
$ sudo td-agent-gem install eventmachine
|
37
|
+
$ sudo td-agent-gem install fluent-plugin-twitter -v 0.5.4
|
38
|
+
```
|
39
|
+
|
40
|
+
## Input Configuration
|
41
|
+
|
42
|
+
### Input Sample
|
43
|
+
|
44
|
+
It require td-agent2 (fluentd v0.12) to use keyword with hashtag.
|
45
|
+
|
46
|
+
`````
|
47
|
+
<source>
|
48
|
+
@type twitter
|
49
|
+
consumer_key YOUR_CONSUMER_KEY # Required
|
50
|
+
consumer_secret YOUR_CONSUMER_SECRET # Required
|
51
|
+
access_token YOUR_ACCESS_TOKEN # Required
|
52
|
+
access_token_secret YOUR_ACCESS_TOKEN_SECRET # Required
|
53
|
+
tag input.twitter.sampling # Required
|
54
|
+
timeline tracking # Required (tracking or sampling or location or userstream)
|
55
|
+
keyword 'Ruby,Python,#fleuntd' # Optional (keyword has priority than follow_ids)
|
56
|
+
follow_ids 14252,53235 # Optional (integers, not screen names)
|
57
|
+
locations 31.110283, 129.431631, 45.619283, 145.510175 # Optional (bounding boxes; first pair specifies longitude/latitude of southwest corner)
|
58
|
+
lang ja,en # Optional
|
59
|
+
output_format nest # Optional (nest or flat or simple[default])
|
60
|
+
flatten_separator _ # Optional
|
61
|
+
</source>
|
62
|
+
|
63
|
+
<match input.twitter.sampling>
|
64
|
+
@type stdout
|
65
|
+
</match>
|
66
|
+
`````
|
67
|
+
|
68
|
+
### Proxy support
|
69
|
+
|
70
|
+
```
|
71
|
+
<source>
|
72
|
+
@type twitter
|
73
|
+
consumer_key YOUR_CONSUMER_KEY # Required
|
74
|
+
consumer_secret YOUR_CONSUMER_SECRET # Required
|
75
|
+
access_token YOUR_ACCESS_TOKEN # Required
|
76
|
+
access_token_secret YOUR_ACCESS_TOKEN_SECRET # Required
|
77
|
+
tag input.twitter.sampling # Required
|
78
|
+
timeline tracking # Required (tracking or sampling or location or userstream)
|
79
|
+
<proxy>
|
80
|
+
host proxy.example.com # Required
|
81
|
+
port 8080 # Required
|
82
|
+
username proxyuser # Optional
|
83
|
+
password proxypass # Optional
|
84
|
+
</proxy>
|
85
|
+
</source>
|
86
|
+
```
|
87
|
+
|
88
|
+
### Testing
|
89
|
+
|
90
|
+
`````
|
91
|
+
$ tail -f /var/log/td-agent/td-agent.log
|
92
|
+
`````
|
93
|
+
|
94
|
+
## Output Configuration
|
95
|
+
|
96
|
+
### Output Sample
|
97
|
+
`````
|
98
|
+
<source>
|
99
|
+
@type http
|
100
|
+
port 8888
|
101
|
+
</source>
|
102
|
+
|
103
|
+
<match notify.twitter>
|
104
|
+
@type twitter
|
105
|
+
consumer_key YOUR_CONSUMER_KEY
|
106
|
+
consumer_secret YOUR_CONSUMER_SECRET
|
107
|
+
access_token YOUR_ACCESS_TOKEN
|
108
|
+
access_token_secret YOUR_ACCESS_TOKEN_SECRET
|
109
|
+
</match>
|
110
|
+
`````
|
111
|
+
|
112
|
+
### Proxy support
|
113
|
+
|
114
|
+
```
|
115
|
+
<match notify.twitter>
|
116
|
+
@type twitter
|
117
|
+
consumer_key YOUR_CONSUMER_KEY
|
118
|
+
consumer_secret YOUR_CONSUMER_SECRET
|
119
|
+
access_token YOUR_ACCESS_TOKEN
|
120
|
+
access_token_secret YOUR_ACCESS_TOKEN_SECRET
|
121
|
+
<proxy>
|
122
|
+
host proxy.example.com
|
123
|
+
port 8080
|
124
|
+
username proxyuser
|
125
|
+
password proxypass
|
126
|
+
</proxy>
|
127
|
+
</match>
|
128
|
+
```
|
129
|
+
|
130
|
+
### Testing
|
131
|
+
|
132
|
+
`````
|
133
|
+
$ curl http://localhost:8888/notify.twitter -F 'json={"message":"foo"}'
|
134
|
+
`````
|
135
|
+
|
136
|
+
## Reference
|
137
|
+
|
138
|
+
### Twitter OAuth Guide
|
139
|
+
http://pocketstudio.jp/log3/2012/02/12/how_to_get_twitter_apikey_and_token/
|
140
|
+
|
141
|
+
### Quick Tour to count a tweet with fluent-plugin-twitter and fluent-plugin-datacounter
|
142
|
+
http://qiita.com/items/fe4258b394190f23fece
|
143
|
+
|
144
|
+
## TODO
|
145
|
+
|
146
|
+
patches welcome!
|
147
|
+
|
148
|
+
## Copyright
|
149
|
+
|
150
|
+
Copyright © 2012- Kentaro Yoshida (@yoshi_ken)
|
151
|
+
|
152
|
+
## License
|
153
|
+
|
154
|
+
Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
$:.push File.expand_path("../lib", __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = "fluent-plugin-twitter-enchanced"
|
5
|
+
s.version = "0.6.1"
|
6
|
+
s.authors = ["Kentaro Yoshida"]
|
7
|
+
s.email = ["y.ken.studio@gmail.com"]
|
8
|
+
s.homepage = "https://github.com/y-ken/fluent-plugin-twitter"
|
9
|
+
s.summary = %q{Fluentd Input/Output plugin to collect/process tweets with Twitter Streaming API.}
|
10
|
+
s.license = "Apache-2.0"
|
11
|
+
|
12
|
+
s.files = `git ls-files`.split("\n")
|
13
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
14
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
15
|
+
s.require_paths = ["lib"]
|
16
|
+
|
17
|
+
s.required_ruby_version = "> 2.1"
|
18
|
+
|
19
|
+
s.add_development_dependency "rake"
|
20
|
+
s.add_development_dependency "test-unit", ">= 3.1.0"
|
21
|
+
s.add_development_dependency "appraisal"
|
22
|
+
|
23
|
+
s.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
|
24
|
+
s.add_runtime_dependency "twitter", "~> 6.0"
|
25
|
+
end
|
@@ -0,0 +1,168 @@
|
|
1
|
+
require "fluent/plugin/input"
|
2
|
+
|
3
|
+
require 'twitter'
|
4
|
+
require 'nkf'
|
5
|
+
|
6
|
+
module Fluent::Plugin
|
7
|
+
class TwitterInput < Fluent::Plugin::Input
|
8
|
+
Fluent::Plugin.register_input('twitter', self)
|
9
|
+
|
10
|
+
helpers :thread
|
11
|
+
|
12
|
+
TIMELINE_TYPE = %i(userstream sampling location tracking)
|
13
|
+
OUTPUT_FORMAT_TYPE = %i(nest flat simple)
|
14
|
+
|
15
|
+
config_param :consumer_key, :string, secret: true
|
16
|
+
config_param :consumer_secret, :string, secret: true
|
17
|
+
config_param :access_token, :string, secret: true
|
18
|
+
config_param :access_token_secret, :string, secret: true
|
19
|
+
config_param :tag, :string
|
20
|
+
config_param :timeline, :enum, list: TIMELINE_TYPE
|
21
|
+
config_param :keyword, :string, default: nil
|
22
|
+
config_param :follow_ids, :string, default: nil
|
23
|
+
config_param :locations, :string, default: nil
|
24
|
+
config_param :lang, :string, default: nil
|
25
|
+
config_param :output_format, :enum, list: OUTPUT_FORMAT_TYPE, default: :simple
|
26
|
+
config_param :flatten_separator, :string, default: '_'
|
27
|
+
|
28
|
+
config_section :proxy, multi: false do
|
29
|
+
config_param :host, :string
|
30
|
+
config_param :port, :string
|
31
|
+
config_param :username, :string, default: nil
|
32
|
+
config_param :password, :string, default: nil, secret: true
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize
|
36
|
+
super
|
37
|
+
@running = false
|
38
|
+
end
|
39
|
+
|
40
|
+
def configure(conf)
|
41
|
+
super
|
42
|
+
|
43
|
+
@keyword = @keyword.gsub('${hashtag}', '#') unless @keyword.nil?
|
44
|
+
|
45
|
+
@client = Twitter::Streaming::Client.new do |config|
|
46
|
+
config.consumer_key = @consumer_key
|
47
|
+
config.consumer_secret = @consumer_secret
|
48
|
+
config.access_token = @access_token
|
49
|
+
config.access_token_secret = @access_token_secret
|
50
|
+
config.proxy = @proxy.to_h if @proxy
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def start
|
55
|
+
@running = true
|
56
|
+
thread_create(:in_twitter) do
|
57
|
+
run
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def shutdown
|
62
|
+
@running = false
|
63
|
+
@client.close if @client.respond_to?(:close)
|
64
|
+
super
|
65
|
+
end
|
66
|
+
|
67
|
+
def run
|
68
|
+
notice = "twitter: starting Twitter Streaming API for #{@timeline}."
|
69
|
+
notice << " tag:#{@tag}"
|
70
|
+
notice << " lang:#{@lang}" unless @lang.nil?
|
71
|
+
notice << " keyword:#{@keyword}" unless @keyword.nil?
|
72
|
+
notice << " follow:#{@follow_ids}" unless @follow_ids.nil? && !@keyword.nil?
|
73
|
+
log.info notice
|
74
|
+
|
75
|
+
if [:sampling, :tracking].include?(@timeline) && @keyword
|
76
|
+
@client.filter(track: @keyword, &method(:handle_object))
|
77
|
+
elsif @timeline == :tracking && @follow_ids
|
78
|
+
@client.filter(follow: @follow_ids, &method(:handle_object))
|
79
|
+
elsif @timeline == :sampling && @keyword.nil? && @follow_ids.nil?
|
80
|
+
@client.sample(&method(:handle_object))
|
81
|
+
elsif @timeline == :userstream
|
82
|
+
@client.user(&method(:handle_object))
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def handle_object(object)
|
87
|
+
return unless @running
|
88
|
+
if is_message?(object)
|
89
|
+
get_message(object)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def is_message?(tweet)
|
94
|
+
return false if !tweet.is_a?(Twitter::Tweet)
|
95
|
+
return false if (!@lang.nil? && @lang != '') && !@lang.include?(tweet.user.lang)
|
96
|
+
if @timeline == :userstream && (!@keyword.nil? && @keyword != '')
|
97
|
+
pattern = NKF::nkf('-WwZ1', @keyword).gsub(/,\s?/, '|')
|
98
|
+
tweet = NKF::nkf('-WwZ1', tweet.text)
|
99
|
+
return false if !Regexp.new(pattern, Regexp::IGNORECASE).match(tweet)
|
100
|
+
end
|
101
|
+
return true
|
102
|
+
end
|
103
|
+
|
104
|
+
def get_message(tweet)
|
105
|
+
case @output_format
|
106
|
+
when :nest
|
107
|
+
record = hash_key_to_s(tweet.to_h)
|
108
|
+
when :flat
|
109
|
+
record = hash_flatten(tweet.to_h)
|
110
|
+
when :simple
|
111
|
+
record = {}
|
112
|
+
record['message'] = tweet.text.scrub('')
|
113
|
+
record['geo'] = tweet.geo
|
114
|
+
record['place'] = tweet.place
|
115
|
+
record['created_at'] = tweet.created_at.to_s
|
116
|
+
record['user_name'] = tweet.user.name
|
117
|
+
record['user_screen_name'] = tweet.user.screen_name
|
118
|
+
record['user_profile_image_url'] = tweet.user.profile_image_url.to_s
|
119
|
+
record['user_time_zone'] = tweet.user.time_zone
|
120
|
+
record['user_lang'] = tweet.user.lang
|
121
|
+
end
|
122
|
+
router.emit(@tag, Fluent::Engine.now, record)
|
123
|
+
end
|
124
|
+
|
125
|
+
def hash_flatten(record, prefix = nil)
|
126
|
+
record.inject({}) do |d, (k, v)|
|
127
|
+
k = prefix.to_s + k.to_s
|
128
|
+
if v.instance_of?(Hash)
|
129
|
+
d.merge(hash_flatten(v, k + @flatten_separator))
|
130
|
+
elsif v.instance_of?(String)
|
131
|
+
d.merge(k => v.scrub(""))
|
132
|
+
else
|
133
|
+
d.merge(k => v)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def hash_key_to_s(hash)
|
139
|
+
newhash = {}
|
140
|
+
hash.each do |k, v|
|
141
|
+
if v.instance_of?(Hash)
|
142
|
+
newhash[k.to_s] = hash_key_to_s(v)
|
143
|
+
elsif v.instance_of?(Array)
|
144
|
+
newhash[k.to_s] = array_key_to_s(v)
|
145
|
+
elsif v.instance_of?(String)
|
146
|
+
newhash[k.to_s] = v.scrub('')
|
147
|
+
else
|
148
|
+
newhash[k.to_s] = v
|
149
|
+
end
|
150
|
+
end
|
151
|
+
newhash
|
152
|
+
end
|
153
|
+
|
154
|
+
def array_key_to_s(array)
|
155
|
+
array.map do |v|
|
156
|
+
if v.instance_of?(Hash)
|
157
|
+
hash_key_to_s(v)
|
158
|
+
elsif v.instance_of?(Array)
|
159
|
+
array_key_to_s(v)
|
160
|
+
elsif v.instance_of?(String)
|
161
|
+
v.scrub('')
|
162
|
+
else
|
163
|
+
v
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require "twitter"
|
2
|
+
require "fluent/plugin/output"
|
3
|
+
|
4
|
+
class Fluent::Plugin::TwitterOutput < Fluent::Plugin::Output
|
5
|
+
Fluent::Plugin.register_output('twitter', self)
|
6
|
+
|
7
|
+
config_param :consumer_key, :string, secret: true
|
8
|
+
config_param :consumer_secret, :string, secret: true
|
9
|
+
config_param :access_token, :string, secret: true
|
10
|
+
config_param :access_token_secret, :string, secret: true
|
11
|
+
|
12
|
+
config_section :proxy, multi: false do
|
13
|
+
config_param :host, :string
|
14
|
+
config_param :port, :string
|
15
|
+
config_param :username, :string, default: nil
|
16
|
+
config_param :password, :string, default: nil, secret: true
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize
|
20
|
+
super
|
21
|
+
end
|
22
|
+
|
23
|
+
def configure(conf)
|
24
|
+
super
|
25
|
+
|
26
|
+
@twitter = Twitter::REST::Client.new do |config|
|
27
|
+
config.consumer_key = @consumer_key
|
28
|
+
config.consumer_secret = @consumer_secret
|
29
|
+
config.access_token = @access_token
|
30
|
+
config.access_token_secret = @access_token_secret
|
31
|
+
config.proxy = @proxy.to_h if @proxy
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def process(tag, es)
|
36
|
+
es.each do |_time, record|
|
37
|
+
tweet(record['message'])
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def tweet(message)
|
42
|
+
begin
|
43
|
+
@twitter.update(message)
|
44
|
+
rescue Twitter::Error => e
|
45
|
+
log.error("Twitter Error: #{e.message}")
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/plugin/in_twitter'
|
3
|
+
require 'fluent/test/driver/input'
|
4
|
+
|
5
|
+
class TwitterInputTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
CONFIG = %[
|
11
|
+
consumer_key CONSUMER_KEY
|
12
|
+
consumer_secret CONSUMER_SECRET
|
13
|
+
access_token ACCESS_TOKEN
|
14
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
15
|
+
tag input.twitter
|
16
|
+
timeline sampling
|
17
|
+
]
|
18
|
+
|
19
|
+
def create_driver(conf = CONFIG, syntax: :v1)
|
20
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::TwitterInput).configure(conf, syntax: syntax)
|
21
|
+
end
|
22
|
+
|
23
|
+
sub_test_case "v0 syntax" do
|
24
|
+
def test_empty
|
25
|
+
assert_raise(Fluent::ConfigError) {
|
26
|
+
create_driver('', syntax: :v0)
|
27
|
+
}
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_configure
|
31
|
+
d = create_driver %[
|
32
|
+
consumer_key CONSUMER_KEY
|
33
|
+
consumer_secret CONSUMER_SECRET
|
34
|
+
access_token ACCESS_TOKEN
|
35
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
36
|
+
tag input.twitter
|
37
|
+
timeline tracking
|
38
|
+
keyword ${hashtag}fluentd,fluentd lang:ja
|
39
|
+
]
|
40
|
+
assert_equal 'CONSUMER_KEY', d.instance.consumer_key
|
41
|
+
assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
|
42
|
+
assert_equal 'ACCESS_TOKEN', d.instance.access_token
|
43
|
+
assert_equal 'ACCESS_TOKEN_SECRET', d.instance.access_token_secret
|
44
|
+
assert_equal '#fluentd,fluentd lang:ja', d.instance.keyword
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
sub_test_case "v1 syntax" do
|
49
|
+
def test_empty
|
50
|
+
assert_raise(Fluent::ConfigError) {
|
51
|
+
create_driver('')
|
52
|
+
}
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_multi_keyword
|
56
|
+
d = create_driver(%[
|
57
|
+
consumer_key CONSUMER_KEY
|
58
|
+
consumer_secret CONSUMER_SECRET
|
59
|
+
access_token ACCESS_TOKEN
|
60
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
61
|
+
tag input.twitter
|
62
|
+
timeline tracking
|
63
|
+
keyword 'treasuredata,treasure data,#treasuredata,fluentd,#fluentd'
|
64
|
+
])
|
65
|
+
assert_equal 'CONSUMER_KEY', d.instance.consumer_key
|
66
|
+
assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
|
67
|
+
assert_equal 'ACCESS_TOKEN', d.instance.access_token
|
68
|
+
assert_equal 'ACCESS_TOKEN_SECRET', d.instance.access_token_secret
|
69
|
+
assert_equal 'treasuredata,treasure data,#treasuredata,fluentd,#fluentd', d.instance.keyword
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
sub_test_case "proxy" do
|
74
|
+
test "simple" do
|
75
|
+
conf = %[
|
76
|
+
consumer_key CONSUMER_KEY
|
77
|
+
consumer_secret CONSUMER_SECRET
|
78
|
+
access_token ACCESS_TOKEN
|
79
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
80
|
+
tag input.twitter
|
81
|
+
timeline tracking
|
82
|
+
keyword 'treasuredata,treasure data,#treasuredata,fluentd,#fluentd'
|
83
|
+
<proxy>
|
84
|
+
host proxy.example.com
|
85
|
+
port 8080
|
86
|
+
username proxyuser
|
87
|
+
password proxypass
|
88
|
+
</proxy>
|
89
|
+
]
|
90
|
+
d = create_driver(conf)
|
91
|
+
expected = {
|
92
|
+
host: "proxy.example.com",
|
93
|
+
port: "8080",
|
94
|
+
username: "proxyuser",
|
95
|
+
password: "proxypass"
|
96
|
+
}
|
97
|
+
assert_equal(expected, d.instance.proxy.to_h)
|
98
|
+
end
|
99
|
+
|
100
|
+
test "multi proxy is not supported" do
|
101
|
+
conf = %[
|
102
|
+
consumer_key CONSUMER_KEY
|
103
|
+
consumer_secret CONSUMER_SECRET
|
104
|
+
access_token ACCESS_TOKEN
|
105
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
106
|
+
tag input.twitter
|
107
|
+
timeline tracking
|
108
|
+
keyword 'treasuredata,treasure data,#treasuredata,fluentd,#fluentd'
|
109
|
+
<proxy>
|
110
|
+
host proxy.example.com
|
111
|
+
port 8080
|
112
|
+
username proxyuser
|
113
|
+
password proxypass
|
114
|
+
</proxy>
|
115
|
+
<proxy>
|
116
|
+
host proxy.example.com
|
117
|
+
port 8081
|
118
|
+
username proxyuser
|
119
|
+
password proxypass
|
120
|
+
</proxy>
|
121
|
+
]
|
122
|
+
assert_raise(Fluent::ConfigError) do
|
123
|
+
create_driver(conf)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/test/driver/output'
|
3
|
+
require 'fluent/plugin/out_twitter'
|
4
|
+
|
5
|
+
class TwitterOutputTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
CONFIG = %[
|
11
|
+
consumer_key CONSUMER_KEY
|
12
|
+
consumer_secret CONSUMER_SECRET
|
13
|
+
access_token ACCESS_TOKEN
|
14
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
15
|
+
]
|
16
|
+
|
17
|
+
def create_driver(conf = CONFIG)
|
18
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::TwitterOutput).configure(conf)
|
19
|
+
end
|
20
|
+
|
21
|
+
sub_test_case "configure" do
|
22
|
+
def test_empty
|
23
|
+
assert_raise(Fluent::ConfigError) {
|
24
|
+
create_driver('')
|
25
|
+
}
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_configure
|
29
|
+
d = create_driver %[
|
30
|
+
consumer_key CONSUMER_KEY
|
31
|
+
consumer_secret CONSUMER_SECRET
|
32
|
+
access_token ACCESS_TOKEN
|
33
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
34
|
+
]
|
35
|
+
assert_equal 'CONSUMER_KEY', d.instance.consumer_key
|
36
|
+
assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
|
37
|
+
assert_equal 'ACCESS_TOKEN', d.instance.access_token
|
38
|
+
assert_equal 'ACCESS_TOKEN_SECRET', d.instance.access_token_secret
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_proxy
|
42
|
+
conf = %[
|
43
|
+
consumer_key CONSUMER_KEY
|
44
|
+
consumer_secret CONSUMER_SECRET
|
45
|
+
access_token ACCESS_TOKEN
|
46
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
47
|
+
<proxy>
|
48
|
+
host proxy.example.com
|
49
|
+
port 8080
|
50
|
+
username proxyuser
|
51
|
+
password proxypass
|
52
|
+
</proxy>
|
53
|
+
]
|
54
|
+
d = create_driver(conf)
|
55
|
+
expected = {
|
56
|
+
host: "proxy.example.com",
|
57
|
+
port: "8080",
|
58
|
+
username: "proxyuser",
|
59
|
+
password: "proxypass"
|
60
|
+
}
|
61
|
+
assert_equal(expected, d.instance.proxy.to_h)
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_multi_proxy
|
65
|
+
conf = %[
|
66
|
+
consumer_key CONSUMER_KEY
|
67
|
+
consumer_secret CONSUMER_SECRET
|
68
|
+
access_token ACCESS_TOKEN
|
69
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
70
|
+
<proxy>
|
71
|
+
host proxy.example.com
|
72
|
+
port 8080
|
73
|
+
username proxyuser
|
74
|
+
password proxypass
|
75
|
+
</proxy>
|
76
|
+
<proxy>
|
77
|
+
host proxy.example.com
|
78
|
+
port 8081
|
79
|
+
username proxyuser
|
80
|
+
password proxypass
|
81
|
+
</proxy>
|
82
|
+
]
|
83
|
+
assert_raise(Fluent::ConfigError) do
|
84
|
+
create_driver(conf)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
metadata
ADDED
@@ -0,0 +1,135 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-twitter-enchanced
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.6.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Kentaro Yoshida
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-04-28 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: test-unit
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 3.1.0
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 3.1.0
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: appraisal
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: fluentd
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.14.0
|
62
|
+
- - "<"
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: '2'
|
65
|
+
type: :runtime
|
66
|
+
prerelease: false
|
67
|
+
version_requirements: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: 0.14.0
|
72
|
+
- - "<"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '2'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: twitter
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - "~>"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '6.0'
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '6.0'
|
89
|
+
description:
|
90
|
+
email:
|
91
|
+
- y.ken.studio@gmail.com
|
92
|
+
executables: []
|
93
|
+
extensions: []
|
94
|
+
extra_rdoc_files: []
|
95
|
+
files:
|
96
|
+
- ".gitignore"
|
97
|
+
- ".travis.yml"
|
98
|
+
- Appraisals
|
99
|
+
- Gemfile
|
100
|
+
- LICENSE.txt
|
101
|
+
- README.md
|
102
|
+
- Rakefile
|
103
|
+
- fluent-plugin-twitter.gemspec
|
104
|
+
- gemfiles/fluentd_v0.14.gemfile
|
105
|
+
- lib/fluent/plugin/in_twitter.rb
|
106
|
+
- lib/fluent/plugin/out_twitter.rb
|
107
|
+
- test/helper.rb
|
108
|
+
- test/plugin/test_in_twitter.rb
|
109
|
+
- test/plugin/test_out_twitter.rb
|
110
|
+
homepage: https://github.com/y-ken/fluent-plugin-twitter
|
111
|
+
licenses:
|
112
|
+
- Apache-2.0
|
113
|
+
metadata: {}
|
114
|
+
post_install_message:
|
115
|
+
rdoc_options: []
|
116
|
+
require_paths:
|
117
|
+
- lib
|
118
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">"
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '2.1'
|
123
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - ">="
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '0'
|
128
|
+
requirements: []
|
129
|
+
rubyforge_project:
|
130
|
+
rubygems_version: 2.6.8
|
131
|
+
signing_key:
|
132
|
+
specification_version: 4
|
133
|
+
summary: Fluentd Input/Output plugin to collect/process tweets with Twitter Streaming
|
134
|
+
API.
|
135
|
+
test_files: []
|