fluent-plugin-twitter-enchanced 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.travis.yml +15 -0
- data/Appraisals +3 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +14 -0
- data/README.md +154 -0
- data/Rakefile +10 -0
- data/fluent-plugin-twitter.gemspec +25 -0
- data/gemfiles/fluentd_v0.14.gemfile +7 -0
- data/lib/fluent/plugin/in_twitter.rb +168 -0
- data/lib/fluent/plugin/out_twitter.rb +48 -0
- data/test/helper.rb +6 -0
- data/test/plugin/test_in_twitter.rb +127 -0
- data/test/plugin/test_out_twitter.rb +88 -0
- metadata +135 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 12ae0331f9ea7e2a19a23bdbbb7118a42d6dc202
|
4
|
+
data.tar.gz: fae48822ad3705aaa6f7e1a756d2758899b9920b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 59e503ed5d3f130c97952b91f5bcabc805d5f5b40ffa362d31027649b3021736294612b3435793dcd856ebe095af35959c6fe385c87e37e0d9bd4c16bb7ccf9f
|
7
|
+
data.tar.gz: 515628a4b9c8b36c12bb64ef826a5412cb6098ec7da13e90c011164a877fef1a9aa9f3e28c9d3ec0970bc69a416305f518e2629d244655f076c8ff20d6242211
|
data/.travis.yml
ADDED
data/Appraisals
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Copyright (c) 2012- Kentaro Yoshida
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
14
|
+
|
data/README.md
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
fluent-plugin-twitter [](https://travis-ci.org/y-ken/fluent-plugin-twitter)
|
2
|
+
=====================
|
3
|
+
|
4
|
+
## Component
|
5
|
+
Fluentd Input/Output plugin to process tweets with Twitter Streaming API.
|
6
|
+
|
7
|
+
## Dependency
|
8
|
+
|
9
|
+
before use, install dependent library as:
|
10
|
+
|
11
|
+
```
|
12
|
+
# for RHEL/CentOS (eventmachine requires build dependency)
|
13
|
+
$ sudo yum -y install gcc gcc-c++ openssl-devel libcurl libcurl-devel
|
14
|
+
|
15
|
+
# for Ubuntu/Debian (eventmachine requires build dependency)
|
16
|
+
$ sudo apt-get install build-essential libssl-dev
|
17
|
+
```
|
18
|
+
|
19
|
+
## Requirements
|
20
|
+
|
21
|
+
| fluent-plugin-twitter | fluentd | ruby |
|
22
|
+
|--------------------|------------|--------|
|
23
|
+
| 0.6.1 | v0.14.x | >= 2.1 |
|
24
|
+
| 0.5.4 | v0.12.x | >= 1.9 |
|
25
|
+
|
26
|
+
## Installation
|
27
|
+
|
28
|
+
install with `gem` or `td-agent-gem` command as:
|
29
|
+
|
30
|
+
```
|
31
|
+
# for fluentd
|
32
|
+
$ gem install eventmachine
|
33
|
+
$ gem install fluent-plugin-twitter
|
34
|
+
|
35
|
+
# for td-agent2
|
36
|
+
$ sudo td-agent-gem install eventmachine
|
37
|
+
$ sudo td-agent-gem install fluent-plugin-twitter -v 0.5.4
|
38
|
+
```
|
39
|
+
|
40
|
+
## Input Configuration
|
41
|
+
|
42
|
+
### Input Sample
|
43
|
+
|
44
|
+
It require td-agent2 (fluentd v0.12) to use keyword with hashtag.
|
45
|
+
|
46
|
+
`````
|
47
|
+
<source>
|
48
|
+
@type twitter
|
49
|
+
consumer_key YOUR_CONSUMER_KEY # Required
|
50
|
+
consumer_secret YOUR_CONSUMER_SECRET # Required
|
51
|
+
access_token YOUR_ACCESS_TOKEN # Required
|
52
|
+
access_token_secret YOUR_ACCESS_TOKEN_SECRET # Required
|
53
|
+
tag input.twitter.sampling # Required
|
54
|
+
timeline tracking # Required (tracking or sampling or location or userstream)
|
55
|
+
keyword 'Ruby,Python,#fleuntd' # Optional (keyword has priority than follow_ids)
|
56
|
+
follow_ids 14252,53235 # Optional (integers, not screen names)
|
57
|
+
locations 31.110283, 129.431631, 45.619283, 145.510175 # Optional (bounding boxes; first pair specifies longitude/latitude of southwest corner)
|
58
|
+
lang ja,en # Optional
|
59
|
+
output_format nest # Optional (nest or flat or simple[default])
|
60
|
+
flatten_separator _ # Optional
|
61
|
+
</source>
|
62
|
+
|
63
|
+
<match input.twitter.sampling>
|
64
|
+
@type stdout
|
65
|
+
</match>
|
66
|
+
`````
|
67
|
+
|
68
|
+
### Proxy support
|
69
|
+
|
70
|
+
```
|
71
|
+
<source>
|
72
|
+
@type twitter
|
73
|
+
consumer_key YOUR_CONSUMER_KEY # Required
|
74
|
+
consumer_secret YOUR_CONSUMER_SECRET # Required
|
75
|
+
access_token YOUR_ACCESS_TOKEN # Required
|
76
|
+
access_token_secret YOUR_ACCESS_TOKEN_SECRET # Required
|
77
|
+
tag input.twitter.sampling # Required
|
78
|
+
timeline tracking # Required (tracking or sampling or location or userstream)
|
79
|
+
<proxy>
|
80
|
+
host proxy.example.com # Required
|
81
|
+
port 8080 # Required
|
82
|
+
username proxyuser # Optional
|
83
|
+
password proxypass # Optional
|
84
|
+
</proxy>
|
85
|
+
</source>
|
86
|
+
```
|
87
|
+
|
88
|
+
### Testing
|
89
|
+
|
90
|
+
`````
|
91
|
+
$ tail -f /var/log/td-agent/td-agent.log
|
92
|
+
`````
|
93
|
+
|
94
|
+
## Output Configuration
|
95
|
+
|
96
|
+
### Output Sample
|
97
|
+
`````
|
98
|
+
<source>
|
99
|
+
@type http
|
100
|
+
port 8888
|
101
|
+
</source>
|
102
|
+
|
103
|
+
<match notify.twitter>
|
104
|
+
@type twitter
|
105
|
+
consumer_key YOUR_CONSUMER_KEY
|
106
|
+
consumer_secret YOUR_CONSUMER_SECRET
|
107
|
+
access_token YOUR_ACCESS_TOKEN
|
108
|
+
access_token_secret YOUR_ACCESS_TOKEN_SECRET
|
109
|
+
</match>
|
110
|
+
`````
|
111
|
+
|
112
|
+
### Proxy support
|
113
|
+
|
114
|
+
```
|
115
|
+
<match notify.twitter>
|
116
|
+
@type twitter
|
117
|
+
consumer_key YOUR_CONSUMER_KEY
|
118
|
+
consumer_secret YOUR_CONSUMER_SECRET
|
119
|
+
access_token YOUR_ACCESS_TOKEN
|
120
|
+
access_token_secret YOUR_ACCESS_TOKEN_SECRET
|
121
|
+
<proxy>
|
122
|
+
host proxy.example.com
|
123
|
+
port 8080
|
124
|
+
username proxyuser
|
125
|
+
password proxypass
|
126
|
+
</proxy>
|
127
|
+
</match>
|
128
|
+
```
|
129
|
+
|
130
|
+
### Testing
|
131
|
+
|
132
|
+
`````
|
133
|
+
$ curl http://localhost:8888/notify.twitter -F 'json={"message":"foo"}'
|
134
|
+
`````
|
135
|
+
|
136
|
+
## Reference
|
137
|
+
|
138
|
+
### Twitter OAuth Guide
|
139
|
+
http://pocketstudio.jp/log3/2012/02/12/how_to_get_twitter_apikey_and_token/
|
140
|
+
|
141
|
+
### Quick Tour to count a tweet with fluent-plugin-twitter and fluent-plugin-datacounter
|
142
|
+
http://qiita.com/items/fe4258b394190f23fece
|
143
|
+
|
144
|
+
## TODO
|
145
|
+
|
146
|
+
patches welcome!
|
147
|
+
|
148
|
+
## Copyright
|
149
|
+
|
150
|
+
Copyright © 2012- Kentaro Yoshida (@yoshi_ken)
|
151
|
+
|
152
|
+
## License
|
153
|
+
|
154
|
+
Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
$:.push File.expand_path("../lib", __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = "fluent-plugin-twitter-enchanced"
|
5
|
+
s.version = "0.6.1"
|
6
|
+
s.authors = ["Kentaro Yoshida"]
|
7
|
+
s.email = ["y.ken.studio@gmail.com"]
|
8
|
+
s.homepage = "https://github.com/y-ken/fluent-plugin-twitter"
|
9
|
+
s.summary = %q{Fluentd Input/Output plugin to collect/process tweets with Twitter Streaming API.}
|
10
|
+
s.license = "Apache-2.0"
|
11
|
+
|
12
|
+
s.files = `git ls-files`.split("\n")
|
13
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
14
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
15
|
+
s.require_paths = ["lib"]
|
16
|
+
|
17
|
+
s.required_ruby_version = "> 2.1"
|
18
|
+
|
19
|
+
s.add_development_dependency "rake"
|
20
|
+
s.add_development_dependency "test-unit", ">= 3.1.0"
|
21
|
+
s.add_development_dependency "appraisal"
|
22
|
+
|
23
|
+
s.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
|
24
|
+
s.add_runtime_dependency "twitter", "~> 6.0"
|
25
|
+
end
|
@@ -0,0 +1,168 @@
|
|
1
|
+
require "fluent/plugin/input"
|
2
|
+
|
3
|
+
require 'twitter'
|
4
|
+
require 'nkf'
|
5
|
+
|
6
|
+
module Fluent::Plugin
|
7
|
+
class TwitterInput < Fluent::Plugin::Input
|
8
|
+
Fluent::Plugin.register_input('twitter', self)
|
9
|
+
|
10
|
+
helpers :thread
|
11
|
+
|
12
|
+
TIMELINE_TYPE = %i(userstream sampling location tracking)
|
13
|
+
OUTPUT_FORMAT_TYPE = %i(nest flat simple)
|
14
|
+
|
15
|
+
config_param :consumer_key, :string, secret: true
|
16
|
+
config_param :consumer_secret, :string, secret: true
|
17
|
+
config_param :access_token, :string, secret: true
|
18
|
+
config_param :access_token_secret, :string, secret: true
|
19
|
+
config_param :tag, :string
|
20
|
+
config_param :timeline, :enum, list: TIMELINE_TYPE
|
21
|
+
config_param :keyword, :string, default: nil
|
22
|
+
config_param :follow_ids, :string, default: nil
|
23
|
+
config_param :locations, :string, default: nil
|
24
|
+
config_param :lang, :string, default: nil
|
25
|
+
config_param :output_format, :enum, list: OUTPUT_FORMAT_TYPE, default: :simple
|
26
|
+
config_param :flatten_separator, :string, default: '_'
|
27
|
+
|
28
|
+
config_section :proxy, multi: false do
|
29
|
+
config_param :host, :string
|
30
|
+
config_param :port, :string
|
31
|
+
config_param :username, :string, default: nil
|
32
|
+
config_param :password, :string, default: nil, secret: true
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize
|
36
|
+
super
|
37
|
+
@running = false
|
38
|
+
end
|
39
|
+
|
40
|
+
def configure(conf)
|
41
|
+
super
|
42
|
+
|
43
|
+
@keyword = @keyword.gsub('${hashtag}', '#') unless @keyword.nil?
|
44
|
+
|
45
|
+
@client = Twitter::Streaming::Client.new do |config|
|
46
|
+
config.consumer_key = @consumer_key
|
47
|
+
config.consumer_secret = @consumer_secret
|
48
|
+
config.access_token = @access_token
|
49
|
+
config.access_token_secret = @access_token_secret
|
50
|
+
config.proxy = @proxy.to_h if @proxy
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def start
|
55
|
+
@running = true
|
56
|
+
thread_create(:in_twitter) do
|
57
|
+
run
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def shutdown
|
62
|
+
@running = false
|
63
|
+
@client.close if @client.respond_to?(:close)
|
64
|
+
super
|
65
|
+
end
|
66
|
+
|
67
|
+
def run
|
68
|
+
notice = "twitter: starting Twitter Streaming API for #{@timeline}."
|
69
|
+
notice << " tag:#{@tag}"
|
70
|
+
notice << " lang:#{@lang}" unless @lang.nil?
|
71
|
+
notice << " keyword:#{@keyword}" unless @keyword.nil?
|
72
|
+
notice << " follow:#{@follow_ids}" unless @follow_ids.nil? && !@keyword.nil?
|
73
|
+
log.info notice
|
74
|
+
|
75
|
+
if [:sampling, :tracking].include?(@timeline) && @keyword
|
76
|
+
@client.filter(track: @keyword, &method(:handle_object))
|
77
|
+
elsif @timeline == :tracking && @follow_ids
|
78
|
+
@client.filter(follow: @follow_ids, &method(:handle_object))
|
79
|
+
elsif @timeline == :sampling && @keyword.nil? && @follow_ids.nil?
|
80
|
+
@client.sample(&method(:handle_object))
|
81
|
+
elsif @timeline == :userstream
|
82
|
+
@client.user(&method(:handle_object))
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def handle_object(object)
|
87
|
+
return unless @running
|
88
|
+
if is_message?(object)
|
89
|
+
get_message(object)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def is_message?(tweet)
|
94
|
+
return false if !tweet.is_a?(Twitter::Tweet)
|
95
|
+
return false if (!@lang.nil? && @lang != '') && !@lang.include?(tweet.user.lang)
|
96
|
+
if @timeline == :userstream && (!@keyword.nil? && @keyword != '')
|
97
|
+
pattern = NKF::nkf('-WwZ1', @keyword).gsub(/,\s?/, '|')
|
98
|
+
tweet = NKF::nkf('-WwZ1', tweet.text)
|
99
|
+
return false if !Regexp.new(pattern, Regexp::IGNORECASE).match(tweet)
|
100
|
+
end
|
101
|
+
return true
|
102
|
+
end
|
103
|
+
|
104
|
+
def get_message(tweet)
|
105
|
+
case @output_format
|
106
|
+
when :nest
|
107
|
+
record = hash_key_to_s(tweet.to_h)
|
108
|
+
when :flat
|
109
|
+
record = hash_flatten(tweet.to_h)
|
110
|
+
when :simple
|
111
|
+
record = {}
|
112
|
+
record['message'] = tweet.text.scrub('')
|
113
|
+
record['geo'] = tweet.geo
|
114
|
+
record['place'] = tweet.place
|
115
|
+
record['created_at'] = tweet.created_at.to_s
|
116
|
+
record['user_name'] = tweet.user.name
|
117
|
+
record['user_screen_name'] = tweet.user.screen_name
|
118
|
+
record['user_profile_image_url'] = tweet.user.profile_image_url.to_s
|
119
|
+
record['user_time_zone'] = tweet.user.time_zone
|
120
|
+
record['user_lang'] = tweet.user.lang
|
121
|
+
end
|
122
|
+
router.emit(@tag, Fluent::Engine.now, record)
|
123
|
+
end
|
124
|
+
|
125
|
+
def hash_flatten(record, prefix = nil)
|
126
|
+
record.inject({}) do |d, (k, v)|
|
127
|
+
k = prefix.to_s + k.to_s
|
128
|
+
if v.instance_of?(Hash)
|
129
|
+
d.merge(hash_flatten(v, k + @flatten_separator))
|
130
|
+
elsif v.instance_of?(String)
|
131
|
+
d.merge(k => v.scrub(""))
|
132
|
+
else
|
133
|
+
d.merge(k => v)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def hash_key_to_s(hash)
|
139
|
+
newhash = {}
|
140
|
+
hash.each do |k, v|
|
141
|
+
if v.instance_of?(Hash)
|
142
|
+
newhash[k.to_s] = hash_key_to_s(v)
|
143
|
+
elsif v.instance_of?(Array)
|
144
|
+
newhash[k.to_s] = array_key_to_s(v)
|
145
|
+
elsif v.instance_of?(String)
|
146
|
+
newhash[k.to_s] = v.scrub('')
|
147
|
+
else
|
148
|
+
newhash[k.to_s] = v
|
149
|
+
end
|
150
|
+
end
|
151
|
+
newhash
|
152
|
+
end
|
153
|
+
|
154
|
+
def array_key_to_s(array)
|
155
|
+
array.map do |v|
|
156
|
+
if v.instance_of?(Hash)
|
157
|
+
hash_key_to_s(v)
|
158
|
+
elsif v.instance_of?(Array)
|
159
|
+
array_key_to_s(v)
|
160
|
+
elsif v.instance_of?(String)
|
161
|
+
v.scrub('')
|
162
|
+
else
|
163
|
+
v
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require "twitter"
|
2
|
+
require "fluent/plugin/output"
|
3
|
+
|
4
|
+
class Fluent::Plugin::TwitterOutput < Fluent::Plugin::Output
|
5
|
+
Fluent::Plugin.register_output('twitter', self)
|
6
|
+
|
7
|
+
config_param :consumer_key, :string, secret: true
|
8
|
+
config_param :consumer_secret, :string, secret: true
|
9
|
+
config_param :access_token, :string, secret: true
|
10
|
+
config_param :access_token_secret, :string, secret: true
|
11
|
+
|
12
|
+
config_section :proxy, multi: false do
|
13
|
+
config_param :host, :string
|
14
|
+
config_param :port, :string
|
15
|
+
config_param :username, :string, default: nil
|
16
|
+
config_param :password, :string, default: nil, secret: true
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize
|
20
|
+
super
|
21
|
+
end
|
22
|
+
|
23
|
+
def configure(conf)
|
24
|
+
super
|
25
|
+
|
26
|
+
@twitter = Twitter::REST::Client.new do |config|
|
27
|
+
config.consumer_key = @consumer_key
|
28
|
+
config.consumer_secret = @consumer_secret
|
29
|
+
config.access_token = @access_token
|
30
|
+
config.access_token_secret = @access_token_secret
|
31
|
+
config.proxy = @proxy.to_h if @proxy
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def process(tag, es)
|
36
|
+
es.each do |_time, record|
|
37
|
+
tweet(record['message'])
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def tweet(message)
|
42
|
+
begin
|
43
|
+
@twitter.update(message)
|
44
|
+
rescue Twitter::Error => e
|
45
|
+
log.error("Twitter Error: #{e.message}")
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/plugin/in_twitter'
|
3
|
+
require 'fluent/test/driver/input'
|
4
|
+
|
5
|
+
class TwitterInputTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
CONFIG = %[
|
11
|
+
consumer_key CONSUMER_KEY
|
12
|
+
consumer_secret CONSUMER_SECRET
|
13
|
+
access_token ACCESS_TOKEN
|
14
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
15
|
+
tag input.twitter
|
16
|
+
timeline sampling
|
17
|
+
]
|
18
|
+
|
19
|
+
def create_driver(conf = CONFIG, syntax: :v1)
|
20
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::TwitterInput).configure(conf, syntax: syntax)
|
21
|
+
end
|
22
|
+
|
23
|
+
sub_test_case "v0 syntax" do
|
24
|
+
def test_empty
|
25
|
+
assert_raise(Fluent::ConfigError) {
|
26
|
+
create_driver('', syntax: :v0)
|
27
|
+
}
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_configure
|
31
|
+
d = create_driver %[
|
32
|
+
consumer_key CONSUMER_KEY
|
33
|
+
consumer_secret CONSUMER_SECRET
|
34
|
+
access_token ACCESS_TOKEN
|
35
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
36
|
+
tag input.twitter
|
37
|
+
timeline tracking
|
38
|
+
keyword ${hashtag}fluentd,fluentd lang:ja
|
39
|
+
]
|
40
|
+
assert_equal 'CONSUMER_KEY', d.instance.consumer_key
|
41
|
+
assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
|
42
|
+
assert_equal 'ACCESS_TOKEN', d.instance.access_token
|
43
|
+
assert_equal 'ACCESS_TOKEN_SECRET', d.instance.access_token_secret
|
44
|
+
assert_equal '#fluentd,fluentd lang:ja', d.instance.keyword
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
sub_test_case "v1 syntax" do
|
49
|
+
def test_empty
|
50
|
+
assert_raise(Fluent::ConfigError) {
|
51
|
+
create_driver('')
|
52
|
+
}
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_multi_keyword
|
56
|
+
d = create_driver(%[
|
57
|
+
consumer_key CONSUMER_KEY
|
58
|
+
consumer_secret CONSUMER_SECRET
|
59
|
+
access_token ACCESS_TOKEN
|
60
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
61
|
+
tag input.twitter
|
62
|
+
timeline tracking
|
63
|
+
keyword 'treasuredata,treasure data,#treasuredata,fluentd,#fluentd'
|
64
|
+
])
|
65
|
+
assert_equal 'CONSUMER_KEY', d.instance.consumer_key
|
66
|
+
assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
|
67
|
+
assert_equal 'ACCESS_TOKEN', d.instance.access_token
|
68
|
+
assert_equal 'ACCESS_TOKEN_SECRET', d.instance.access_token_secret
|
69
|
+
assert_equal 'treasuredata,treasure data,#treasuredata,fluentd,#fluentd', d.instance.keyword
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
sub_test_case "proxy" do
|
74
|
+
test "simple" do
|
75
|
+
conf = %[
|
76
|
+
consumer_key CONSUMER_KEY
|
77
|
+
consumer_secret CONSUMER_SECRET
|
78
|
+
access_token ACCESS_TOKEN
|
79
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
80
|
+
tag input.twitter
|
81
|
+
timeline tracking
|
82
|
+
keyword 'treasuredata,treasure data,#treasuredata,fluentd,#fluentd'
|
83
|
+
<proxy>
|
84
|
+
host proxy.example.com
|
85
|
+
port 8080
|
86
|
+
username proxyuser
|
87
|
+
password proxypass
|
88
|
+
</proxy>
|
89
|
+
]
|
90
|
+
d = create_driver(conf)
|
91
|
+
expected = {
|
92
|
+
host: "proxy.example.com",
|
93
|
+
port: "8080",
|
94
|
+
username: "proxyuser",
|
95
|
+
password: "proxypass"
|
96
|
+
}
|
97
|
+
assert_equal(expected, d.instance.proxy.to_h)
|
98
|
+
end
|
99
|
+
|
100
|
+
test "multi proxy is not supported" do
|
101
|
+
conf = %[
|
102
|
+
consumer_key CONSUMER_KEY
|
103
|
+
consumer_secret CONSUMER_SECRET
|
104
|
+
access_token ACCESS_TOKEN
|
105
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
106
|
+
tag input.twitter
|
107
|
+
timeline tracking
|
108
|
+
keyword 'treasuredata,treasure data,#treasuredata,fluentd,#fluentd'
|
109
|
+
<proxy>
|
110
|
+
host proxy.example.com
|
111
|
+
port 8080
|
112
|
+
username proxyuser
|
113
|
+
password proxypass
|
114
|
+
</proxy>
|
115
|
+
<proxy>
|
116
|
+
host proxy.example.com
|
117
|
+
port 8081
|
118
|
+
username proxyuser
|
119
|
+
password proxypass
|
120
|
+
</proxy>
|
121
|
+
]
|
122
|
+
assert_raise(Fluent::ConfigError) do
|
123
|
+
create_driver(conf)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/test/driver/output'
|
3
|
+
require 'fluent/plugin/out_twitter'
|
4
|
+
|
5
|
+
class TwitterOutputTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
CONFIG = %[
|
11
|
+
consumer_key CONSUMER_KEY
|
12
|
+
consumer_secret CONSUMER_SECRET
|
13
|
+
access_token ACCESS_TOKEN
|
14
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
15
|
+
]
|
16
|
+
|
17
|
+
def create_driver(conf = CONFIG)
|
18
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::TwitterOutput).configure(conf)
|
19
|
+
end
|
20
|
+
|
21
|
+
sub_test_case "configure" do
|
22
|
+
def test_empty
|
23
|
+
assert_raise(Fluent::ConfigError) {
|
24
|
+
create_driver('')
|
25
|
+
}
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_configure
|
29
|
+
d = create_driver %[
|
30
|
+
consumer_key CONSUMER_KEY
|
31
|
+
consumer_secret CONSUMER_SECRET
|
32
|
+
access_token ACCESS_TOKEN
|
33
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
34
|
+
]
|
35
|
+
assert_equal 'CONSUMER_KEY', d.instance.consumer_key
|
36
|
+
assert_equal 'CONSUMER_SECRET', d.instance.consumer_secret
|
37
|
+
assert_equal 'ACCESS_TOKEN', d.instance.access_token
|
38
|
+
assert_equal 'ACCESS_TOKEN_SECRET', d.instance.access_token_secret
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_proxy
|
42
|
+
conf = %[
|
43
|
+
consumer_key CONSUMER_KEY
|
44
|
+
consumer_secret CONSUMER_SECRET
|
45
|
+
access_token ACCESS_TOKEN
|
46
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
47
|
+
<proxy>
|
48
|
+
host proxy.example.com
|
49
|
+
port 8080
|
50
|
+
username proxyuser
|
51
|
+
password proxypass
|
52
|
+
</proxy>
|
53
|
+
]
|
54
|
+
d = create_driver(conf)
|
55
|
+
expected = {
|
56
|
+
host: "proxy.example.com",
|
57
|
+
port: "8080",
|
58
|
+
username: "proxyuser",
|
59
|
+
password: "proxypass"
|
60
|
+
}
|
61
|
+
assert_equal(expected, d.instance.proxy.to_h)
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_multi_proxy
|
65
|
+
conf = %[
|
66
|
+
consumer_key CONSUMER_KEY
|
67
|
+
consumer_secret CONSUMER_SECRET
|
68
|
+
access_token ACCESS_TOKEN
|
69
|
+
access_token_secret ACCESS_TOKEN_SECRET
|
70
|
+
<proxy>
|
71
|
+
host proxy.example.com
|
72
|
+
port 8080
|
73
|
+
username proxyuser
|
74
|
+
password proxypass
|
75
|
+
</proxy>
|
76
|
+
<proxy>
|
77
|
+
host proxy.example.com
|
78
|
+
port 8081
|
79
|
+
username proxyuser
|
80
|
+
password proxypass
|
81
|
+
</proxy>
|
82
|
+
]
|
83
|
+
assert_raise(Fluent::ConfigError) do
|
84
|
+
create_driver(conf)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
metadata
ADDED
@@ -0,0 +1,135 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-twitter-enchanced
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.6.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Kentaro Yoshida
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-04-28 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: test-unit
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 3.1.0
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 3.1.0
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: appraisal
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: fluentd
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.14.0
|
62
|
+
- - "<"
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: '2'
|
65
|
+
type: :runtime
|
66
|
+
prerelease: false
|
67
|
+
version_requirements: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: 0.14.0
|
72
|
+
- - "<"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '2'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: twitter
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - "~>"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '6.0'
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '6.0'
|
89
|
+
description:
|
90
|
+
email:
|
91
|
+
- y.ken.studio@gmail.com
|
92
|
+
executables: []
|
93
|
+
extensions: []
|
94
|
+
extra_rdoc_files: []
|
95
|
+
files:
|
96
|
+
- ".gitignore"
|
97
|
+
- ".travis.yml"
|
98
|
+
- Appraisals
|
99
|
+
- Gemfile
|
100
|
+
- LICENSE.txt
|
101
|
+
- README.md
|
102
|
+
- Rakefile
|
103
|
+
- fluent-plugin-twitter.gemspec
|
104
|
+
- gemfiles/fluentd_v0.14.gemfile
|
105
|
+
- lib/fluent/plugin/in_twitter.rb
|
106
|
+
- lib/fluent/plugin/out_twitter.rb
|
107
|
+
- test/helper.rb
|
108
|
+
- test/plugin/test_in_twitter.rb
|
109
|
+
- test/plugin/test_out_twitter.rb
|
110
|
+
homepage: https://github.com/y-ken/fluent-plugin-twitter
|
111
|
+
licenses:
|
112
|
+
- Apache-2.0
|
113
|
+
metadata: {}
|
114
|
+
post_install_message:
|
115
|
+
rdoc_options: []
|
116
|
+
require_paths:
|
117
|
+
- lib
|
118
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">"
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '2.1'
|
123
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - ">="
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '0'
|
128
|
+
requirements: []
|
129
|
+
rubyforge_project:
|
130
|
+
rubygems_version: 2.6.8
|
131
|
+
signing_key:
|
132
|
+
specification_version: 4
|
133
|
+
summary: Fluentd Input/Output plugin to collect/process tweets with Twitter Streaming
|
134
|
+
API.
|
135
|
+
test_files: []
|