fluent-plugin-twitter 0.3.2 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +23 -20
- data/fluent-plugin-twitter.gemspec +2 -1
- data/lib/fluent/plugin/in_twitter.rb +15 -7
- data/test/plugin/test_in_twitter.rb +1 -1
- metadata +18 -2
data/README.md
CHANGED
@@ -8,9 +8,9 @@ Fluentd Input/Output plugin to process tweets with Twitter Streaming API.
|
|
8
8
|
|
9
9
|
before use, install dependent library as:
|
10
10
|
|
11
|
-
```
|
11
|
+
```
|
12
12
|
# for RHEL/CentOS
|
13
|
-
$ sudo yum install openssl-devel
|
13
|
+
$ sudo yum -y install openssl-devel libcurl libcurl-devel
|
14
14
|
|
15
15
|
# for Ubuntu/Debian
|
16
16
|
$ sudo apt-get install libssl-dev
|
@@ -18,20 +18,26 @@ $ sudo apt-get install libssl-dev
|
|
18
18
|
|
19
19
|
## Installation
|
20
20
|
|
21
|
-
|
21
|
+
install with `gem` or `fluent-gem` command as:
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
23
|
+
```
|
24
|
+
# for fluentd
|
25
|
+
$ gem install eventmachine
|
26
|
+
$ gem install fluent-plugin-twitter
|
26
27
|
|
27
|
-
|
28
|
-
|
29
|
-
/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-twitter
|
30
|
-
|
28
|
+
# for td-agent
|
29
|
+
$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install eventmachine
|
30
|
+
$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-twitter
|
31
|
+
|
32
|
+
# for td-agent2
|
33
|
+
$ sudo td-agent-gem install eventmachine
|
34
|
+
$ sudo td-agent-gem install fluent-plugin-twitter
|
35
|
+
```
|
31
36
|
|
32
37
|
## Input Configuration
|
33
38
|
|
34
39
|
### Input Sample
|
40
|
+
|
35
41
|
`````
|
36
42
|
<source>
|
37
43
|
type twitter
|
@@ -40,8 +46,9 @@ gem install fluent-plugin-twitter
|
|
40
46
|
oauth_token YOUR_OAUTH_TOKEN # Required
|
41
47
|
oauth_token_secret YOUR_OAUTH_TOKEN_SECRET # Required
|
42
48
|
tag input.twitter.sampling # Required
|
43
|
-
timeline
|
44
|
-
keyword Ruby,Python # Optional
|
49
|
+
timeline tracking # Required (tracking or sampling or userstream)
|
50
|
+
keyword Ruby,Python # Optional (keyword is priority than follow_ids)
|
51
|
+
follow_ids 14252,53235 # Optional (integers, not screen names)
|
45
52
|
lang ja,en # Optional
|
46
53
|
output_format nest # Optional (nest or flat or simple[default])
|
47
54
|
</source>
|
@@ -51,7 +58,8 @@ gem install fluent-plugin-twitter
|
|
51
58
|
</match>
|
52
59
|
`````
|
53
60
|
|
54
|
-
###
|
61
|
+
### Testing
|
62
|
+
|
55
63
|
`````
|
56
64
|
$ tail -f /var/log/td-agent/td-agent.log
|
57
65
|
`````
|
@@ -74,7 +82,8 @@ $ tail -f /var/log/td-agent/td-agent.log
|
|
74
82
|
</match>
|
75
83
|
`````
|
76
84
|
|
77
|
-
###
|
85
|
+
### Testing
|
86
|
+
|
78
87
|
`````
|
79
88
|
$ curl http://localhost:8888/notify.twitter -F 'json={"message":"foo"}'
|
80
89
|
`````
|
@@ -88,15 +97,9 @@ http://pocketstudio.jp/log3/2012/02/12/how_to_get_twitter_apikey_and_token/
|
|
88
97
|
http://qiita.com/items/fe4258b394190f23fece
|
89
98
|
|
90
99
|
## TODO
|
91
|
-
* support fetching twitter search
|
92
|
-
* support fetching specific user userstream
|
93
100
|
|
94
101
|
patches welcome!
|
95
102
|
|
96
|
-
## Known Issue
|
97
|
-
On starting fluentd, appearing alert message below. Please tell me how to fix up.
|
98
|
-
`/usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/eventmachine-1.0.0/lib/eventmachine.rb:1530: warning: already initialized constant EM`
|
99
|
-
|
100
103
|
## Copyright
|
101
104
|
|
102
105
|
Copyright © 2012- Kentaro Yoshida (@yoshi_ken)
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-twitter"
|
6
|
-
s.version = "0.
|
6
|
+
s.version = "0.4.0"
|
7
7
|
s.authors = ["Kentaro Yoshida"]
|
8
8
|
s.email = ["y.ken.studio@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/y-ken/fluent-plugin-twitter"
|
@@ -18,4 +18,5 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.add_runtime_dependency "fluentd", [">= 0.10.46"]
|
19
19
|
s.add_runtime_dependency "twitter", ">= 5.0.0"
|
20
20
|
s.add_runtime_dependency "tweetstream", [">= 2.6.1"]
|
21
|
+
s.add_runtime_dependency "string-scrub" if RUBY_VERSION < "2.1"
|
21
22
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module Fluent
|
2
2
|
class TwitterInput < Fluent::Input
|
3
|
-
TIMELINE_TYPE = %w(userstream sampling)
|
3
|
+
TIMELINE_TYPE = %w(userstream sampling tracking)
|
4
4
|
OUTPUT_FORMAT_TYPE = %w(nest flat simple)
|
5
5
|
Plugin.register_input('twitter', self)
|
6
6
|
|
@@ -11,6 +11,7 @@ module Fluent
|
|
11
11
|
config_param :tag, :string
|
12
12
|
config_param :timeline, :string
|
13
13
|
config_param :keyword, :string, :default => nil
|
14
|
+
config_param :follow_ids, :string, :default => nil
|
14
15
|
config_param :lang, :string, :default => nil
|
15
16
|
config_param :output_format, :string, :default => 'simple'
|
16
17
|
config_param :flatten_separator, :string, :default => '_'
|
@@ -55,14 +56,14 @@ module Fluent
|
|
55
56
|
|
56
57
|
def run
|
57
58
|
client = get_twitter_connection
|
58
|
-
if
|
59
|
+
if ['sampling', 'tracking'].include?(@timeline) && @keyword
|
59
60
|
client.track(@keyword)
|
60
|
-
elsif @timeline == '
|
61
|
+
elsif @timeline == 'tracking' && @follow_ids
|
62
|
+
client.follow(@follow_ids)
|
63
|
+
elsif @timeline == 'sampling' && @keyword.nil? && @follow_ids.nil?
|
61
64
|
client.sample
|
62
65
|
elsif @timeline == 'userstream'
|
63
66
|
client.userstream
|
64
|
-
#elsif @timeline == 'follow'
|
65
|
-
# client.follow(@follow_ids)
|
66
67
|
end
|
67
68
|
end
|
68
69
|
|
@@ -71,6 +72,7 @@ module Fluent
|
|
71
72
|
notice << " tag:#{@tag}"
|
72
73
|
notice << " lang:#{@lang}" unless @lang.nil?
|
73
74
|
notice << " keyword:#{@keyword}" unless @keyword.nil?
|
75
|
+
notice << " follow:#{@follow_ids}" unless @follow_ids.nil? && !@keyword.nil?
|
74
76
|
$log.info notice
|
75
77
|
client = TweetStream::Client.new
|
76
78
|
client.on_anything(&@any)
|
@@ -100,7 +102,7 @@ module Fluent
|
|
100
102
|
record = hash_flatten(status)
|
101
103
|
when 'simple'
|
102
104
|
record = Hash.new
|
103
|
-
record.store('message', status[:text])
|
105
|
+
record.store('message', status[:text]).scrub('')
|
104
106
|
record.store('geo', status[:geo])
|
105
107
|
record.store('place', status[:place])
|
106
108
|
record.store('created_at', status[:created_at])
|
@@ -116,8 +118,10 @@ module Fluent
|
|
116
118
|
def hash_flatten(record, prefix = nil)
|
117
119
|
record.inject({}) do |d, (k, v)|
|
118
120
|
k = prefix.to_s + k.to_s
|
119
|
-
if v.
|
121
|
+
if v.instance_of?(Hash)
|
120
122
|
d.merge(hash_flatten(v, k + @flatten_separator))
|
123
|
+
elsif v.instance_of?(String)
|
124
|
+
d.merge(k => v.scrub(""))
|
121
125
|
else
|
122
126
|
d.merge(k => v)
|
123
127
|
end
|
@@ -131,6 +135,8 @@ module Fluent
|
|
131
135
|
newhash[k.to_s] = hash_key_to_s(v)
|
132
136
|
elsif v.instance_of?(Array) then
|
133
137
|
newhash[k.to_s] = array_key_to_s(v)
|
138
|
+
elsif v.instance_of?(String)
|
139
|
+
newhash[k.to_s] = v.scrub('')
|
134
140
|
else
|
135
141
|
newhash[k.to_s] = v
|
136
142
|
end
|
@@ -144,6 +150,8 @@ module Fluent
|
|
144
150
|
hash_key_to_s(v)
|
145
151
|
elsif v.instance_of?(Array) then
|
146
152
|
array_key_to_s(v)
|
153
|
+
elsif v.instance_of?(String) then
|
154
|
+
v.scrub('')
|
147
155
|
else
|
148
156
|
v
|
149
157
|
end
|
@@ -28,7 +28,7 @@ class TwitterInputTest < Test::Unit::TestCase
|
|
28
28
|
oauth_token OAUTH_TOKEN
|
29
29
|
oauth_token_secret OAUTH_TOKEN_SECRET
|
30
30
|
tag input.twitter
|
31
|
-
timeline
|
31
|
+
timeline tracking
|
32
32
|
keyword ${hashtag}ruby lang:ja "happy hour" :)
|
33
33
|
]
|
34
34
|
assert_equal 'CONSUMER_KEY', d.instance.consumer_key
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-twitter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-
|
12
|
+
date: 2014-10-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -75,6 +75,22 @@ dependencies:
|
|
75
75
|
- - ! '>='
|
76
76
|
- !ruby/object:Gem::Version
|
77
77
|
version: 2.6.1
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: string-scrub
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :runtime
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
78
94
|
description:
|
79
95
|
email:
|
80
96
|
- y.ken.studio@gmail.com
|