powertrack 1.2.0 → 1.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- OTYwNTFiZWQwNTAwYzc2ZWJkYmIyNjA4YWNmNDYxYzMxNWY5ZmYzMA==
4
+ YTQwZjFmMTc0YTFhNTlmZWIwYzUyYzRlMWMzMjBiNDZkMjBjN2E5MA==
5
5
  data.tar.gz: !binary |-
6
- MzJhNTVlNzMwMjIwNWEyNzEzZDY3YTEwYjU2ZWM1NmQ3NzE3OWE4ZQ==
6
+ NzdkY2IwYTk3ZjVlMzdjZmQzYjZkMmM1ZjI5ZGIzMjIyODI2NTllMw==
7
7
  SHA512:
8
8
  metadata.gz: !binary |-
9
- ZTljMmM0NWQzMzhiYzU3NjE1Y2ZmYzIxNTA1OWE5MTczYzc3NTE5MDExYzNi
10
- MDJmODA5ZGQ2ODk0OGRlMzgyNDdhYjcwMGM5MjBkNDZhMzBmOWI0NjJjNzMx
11
- ZmNmMmU3MmNjMjA2Y2RjZmE1OGVmM2FkZTUwOWI4MTQ0ZDBmNjk=
9
+ MjhiOTk0NWVhYmRmYzUwNTIyNDI5ZDQ3OGMxYmM2NGI1Y2Y5NGJjZGU3ZTg2
10
+ MjAyYzFiZjcyYmIyNDE0ODk3MWNiNTIyMWE1Mzk1NWI1MDc4N2Y5YjE2YmJh
11
+ OTMxYjM1NjFjZmIxODczMTRmNzFjZWRhNGI0ODZkYjRlMWE1MGE=
12
12
  data.tar.gz: !binary |-
13
- ODJlYjQ1OTU5NTY0YjdhZTczMTY0OWExZmFlNTk1OGI0YzBiY2ZiY2Y4Yzcy
14
- NzQ4N2E0MGNmZDA4MTFmMzE0NzQyNDU0MGM5NTczNWMyZGEwMDQ1YzgyMjZk
15
- MzcxYTUzNjcxZjZhY2YyZTc2YjcxYTBkZjZiNjQyNjEyZDMzZWM=
13
+ NWJhNjA3MDIyMDkyMDFmZTViNzAxNmNmZTRiOThkYzQ0YWI3YmUwYWYyZTE2
14
+ OGMzZGM1NWY4NjJlNTYyOWJmMmUxNTI5NjJjYmZjZDk1YmI0NTc4NWZlYTdl
15
+ NzEzODgxMzYwMzNkYzlkZWQyZWQ2YTg0MWRmNmM3MjQ3YjI5NTc=
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- powertrack (1.2.0)
4
+ powertrack (1.3.0)
5
5
  em-http-request (~> 1.1)
6
6
  eventmachine (~> 1.0)
7
7
  exponential-backoff (~> 0.0.2)
data/History.txt CHANGED
@@ -1,3 +1,9 @@
1
+ v1.3.0
2
+ ------
3
+
4
+ * Support Replay v2
5
+ * Support specificities of rules v2 in PowerTrack::Rule class
6
+
1
7
  v1.2.0
2
8
  ------
3
9
 
data/README.md CHANGED
@@ -147,9 +147,14 @@ o The Backfill feature is configured by the ```:backfill_minutes``` option passe
147
147
  option to the ```PowerTrack::Stream``` initializer (which is simply ignored
148
148
  when v2 is turned on). The new option specifies a number of minutes of backfill
149
149
  data to receive.
150
- o The Replay feature still uses v1 even if you explicitly turn v2 on. Support
151
- for [Replay v2](http://support.gnip.com/apis/replay2.0/api_reference.html) is
152
- planned but not scheduled yet.
150
+ o A v2 ```PowerTrack::Rule``` instance (initialized by passing the ```v2: true```
151
+ feature to the constructor) has a few specificities described in
152
+ [Migrating PowerTrack Rules from Version 1.0 to 2.0](http://support.gnip.com/articles/migrating-powertrack-rules.html).
153
+
154
+ In particular,
155
+ o it is always long (accepting up to 2048 characters),
156
+ o it has no limits on the number of positive or negative terms used,
157
+ o it forbids the usage of *AND*, *or* and *NOT* logical phrases.
153
158
 
154
159
  Finally, PowerTrack v2 has a new endpoint for rule validation that is not
155
160
  supported by this library yet.
data/TODO.md CHANGED
@@ -99,5 +99,5 @@ and [PowerTrack API v2](http://support.gnip.com/apis/powertrack2.0/index.html).
99
99
  stream was disconnected more than 5 minutes (tweets were probably lost)
100
100
  * _[DONE]_ Use HTTP POST verb (instead of DELETE) for rule deletions
101
101
  * _[DONE]_ Fallback to v1 when Replay mode wants to use v2. Emit a warning.
102
- * Support Replay v2
102
+ * _[DONE]_ Support Replay v2
103
103
  [Replay API 2.0 Reference](http://support.gnip.com/apis/replay2.0/api_reference.html)
@@ -73,6 +73,14 @@ module PowerTrack
73
73
  end
74
74
  end
75
75
 
76
+ # Invalid Endpoint. This occurs when your client tries to connect to an
77
+ # endpoint URL that does not exist.
78
+ class InvalidEndpoint < PredefinedStatusPowerTrackError
79
+ def initialize(message, body)
80
+ super(404, message, body)
81
+ end
82
+ end
83
+
76
84
  # Generally, this occurs where your client fails to properly include the
77
85
  # headers to accept gzip encoding from the stream, but can occur in other
78
86
  # circumstances as well.
@@ -19,6 +19,13 @@ module PowerTrack
19
19
  # The maximum number of negative terms in a single rule value
20
20
  MAX_NEGATIVE_TERMS = 50
21
21
 
22
+ # The maximum size of the HTTP body accepted by PowerTrack /rules calls (in bytes)
23
+ # 1 MB for v1, 5MB for v2
24
+ MAX_RULES_BODY_SIZE = {
25
+ v1: 1024**2,
26
+ v2: 5*1024**2
27
+ }
28
+
22
29
  # The default rule features
23
30
  DEFAULT_RULE_FEATURES = {
24
31
  # no id by default
@@ -26,25 +33,32 @@ module PowerTrack
26
33
  # no tag by default
27
34
  tag: nil,
28
35
  # long determined by value length
29
- long: nil
36
+ long: nil,
37
+ # v1 by default
38
+ v2: false
30
39
  }.freeze
31
40
 
32
41
  attr_reader :value, :id, :tag, :error
33
42
 
34
43
  # Builds a new rule based on a value and some optional features
35
- # (:id, :tag, :long).
44
+ # (:id, :tag, :long, :v2).
36
45
  #
37
46
  # By default, the constructor assesses if it's a long rule or not
38
47
  # based on the length of the value. But the 'long' feature can be
39
- # explicitly specified with the :long feature.
48
+ # explicitly specified with the :long feature. Finally, if :v2 is
49
+ # true the rule is always considered long.
40
50
  def initialize(value, features=nil)
41
51
  @value = value || ''
42
52
  features = DEFAULT_RULE_FEATURES.merge(features || {})
43
53
  @tag = features[:tag]
44
54
  @id = features[:id]
55
+ # only accept boolean values
56
+ _v2 = features[:v2]
57
+ @v2 = (_v2 == !!_v2) ? _v2 : false
45
58
  # check if long is a boolean
46
59
  _long = features[:long]
47
- @long = _long == !!_long ? _long : @value.size > MAX_STD_RULE_VALUE_LENGTH
60
+ # v2 rules are always long
61
+ @long = (@v2 ? true : (_long == !!_long ? _long : @value.size > MAX_STD_RULE_VALUE_LENGTH))
48
62
  @error = nil
49
63
  end
50
64
 
@@ -53,19 +67,39 @@ module PowerTrack
53
67
  @long
54
68
  end
55
69
 
70
+ # Returns true if the rule is v2.
71
+ def v2?
72
+ @v2
73
+ end
74
+
56
75
  # Returns true if the rule is valid, false otherwise. The validation error
57
76
  # can be through the error method.
58
77
  def valid?
59
78
  # reset error
60
79
  @error = nil
61
80
 
62
- [ :too_long_value?,
63
- :too_many_positive_terms?,
64
- :too_many_negative_terms?,
81
+ validation_rules = [
82
+ :too_long_value?,
65
83
  :contains_empty_source?,
66
84
  :contains_negated_or?,
67
- :too_long_tag? ].each do |validator|
85
+ :too_long_tag?
86
+ ]
87
+
88
+ if @v2
89
+ validation_rules += [
90
+ :contains_explicit_and?,
91
+ :contains_lowercase_or?,
92
+ :contains_explicit_not?
93
+ ]
94
+ else
95
+ # no more restriction on the number of positive and negative terms in v2
96
+ validation_rules += [
97
+ :too_many_positive_terms?,
98
+ :too_many_negative_terms?
99
+ ]
100
+ end
68
101
 
102
+ validation_rules.each do |validator|
69
103
  # stop when 1 validator fails
70
104
  if self.send(validator)
71
105
  @error = validator.to_s.gsub(/_/, ' ').gsub(/\?/, '').capitalize
@@ -129,6 +163,21 @@ module PowerTrack
129
163
  !@value[/\-\w+ OR/].nil? || !@value[/OR \-\w+/].nil?
130
164
  end
131
165
 
166
+ # Does the rule value contain a forbidden AND ?
167
+ def contains_explicit_and?
168
+ !@value[/ AND /].nil?
169
+ end
170
+
171
+ # Does the rule value contain a forbidden lowercase or ?
172
+ def contains_lowercase_or?
173
+ !@value[/ or /].nil?
174
+ end
175
+
176
+ # Does the rule value contain a forbidden NOT ?
177
+ def contains_explicit_not?
178
+ !@value[/(^| )NOT /].nil?
179
+ end
180
+
132
181
  # Does the rule value contain too many positive terms ?
133
182
  def too_many_positive_terms?
134
183
  return false if long?
@@ -21,6 +21,7 @@ module PowerTrack
21
21
  # Removes the specified rules from the stream.
22
22
  #
23
23
  # <tt>DELETE /rules</tt>
24
+ # <tt>POST /rules?method=_delete</tt>
24
25
  #
25
26
  # See http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
26
27
  def delete_rules(*rules)
@@ -30,6 +31,7 @@ module PowerTrack
30
31
  # Removes the specified rule from the stream.
31
32
  #
32
33
  # <tt>DELETE /rules</tt>
34
+ # <tt>POST /rules?method=_delete</tt>
33
35
  #
34
36
  # See http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
35
37
  def delete_rule(rule)
@@ -21,8 +21,8 @@ module PowerTrack
21
21
  FEATURE_URL_FORMAT = {
22
22
  # [ hostname, account, source, mode, label, feature ]
23
23
  v1: "https://%s.gnip.com/accounts/%s/publishers/%s/%s/track/%s%s.json".freeze,
24
- # [ hostname, feature, account, source, label, sub-feature ]
25
- v2: "https://gnip-%s.twitter.com/%s/powertrack/accounts/%s/publishers/%s/%s%s.json".freeze
24
+ # [ hostname, domain, feature, stream type, account, source, label, sub-feature ]
25
+ v2: "https://gnip-%s.%s.com/%s/%s/accounts/%s/publishers/%s/%s%s.json".freeze
26
26
  }.freeze
27
27
 
28
28
  # The default timeout on a connection to PowerTrack. Can be overriden per call.
@@ -68,10 +68,7 @@ module PowerTrack
68
68
  @options = DEFAULT_STREAM_OPTIONS.merge(options || {})
69
69
  @replay = !!@options[:replay]
70
70
  @client_id = @options[:client_id]
71
- @stream_mode = @replay ? 'replay' : 'streams'
72
-
73
- # force v1 if Replay activated
74
- @v2 = !@replay && !!@options[:v2]
71
+ @v2 = !!@options[:v2]
75
72
  end
76
73
 
77
74
  # Adds many rules to your PowerTrack stream’s ruleset.
@@ -181,22 +178,30 @@ module PowerTrack
181
178
  def feature_url(hostname, feature=nil, sub_feature=nil)
182
179
  _url = nil
183
180
  if @v2
184
- feature ||= hostname
181
+ feature ||= @replay ? 'replay' : hostname
185
182
  sub_feature = sub_feature ? "/#{sub_feature}" : ''
183
+ stream_type = (feature == 'rules' && @replay ? 'powertrack-replay' : 'powertrack')
184
+ # replay streaming is on gnip.com while replay rules are on twitter.com...
185
+ domain = (feature == 'replay' && @replay ? 'gnip' : 'twitter')
186
+
186
187
  _url = FEATURE_URL_FORMAT[:v2] %
187
188
  [ hostname,
189
+ domain,
188
190
  feature,
191
+ stream_type,
189
192
  @account_name,
190
193
  @data_source,
191
194
  @label,
192
195
  sub_feature ]
193
196
  else
194
197
  feature = feature ? "/#{feature}" : ''
198
+ mode = @replay ? 'replay' : 'streams'
199
+
195
200
  _url = FEATURE_URL_FORMAT[:v1] %
196
201
  [ hostname,
197
202
  @account_name,
198
203
  @data_source,
199
- @stream_mode,
204
+ mode,
200
205
  @label,
201
206
  feature ]
202
207
 
@@ -1,3 +1,3 @@
1
1
  module PowerTrack
2
- VERSION = '1.2.0'.freeze
2
+ VERSION = '1.3.0'.freeze
3
3
  end
@@ -36,7 +36,7 @@ class Minitest::Test
36
36
  powertrack_config[:password],
37
37
  powertrack_config[:account_name],
38
38
  powertrack_config[:data_source],
39
- replay ? 'prod' : (v2 ? 'prod2' : powertrack_config[:stream_label]),
39
+ replay ? 'prod' : powertrack_config[:stream_label],
40
40
  replay: replay,
41
41
  v2: v2)
42
42
  end
@@ -12,10 +12,14 @@ class TestManageRules < Minitest::Test
12
12
  add_then_delete_a_single_rule(true, false)
13
13
  end
14
14
 
15
- def test_add_then_delete_a_single_rule_in_replay_mode
15
+ def test_add_then_delete_a_single_rule_in_replay_mode_v1
16
16
  add_then_delete_a_single_rule(false, true)
17
17
  end
18
18
 
19
+ def test_add_then_delete_a_single_rule_in_replay_mode_v2
20
+ add_then_delete_a_single_rule(true, true)
21
+ end
22
+
19
23
  def add_then_delete_a_single_rule(v2, replay)
20
24
  stream = new_stream(v2, replay)
21
25
 
data/test/test_rule.rb CHANGED
@@ -12,12 +12,20 @@ class TestRule < Minitest::Test
12
12
  assert rule.valid?
13
13
  assert_nil rule.error
14
14
 
15
- rule = PowerTrack::Rule.new('pepsi', tag: 'soda', long: true)
16
- assert_equal 'pepsi', rule.value
17
- assert_equal 'soda', rule.tag
18
- assert rule.long?
19
- assert rule.valid?
20
- assert_nil rule.error
15
+ long_rule = PowerTrack::Rule.new('pepsi', tag: 'soda', long: true)
16
+ assert_equal 'pepsi', long_rule.value
17
+ assert_equal 'soda', long_rule.tag
18
+ assert long_rule.long?
19
+ assert long_rule.valid?
20
+ assert_nil long_rule.error
21
+
22
+ v2_rule = PowerTrack::Rule.new('dr pepper', tag: 'soda', v2: true)
23
+ assert v2_rule.v2?
24
+ assert_equal 'dr pepper', v2_rule.value
25
+ assert_equal 'soda', v2_rule.tag
26
+ assert v2_rule.long?
27
+ assert v2_rule.valid?
28
+ assert_nil v2_rule.error
21
29
  end
22
30
 
23
31
  def test_too_long_tag
@@ -34,22 +42,40 @@ class TestRule < Minitest::Test
34
42
 
35
43
  def test_too_long_value
36
44
  long_val = 'a' * PowerTrack::Rule::MAX_STD_RULE_VALUE_LENGTH
45
+ # v1
37
46
  rule = PowerTrack::Rule.new(long_val)
38
47
  assert rule.valid?
39
48
 
49
+ # v2
50
+ v2_rule = PowerTrack::Rule.new(long_val, v2: true)
51
+ assert v2_rule.v2?
52
+ assert v2_rule.valid?
53
+ assert_nil v2_rule.error
54
+
40
55
  long_val = 'c' * PowerTrack::Rule::MAX_LONG_RULE_VALUE_LENGTH
56
+ # v1
41
57
  rule = long_val.to_pwtk_rule(long: false)
42
-
43
58
  assert !rule.valid?
44
59
  assert_match /too long value/i, rule.error
45
60
 
46
61
  assert long_val.to_pwtk_rule.valid?
47
62
  assert long_val.to_pwtk_rule(long: true).valid?
48
63
 
64
+ # v2
65
+ assert long_val.to_pwtk_rule(v2: true).valid?
66
+ assert long_val.to_pwtk_rule(long: false, v2: true).valid?
67
+
49
68
  very_long_val = 'rrr' * PowerTrack::Rule::MAX_LONG_RULE_VALUE_LENGTH
69
+ # v1
50
70
  rule = very_long_val.to_pwtk_rule
51
71
  assert !rule.valid?
52
72
  assert_match /too long value/i, rule.error
73
+
74
+ # v2
75
+ v2_rule = very_long_val.to_pwtk_rule(v2: true)
76
+ assert v2_rule.v2?
77
+ assert !v2_rule.valid?
78
+ assert_match /too long value/i, v2_rule.error
53
79
  end
54
80
 
55
81
  def test_too_many_positive_terms
@@ -64,11 +90,25 @@ class TestRule < Minitest::Test
64
90
  assert long_rule.valid?
65
91
  assert_nil long_rule.error
66
92
 
93
+ # v2
94
+ v2_rule = PowerTrack::Rule.new(phrase, v2: true)
95
+ assert v2_rule.v2?
96
+ assert v2_rule.long?
97
+ assert v2_rule.valid?
98
+ assert_nil v2_rule.error
99
+
67
100
  phrase = ([ 'coke' ] * (2 * PowerTrack::Rule::MAX_POSITIVE_TERMS)).join(' ')
101
+ # v1
68
102
  rule = PowerTrack::Rule.new(phrase, long: false)
69
103
  assert !rule.long?
70
104
  assert !rule.valid?
71
105
  assert_match /too many positive terms/i, rule.error
106
+ # v2
107
+ v2_rule = PowerTrack::Rule.new(phrase, v2: true)
108
+ assert v2_rule.v2?
109
+ assert v2_rule.long?
110
+ assert v2_rule.valid?
111
+ assert_nil v2_rule.error
72
112
 
73
113
  long_rule = PowerTrack::Rule.new(phrase, long: true)
74
114
  assert long_rule.long?
@@ -84,6 +124,11 @@ class TestRule < Minitest::Test
84
124
  long_rule = PowerTrack::Rule.new(phrase + " OR from:michel")
85
125
  assert !rule.valid?
86
126
  assert_match /too many positive terms/i, rule.error
127
+
128
+ v2_rule = PowerTrack::Rule.new(phrase + " OR from:michel", v2: true)
129
+ assert v2_rule.v2?
130
+ assert v2_rule.valid?
131
+ assert_nil v2_rule.error
87
132
  end
88
133
 
89
134
  def test_too_many_negative_terms
@@ -98,6 +143,12 @@ class TestRule < Minitest::Test
98
143
  assert long_rule.valid?
99
144
  assert_nil long_rule.error
100
145
 
146
+ v2_rule = PowerTrack::Rule.new(phrase, v2: true)
147
+ assert v2_rule.v2?
148
+ assert v2_rule.long?
149
+ assert v2_rule.valid?
150
+ assert_nil v2_rule.error
151
+
101
152
  phrase = ([ '-pepsi' ] * (2 * PowerTrack::Rule::MAX_POSITIVE_TERMS)).join(' ')
102
153
  rule = PowerTrack::Rule.new(phrase)
103
154
  assert !rule.long?
@@ -108,6 +159,12 @@ class TestRule < Minitest::Test
108
159
  assert long_rule.long?
109
160
  assert long_rule.valid?
110
161
  assert_nil long_rule.error
162
+
163
+ v2_rule = PowerTrack::Rule.new(phrase, v2: true)
164
+ assert v2_rule.v2?
165
+ assert v2_rule.long?
166
+ assert v2_rule.valid?
167
+ assert_nil v2_rule.error
111
168
  end
112
169
 
113
170
  def test_contains_negated_or
@@ -116,6 +173,55 @@ class TestRule < Minitest::Test
116
173
  assert !rule.long?
117
174
  assert !rule.valid?
118
175
  assert_match /contains negated or/i, rule.error
176
+
177
+ v2_rule = PowerTrack::Rule.new(phrase, v2: true)
178
+ assert v2_rule.v2?
179
+ assert v2_rule.long?
180
+ assert !v2_rule.valid?
181
+ assert_match /contains negated or/i, v2_rule.error
182
+ end
183
+
184
+ def test_contains_explicit_and
185
+ phrase = 'coke AND pepsi'
186
+ rule = PowerTrack::Rule.new(phrase)
187
+ assert !rule.long?
188
+ assert rule.valid?
189
+ assert_nil rule.error
190
+
191
+ v2_rule = PowerTrack::Rule.new(phrase, v2: true)
192
+ assert v2_rule.v2?
193
+ assert v2_rule.long?
194
+ assert !v2_rule.valid?
195
+ assert_match /contains explicit and/i, v2_rule.error
196
+ end
197
+
198
+ def test_contains_explicit_not
199
+ [ 'coke NOT pepsi', 'NOT (pepsi OR "dr pepper")' ].each do |phrase|
200
+ rule = PowerTrack::Rule.new(phrase)
201
+ assert !rule.long?
202
+ assert rule.valid?
203
+ assert_nil rule.error
204
+
205
+ v2_rule = PowerTrack::Rule.new(phrase, v2: true)
206
+ assert v2_rule.v2?
207
+ assert v2_rule.long?
208
+ assert !v2_rule.valid?
209
+ assert_match /contains explicit not/i, v2_rule.error
210
+ end
211
+ end
212
+
213
+ def test_contains_lowercase_or
214
+ phrase = 'coke or pepsi'
215
+ rule = PowerTrack::Rule.new(phrase)
216
+ assert !rule.long?
217
+ assert rule.valid?
218
+ assert_nil rule.error
219
+
220
+ v2_rule = PowerTrack::Rule.new(phrase, v2: true)
221
+ assert v2_rule.v2?
222
+ assert v2_rule.long?
223
+ assert !v2_rule.valid?
224
+ assert_match /contains lowercase or/i, v2_rule.error
119
225
  end
120
226
 
121
227
  def test_to_hash_and_json
@@ -16,9 +16,9 @@ class TestTrackStream < Minitest::Test
16
16
  track_simple_stream(false, true)
17
17
  end
18
18
 
19
- # def test_track_replay_stream_v2
20
- # track_simple_stream(true, true)
21
- # end
19
+ def test_track_replay_stream_v2
20
+ track_simple_stream(true, true)
21
+ end
22
22
 
23
23
  def track_simple_stream(v2, replay)
24
24
  stream = new_stream(v2, replay)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: powertrack
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.0
4
+ version: 1.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Laurent Farcy
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2016-08-10 00:00:00.000000000 Z
13
+ date: 2016-09-22 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: bundler