powertrack 1.2.0 → 1.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OTYwNTFiZWQwNTAwYzc2ZWJkYmIyNjA4YWNmNDYxYzMxNWY5ZmYzMA==
+    YTQwZjFmMTc0YTFhNTlmZWIwYzUyYzRlMWMzMjBiNDZkMjBjN2E5MA==
   data.tar.gz: !binary |-
-    MzJhNTVlNzMwMjIwNWEyNzEzZDY3YTEwYjU2ZWM1NmQ3NzE3OWE4ZQ==
+    NzdkY2IwYTk3ZjVlMzdjZmQzYjZkMmM1ZjI5ZGIzMjIyODI2NTllMw==
 SHA512:
   metadata.gz: !binary |-
-    ZTljMmM0NWQzMzhiYzU3NjE1Y2ZmYzIxNTA1OWE5MTczYzc3NTE5MDExYzNi
-    MDJmODA5ZGQ2ODk0OGRlMzgyNDdhYjcwMGM5MjBkNDZhMzBmOWI0NjJjNzMx
-    ZmNmMmU3MmNjMjA2Y2RjZmE1OGVmM2FkZTUwOWI4MTQ0ZDBmNjk=
+    MjhiOTk0NWVhYmRmYzUwNTIyNDI5ZDQ3OGMxYmM2NGI1Y2Y5NGJjZGU3ZTg2
+    MjAyYzFiZjcyYmIyNDE0ODk3MWNiNTIyMWE1Mzk1NWI1MDc4N2Y5YjE2YmJh
+    OTMxYjM1NjFjZmIxODczMTRmNzFjZWRhNGI0ODZkYjRlMWE1MGE=
   data.tar.gz: !binary |-
-    ODJlYjQ1OTU5NTY0YjdhZTczMTY0OWExZmFlNTk1OGI0YzBiY2ZiY2Y4Yzcy
-    NzQ4N2E0MGNmZDA4MTFmMzE0NzQyNDU0MGM5NTczNWMyZGEwMDQ1YzgyMjZk
-    MzcxYTUzNjcxZjZhY2YyZTc2YjcxYTBkZjZiNjQyNjEyZDMzZWM=
+    NWJhNjA3MDIyMDkyMDFmZTViNzAxNmNmZTRiOThkYzQ0YWI3YmUwYWYyZTE2
+    OGMzZGM1NWY4NjJlNTYyOWJmMmUxNTI5NjJjYmZjZDk1YmI0NTc4NWZlYTdl
+    NzEzODgxMzYwMzNkYzlkZWQyZWQ2YTg0MWRmNmM3MjQ3YjI5NTc=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    powertrack (1.2.0)
+    powertrack (1.3.0)
       em-http-request (~> 1.1)
       eventmachine (~> 1.0)
       exponential-backoff (~> 0.0.2)

data/History.txt

@@ -1,3 +1,9 @@
+v1.3.0
+------
+
+  * Support Replay v2
+  * Support specificities of rules v2 in PowerTrack::Rule class
+
 v1.2.0
 ------
 

data/README.md

@@ -147,9 +147,14 @@ o The Backfill feature is configured by the ```:backfill_minutes``` option passe
   option to the ```PowerTrack::Stream``` initializer (which is simply ignored
   when v2 is turned on). The new option specifies a number of minutes of backfill
   data to receive.
-o The Replay feature still uses v1 even if you explicitly turn v2 on. Support
-  for [Replay v2](http://support.gnip.com/apis/replay2.0/api_reference.html) is
-  planned but not scheduled yet.
+o A v2 ```PowerTrack::Rule``` instance (initialized by passing the ```v2: true```
+  feature to the constructor) has a few specificities described in
+  [Migrating PowerTrack Rules from Version 1.0 to 2.0](http://support.gnip.com/articles/migrating-powertrack-rules.html).
+
+  In particular,
+  o it is always long (accepting up to 2048 characters),
+  o it has no limits on the number of positive or negative terms used,
+  o it forbids the usage of *AND*, *or* and *NOT* logical phrases.
 
 Finally, PowerTrack v2 has a new endpoint for rule validation that is not
 supported by this library yet.

data/TODO.md

@@ -99,5 +99,5 @@ and [PowerTrack API v2](http://support.gnip.com/apis/powertrack2.0/index.html).
     stream was disconnected more than 5 minutes (tweets were probably lost)
 * _[DONE]_ Use HTTP POST verb (instead of DELETE) for rule deletions
 * _[DONE]_ Fallback to v1 when Replay mode wants to use v2. Emit a warning.
-* Support Replay v2
+* _[DONE]_ Support Replay v2
   [Replay API 2.0 Reference](http://support.gnip.com/apis/replay2.0/api_reference.html)

data/lib/powertrack/errors.rb

@@ -73,6 +73,14 @@ module PowerTrack
     end
   end
 
+  # Invalid Endpoint. This occurs when your client tries to connect to an
+  # endpoint URL that does not exist.
+  class InvalidEndpoint < PredefinedStatusPowerTrackError
+    def initialize(message, body)
+      super(404, message, body)
+    end
+  end
+
   # Generally, this occurs where your client fails to properly include the
   # headers to accept gzip encoding from the stream, but can occur in other
   # circumstances as well.

data/lib/powertrack/rules/rule.rb

@@ -19,6 +19,13 @@ module PowerTrack
     # The maximum number of negative terms in a single rule value
     MAX_NEGATIVE_TERMS = 50
 
+    # The maximum size of the HTTP body accepted by PowerTrack /rules calls (in bytes)
+    # 1 MB for v1, 5MB for v2
+    MAX_RULES_BODY_SIZE = {
+      v1: 1024**2,
+      v2: 5*1024**2
+    }
+
     # The default rule features
     DEFAULT_RULE_FEATURES = {
       # no id by default
@@ -26,25 +33,32 @@ module PowerTrack
       # no tag by default
       tag: nil,
       # long determined by value length
-      long: nil
+      long: nil,
+      # v1 by default
+      v2: false
     }.freeze
 
     attr_reader :value, :id, :tag, :error
 
     # Builds a new rule based on a value and some optional features
-    # (:id, :tag, :long).
+    # (:id, :tag, :long, :v2).
     #
     # By default, the constructor assesses if it's a long rule or not
     # based on the length of the value. But the 'long' feature can be
-    # explicitly specified with the :long feature.
+    # explicitly specified with the :long feature. Finally, if :v2 is
+    # true the rule is always considered long.
     def initialize(value, features=nil)
       @value = value || ''
       features = DEFAULT_RULE_FEATURES.merge(features || {})
       @tag = features[:tag]
       @id = features[:id]
+      # only accept boolean values
+      _v2 = features[:v2]
+      @v2 = (_v2 == !!_v2) ? _v2 : false
       # check if long is a boolean
       _long = features[:long]
-      @long = _long == !!_long ? _long : @value.size > MAX_STD_RULE_VALUE_LENGTH
+      # v2 rules are always long
+      @long = (@v2 ? true : (_long == !!_long ? _long : @value.size > MAX_STD_RULE_VALUE_LENGTH))
       @error = nil
     end
 
@@ -53,19 +67,39 @@ module PowerTrack
       @long
     end
 
+    # Returns true if the rule is v2.
+    def v2?
+      @v2
+    end
+
     # Returns true if the rule is valid, false otherwise. The validation error
     # can be through the error method.
     def valid?
       # reset error
       @error = nil
 
-      [ :too_long_value?,
-        :too_many_positive_terms?,
-        :too_many_negative_terms?,
+      validation_rules = [
+        :too_long_value?,
         :contains_empty_source?,
         :contains_negated_or?,
-        :too_long_tag? ].each do |validator|
+        :too_long_tag?
+      ]
+
+      if @v2
+        validation_rules += [
+          :contains_explicit_and?,
+          :contains_lowercase_or?,
+          :contains_explicit_not?
+        ]
+      else
+        # no more restriction on the number of positive and negative terms in v2
+        validation_rules += [
+          :too_many_positive_terms?,
+          :too_many_negative_terms?
+        ]
+      end
 
+      validation_rules.each do |validator|
         # stop when 1 validator fails
         if self.send(validator)
           @error = validator.to_s.gsub(/_/, ' ').gsub(/\?/, '').capitalize
@@ -129,6 +163,21 @@ module PowerTrack
       !@value[/\-\w+ OR/].nil? || !@value[/OR \-\w+/].nil?
     end
 
+    # Does the rule value contain a forbidden AND ?
+    def contains_explicit_and?
+      !@value[/ AND /].nil?
+    end
+
+    # Does the rule value contain a forbidden lowercase or ?
+    def contains_lowercase_or?
+      !@value[/ or /].nil?
+    end
+
+    # Does the rule value contain a forbidden NOT ?
+    def contains_explicit_not?
+      !@value[/(^| )NOT /].nil?
+    end
+
     # Does the rule value contain too many positive terms ?
     def too_many_positive_terms?
       return false if long?

data/lib/powertrack/streaming/api.rb

@@ -21,6 +21,7 @@ module PowerTrack
     # Removes the specified rules from the stream.
     #
     # <tt>DELETE /rules</tt>
+    # <tt>POST /rules?method=_delete</tt>
     #
     # See http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
     def delete_rules(*rules)
@@ -30,6 +31,7 @@ module PowerTrack
     # Removes the specified rule from the stream.
     #
     # <tt>DELETE /rules</tt>
+    # <tt>POST /rules?method=_delete</tt>
     #
     # See http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
     def delete_rule(rule)

data/lib/powertrack/streaming/stream.rb

@@ -21,8 +21,8 @@ module PowerTrack
     FEATURE_URL_FORMAT = {
       # [ hostname, account, source, mode, label, feature ]
       v1: "https://%s.gnip.com/accounts/%s/publishers/%s/%s/track/%s%s.json".freeze,
-      # [ hostname, feature, account, source, label, sub-feature ]
-      v2: "https://gnip-%s.twitter.com/%s/powertrack/accounts/%s/publishers/%s/%s%s.json".freeze
+      # [ hostname, domain, feature, stream type, account, source, label, sub-feature ]
+      v2: "https://gnip-%s.%s.com/%s/%s/accounts/%s/publishers/%s/%s%s.json".freeze
     }.freeze
 
     # The default timeout on a connection to PowerTrack. Can be overriden per call.
@@ -68,10 +68,7 @@ module PowerTrack
       @options = DEFAULT_STREAM_OPTIONS.merge(options || {})
       @replay = !!@options[:replay]
       @client_id = @options[:client_id]
-      @stream_mode = @replay ? 'replay' : 'streams'
-
-      # force v1 if Replay activated
-      @v2 = !@replay && !!@options[:v2]
+      @v2 = !!@options[:v2]
     end
 
     # Adds many rules to your PowerTrack stream’s ruleset.
@@ -181,22 +178,30 @@ module PowerTrack
     def feature_url(hostname, feature=nil, sub_feature=nil)
       _url = nil
       if @v2
-        feature ||= hostname
+        feature ||= @replay ? 'replay' : hostname
         sub_feature = sub_feature ? "/#{sub_feature}" : ''
+        stream_type = (feature == 'rules' && @replay ? 'powertrack-replay' : 'powertrack')
+        # replay streaming is on gnip.com while replay rules are on twitter.com...
+        domain = (feature == 'replay' && @replay  ? 'gnip' : 'twitter')
+
         _url = FEATURE_URL_FORMAT[:v2] %
                 [ hostname,
+                  domain,
                   feature,
+                  stream_type,
                   @account_name,
                   @data_source,
                   @label,
                   sub_feature ]
       else
         feature = feature ? "/#{feature}" : ''
+        mode = @replay ? 'replay' : 'streams'
+
         _url = FEATURE_URL_FORMAT[:v1] %
                 [ hostname,
                   @account_name,
                   @data_source,
-                  @stream_mode,
+                  mode,
                   @label,
                   feature ]
 

data/lib/powertrack/version.rb

@@ -1,3 +1,3 @@
 module PowerTrack
-  VERSION = '1.2.0'.freeze
+  VERSION = '1.3.0'.freeze
 end

data/test/minitest_helper.rb

@@ -36,7 +36,7 @@ class Minitest::Test
       powertrack_config[:password],
       powertrack_config[:account_name],
       powertrack_config[:data_source],
-      replay ? 'prod' : (v2 ? 'prod2' : powertrack_config[:stream_label]),
+      replay ? 'prod' : powertrack_config[:stream_label],
       replay: replay,
       v2: v2)
   end

data/test/test_manage_rules.rb

@@ -12,10 +12,14 @@ class TestManageRules < Minitest::Test
     add_then_delete_a_single_rule(true, false)
   end
 
-  def test_add_then_delete_a_single_rule_in_replay_mode
+  def test_add_then_delete_a_single_rule_in_replay_mode_v1
     add_then_delete_a_single_rule(false, true)
   end
 
+  def test_add_then_delete_a_single_rule_in_replay_mode_v2
+    add_then_delete_a_single_rule(true, true)
+  end
+
   def add_then_delete_a_single_rule(v2, replay)
     stream = new_stream(v2, replay)
 

data/test/test_rule.rb

@@ -12,12 +12,20 @@ class TestRule < Minitest::Test
     assert rule.valid?
     assert_nil rule.error
 
-    rule = PowerTrack::Rule.new('pepsi', tag: 'soda', long: true)
-    assert_equal 'pepsi', rule.value
-    assert_equal 'soda', rule.tag
-    assert rule.long?
-    assert rule.valid?
-    assert_nil rule.error
+    long_rule = PowerTrack::Rule.new('pepsi', tag: 'soda', long: true)
+    assert_equal 'pepsi', long_rule.value
+    assert_equal 'soda', long_rule.tag
+    assert long_rule.long?
+    assert long_rule.valid?
+    assert_nil long_rule.error
+
+    v2_rule = PowerTrack::Rule.new('dr pepper', tag: 'soda', v2: true)
+    assert v2_rule.v2?
+    assert_equal 'dr pepper', v2_rule.value
+    assert_equal 'soda', v2_rule.tag
+    assert v2_rule.long?
+    assert v2_rule.valid?
+    assert_nil v2_rule.error
   end
 
   def test_too_long_tag
@@ -34,22 +42,40 @@ class TestRule < Minitest::Test
 
   def test_too_long_value
     long_val = 'a' * PowerTrack::Rule::MAX_STD_RULE_VALUE_LENGTH
+    # v1
     rule = PowerTrack::Rule.new(long_val)
     assert rule.valid?
 
+    # v2
+    v2_rule = PowerTrack::Rule.new(long_val, v2: true)
+    assert v2_rule.v2?
+    assert v2_rule.valid?
+    assert_nil v2_rule.error
+
     long_val = 'c' * PowerTrack::Rule::MAX_LONG_RULE_VALUE_LENGTH
+    # v1
     rule = long_val.to_pwtk_rule(long: false)
-
     assert !rule.valid?
     assert_match /too long value/i, rule.error
 
     assert long_val.to_pwtk_rule.valid?
     assert long_val.to_pwtk_rule(long: true).valid?
 
+    # v2
+    assert long_val.to_pwtk_rule(v2: true).valid?
+    assert long_val.to_pwtk_rule(long: false, v2: true).valid?
+
     very_long_val = 'rrr' * PowerTrack::Rule::MAX_LONG_RULE_VALUE_LENGTH
+    # v1
     rule = very_long_val.to_pwtk_rule
     assert !rule.valid?
     assert_match /too long value/i, rule.error
+
+    # v2
+    v2_rule = very_long_val.to_pwtk_rule(v2: true)
+    assert v2_rule.v2?
+    assert !v2_rule.valid?
+    assert_match /too long value/i, v2_rule.error
   end
 
   def test_too_many_positive_terms
@@ -64,11 +90,25 @@ class TestRule < Minitest::Test
     assert long_rule.valid?
     assert_nil long_rule.error
 
+    # v2
+    v2_rule = PowerTrack::Rule.new(phrase, v2: true)
+    assert v2_rule.v2?
+    assert v2_rule.long?
+    assert v2_rule.valid?
+    assert_nil v2_rule.error
+
     phrase = ([ 'coke' ] * (2 * PowerTrack::Rule::MAX_POSITIVE_TERMS)).join(' ')
+    # v1
     rule = PowerTrack::Rule.new(phrase, long: false)
     assert !rule.long?
     assert !rule.valid?
     assert_match /too many positive terms/i, rule.error
+    # v2
+    v2_rule = PowerTrack::Rule.new(phrase, v2: true)
+    assert v2_rule.v2?
+    assert v2_rule.long?
+    assert v2_rule.valid?
+    assert_nil v2_rule.error
 
     long_rule = PowerTrack::Rule.new(phrase, long: true)
     assert long_rule.long?
@@ -84,6 +124,11 @@ class TestRule < Minitest::Test
     long_rule = PowerTrack::Rule.new(phrase + " OR from:michel")
     assert !rule.valid?
     assert_match /too many positive terms/i, rule.error
+
+    v2_rule = PowerTrack::Rule.new(phrase + " OR from:michel", v2: true)
+    assert v2_rule.v2?
+    assert v2_rule.valid?
+    assert_nil v2_rule.error
   end
 
   def test_too_many_negative_terms
@@ -98,6 +143,12 @@ class TestRule < Minitest::Test
     assert long_rule.valid?
     assert_nil long_rule.error
 
+    v2_rule = PowerTrack::Rule.new(phrase, v2: true)
+    assert v2_rule.v2?
+    assert v2_rule.long?
+    assert v2_rule.valid?
+    assert_nil v2_rule.error
+
     phrase = ([ '-pepsi' ] * (2 * PowerTrack::Rule::MAX_POSITIVE_TERMS)).join(' ')
     rule = PowerTrack::Rule.new(phrase)
     assert !rule.long?
@@ -108,6 +159,12 @@ class TestRule < Minitest::Test
     assert long_rule.long?
     assert long_rule.valid?
     assert_nil long_rule.error
+
+    v2_rule = PowerTrack::Rule.new(phrase, v2: true)
+    assert v2_rule.v2?
+    assert v2_rule.long?
+    assert v2_rule.valid?
+    assert_nil v2_rule.error
   end
 
   def test_contains_negated_or
@@ -116,6 +173,55 @@ class TestRule < Minitest::Test
     assert !rule.long?
     assert !rule.valid?
     assert_match /contains negated or/i, rule.error
+
+    v2_rule = PowerTrack::Rule.new(phrase, v2: true)
+    assert v2_rule.v2?
+    assert v2_rule.long?
+    assert !v2_rule.valid?
+    assert_match /contains negated or/i, v2_rule.error
+  end
+
+  def test_contains_explicit_and
+    phrase = 'coke AND pepsi'
+    rule = PowerTrack::Rule.new(phrase)
+    assert !rule.long?
+    assert rule.valid?
+    assert_nil rule.error
+
+    v2_rule = PowerTrack::Rule.new(phrase, v2: true)
+    assert v2_rule.v2?
+    assert v2_rule.long?
+    assert !v2_rule.valid?
+    assert_match /contains explicit and/i, v2_rule.error
+  end
+
+  def test_contains_explicit_not
+    [ 'coke NOT pepsi', 'NOT (pepsi OR "dr pepper")' ].each do |phrase|
+      rule = PowerTrack::Rule.new(phrase)
+      assert !rule.long?
+      assert rule.valid?
+      assert_nil rule.error
+
+      v2_rule = PowerTrack::Rule.new(phrase, v2: true)
+      assert v2_rule.v2?
+      assert v2_rule.long?
+      assert !v2_rule.valid?
+      assert_match /contains explicit not/i, v2_rule.error
+    end
+  end
+
+  def test_contains_lowercase_or
+    phrase = 'coke or pepsi'
+    rule = PowerTrack::Rule.new(phrase)
+    assert !rule.long?
+    assert rule.valid?
+    assert_nil rule.error
+
+    v2_rule = PowerTrack::Rule.new(phrase, v2: true)
+    assert v2_rule.v2?
+    assert v2_rule.long?
+    assert !v2_rule.valid?
+    assert_match /contains lowercase or/i, v2_rule.error
   end
 
   def test_to_hash_and_json

data/test/test_track_stream.rb

@@ -16,9 +16,9 @@ class TestTrackStream < Minitest::Test
     track_simple_stream(false, true)
   end
 
-#  def test_track_replay_stream_v2
-#    track_simple_stream(true, true)
-#  end
+  def test_track_replay_stream_v2
+    track_simple_stream(true, true)
+  end
 
   def track_simple_stream(v2, replay)
     stream = new_stream(v2, replay)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: powertrack
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Laurent Farcy
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-10 00:00:00.000000000 Z
+date: 2016-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler