powertrack 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OGNhMTRiZDk2MDJlN2VjNGE3Zjk1MmVmMTVjNjJiODQxYThkNDdkNg==
+  data.tar.gz: !binary |-
+    MjVjZTk3NjZhOWMxNTFkMTE4NjQ0MmM5Y2JjOTY0ZDE1MzZkYTUwYg==
+SHA512:
+  metadata.gz: !binary |-
+    ODk2NGZlOWY5ZTM3N2MzYWZlYzQwNWUzNjhjNmYyYjBhMzUxODMxYzhiNjMz
+    YzNhOTE3NWQ5ZTE2N2M0YTZmOTNkY2M4NmRkMWJjYjM0Yjc2YzQ0OWIzNGI5
+    MGI4ZTg0YzdmMDUzZTYwYjc3MDAwN2FkZmY3ZWQyN2ZjYzhhODc=
+  data.tar.gz: !binary |-
+    ZjIzZGY3ZWZkNTA0MTJhZGQwZjNjN2FiNTFiZTQ3MmI2NTIzNWEyZGVkNzFj
+    YzczZWY5NTU0MDczNjQyZTA5YzMzMzE1ZmViODkyMDk2M2RjNGU4MTZhMGE2
+    M2VkODJjNzUwY2M3ZTBmOWUwMDM4NTZkYjE0NjFhZGI1YTFhMDI=

data/.gitignore

@@ -0,0 +1,41 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# ignore Eclipse config
+.project
+
+# ignore tests-related config file
+test/powertrack.yml

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+group :test do
+  gem 'yajl-ruby', '~> 1.0', require: 'yajl'
+end
+
+# Specify your gem's dependencies in powertrack-rb.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,43 @@
+PATH
+  remote: .
+  specs:
+    powertrack (1.0.0)
+      em-http-request (~> 1.1)
+      eventmachine (~> 1.0)
+      exponential-backoff (~> 0.0.2)
+      multi_json (~> 1.11)
+      void_logger (~> 0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    cookiejar (0.3.2)
+    em-http-request (1.1.2)
+      addressable (>= 2.3.4)
+      cookiejar
+      em-socksify (>= 0.3)
+      eventmachine (>= 1.0.3)
+      http_parser.rb (>= 0.6.0)
+    em-socksify (0.3.0)
+      eventmachine (>= 1.0.0.beta.4)
+    eventmachine (1.0.7)
+    exponential-backoff (0.0.2)
+    http_parser.rb (0.6.0)
+    minitest (5.7.0)
+    multi_json (1.11.2)
+    rake (10.4.2)
+    ruby-prof (0.15.8)
+    void_logger (0.1)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.7)
+  minitest (~> 5.5)
+  powertrack!
+  rake (~> 10.3)
+  ruby-prof (~> 0.15)
+  yajl-ruby (~> 1.0)

data/LICENSE.md

@@ -0,0 +1,25 @@
+## The MIT License (MIT)
+
+- Copyright (c) 2015 eCairn Inc.
+- Copyright (c) 2012-2015 Eric Wendelin
+- Copyright (c) 2011-2015 Ryan Weald / Sharethrough
+
+```
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```

data/README.md

@@ -0,0 +1,117 @@
+# powertrack-rb
+A Ruby gem for building GNIP PowerTrack streaming clients.
+
+## How to use it ?
+
+1. Create a PowerTrack stream based on your credentials
+
+  ```ruby
+  require 'powertrack'
+
+  stream = PowerTrack::Stream.new(
+    powertrack_config[:username],
+    powertrack_config[:password],
+    powertrack_config[:account_name],
+    powertrack_config[:data_source],  # often 'twitter'
+    powertrack_config[:stream_label]) # often 'prod'
+  ```
+
+2. Add a few rules to the stream
+
+  ```ruby
+  rule = PowerTrack::Rule.new('coke')
+  if rule.valid?
+    stream.add_rule(rule)
+    # double-check that the rule was actually added
+    raise 'Fail to add a rule' unless stream.list_rules.include?(rule)
+  end
+  ```
+
+3. Get the activities out of the stream
+
+  ```ruby
+  received, heartbeats = 0, 0
+  activities = []
+
+  ## defining callbacks on messages received
+  # callback triggered for each message received
+  on_message = lambda { |message| received += 1 }
+  # callback triggered for each heartbeat received
+  on_heartbeat = lambda { heartbeats += 1 }
+  # callback triggered for each activity received
+  on_activity = lambda { |activity| activities += activity }
+
+  ## defining the block that will command the stop of the tracking
+  closed = false
+  close_now = lambda { closed }
+
+  delay = 60
+  Thread.new do
+    $stderr.puts "Time-bomb thread running for #{delay} seconds..."
+    sleep delay
+    $stderr.puts "Time to shut down !"
+    closed = true
+  end
+
+  started_at = Time.now
+  res = stream.track(on_message: on_message,
+                     on_heartbeat: on_heartbeat,
+                     on_activity: on_activity,
+                     close_now: close_now)
+
+  puts "After #{delay} seconds tracking '#{rule.value}':"
+  puts "  o #{received} messages received"
+  puts "  o #{heartbeats} heartbeats received"
+  puts "  o #{activities.size} activities captured"
+  ```
+
+Please note that each message callback must be thread-safe since it can be called
+multiple times simultaneously.
+
+## Tracking response format
+
+By default, messages received are passed to callbacks as plain Ruby objects. Enable
+the ```raw``` option to get raw JSON-formatted string and make the parsing by
+yourself.
+
+## Stop tracking
+
+The tracker calls the ```close_now``` block each second and stops whenever the call
+returns true. The stop procedure includes an additional timeframe where the tracker
+waits for each pending message to be completely processed.
+
+It's up to the developer's responsibility to complete message processing as soon as
+possible. After 10 seconds (by default), the stop will be forced and a few messages
+already received but not processed yet may be lost.
+
+The ```:stop_timeout``` may be fine-tune when passing options to the tracker.
+
+## Disconnections and Retries
+
+As highly recommended by GNIP, the PowerTrack::Stream client manages an exponential
+backoff retry mechanism when a disconnection happens. The reconnections can be
+fine-tuned through the ```max_retries``` and ```backoff``` options passed to the
+```track``` call.
+
+## Backfill
+
+Backfill is a feature provided by GNIP to avoid losing activities when being
+disconnected. It automatically resends the messages sent on the stream for the
+last 5 minutes when reconnecting.
+
+Provide a (numerical) client id as the last (but optional) argument of the
+PowerTrack::Stream constructor to enable this feature.
+
+## Errors
+
+All the errors that come from PowerTrack are defined through an ad-hoc exception
+class hierarchy. See ```lib/powertrack/errors.rb```.
+
+## Credits
+
+The ```powertrack``` gem heavily relies on *EventMachine* and the *em-http-request*
+companion gem. It also got inspiration from a few other gems
+
+* The [gnip-rule](https://github.com/singlebrook/gnip-rule) gem
+* The [gnip-stream](https://github.com/rweald/gnip-stream) gem
+* The [exponential-backoff](https://github.com/pawelpacana/exponential-backoff) gem

data/Rakefile

@@ -0,0 +1,16 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+end
+
+require 'rdoc/task'
+namespace :doc do
+  RDoc::Task.new do |rd|
+    rd.rdoc_dir = 'doc'
+    rd.rdoc_files.include('lib/**/*.rb')
+  end
+end
+
+task :default => :test

data/TODO.md

@@ -0,0 +1,82 @@
+## General
+
+* _[DONE]_ Rely upon MultiJson for JSON encoding and decoding
+* Support thread-safe streams
+
+  A single stream used in several threads to perform several simultaneous actions
+  like consuming the stream while changing its rules.
+
+  It's currently impossible due to EventMachine. A transition to Celluloid::IO and
+  http.rb is required to be thread-friendly.
+
+## Rules
+
+* _[DONE]_ Check rule size
+* _[DONE]_ Add 1 or more rules to a stream
+* _[DONE]_ Delete some rules from the stream
+* _[DONE]_ Get all existing rules for a stream
+* _[DONE]_ Rules equality and usage in hash as keys
+* Rule encoding (UTF-8 enforcement ?)
+* Double check a rule supports all the syntactical and semantic restrictions
+  as defined by GNIP [PowerTrack Rules](http://support.gnip.com/apis/powertrack/rules.html#Restrictions)
+  reference documentation
+* Support evolution of rules in terms of addition, removal and updates.
+
+## Real-time PowerTrack
+
+* _[DONE]_ Manage persitent connection to a data stream. See
+  [Powertrack API reference](http://support.gnip.com/apis/powertrack/api_reference.html)
+* _[DONE]_ [Consume streaming data](http://support.gnip.com/apis/consuming_streaming_data.html)
+* _[DONE]_ Capture heartbeat activities
+* _[DONE]_ Capture system-related activities
+
+## Compliance activities
+
+See [Honoring user intent on Twitter](http://support.gnip.com/articles/honoring-user-intent-on-twitter.html)
+and [Compliance Activities](http://support.gnip.com/sources/twitter/data_format.html#ComplianceActivities).
+
+* _[DROPPED]_ Add a comply method to PowerTrack::API ?
+  The compliance activities are broadcasted on a specific compliance stream.
+* Support the Compliance Firehose stream
+  [Compliance Firehose Reference](http://support.gnip.com/apis/compliance_firehose/api_reference.html)
+
+### Account
+
+* Protect / Unprotect account
+* Delete account
+* Scrub geo
+* Suspend account
+* Withhold account
+
+### Status
+
+* Delete status
+* Withhold status
+
+## Data formats
+
+See [Data format](http://support.gnip.com/sources/twitter/data_format.html)
+
+* _[DONE]_ Support Original output format
+* _[DONE]_ Support Activity Stream output format
+* _[DONE]_ Support raw format
+*
+* _[OUT]_ Manage retweets.
+  See [Identifying and Understanding retweets](http://support.gnip.com/articles/identifying-and-understanding-retweets.html)
+
+## Disconnections
+
+See [Managing disconnections](http://support.gnip.com/articles/disconnections-explained.html)
+
+* _[DONE]_ Reconnect after disconnect. See
+  [Disconnections & Reconnecting](http://support.gnip.com/apis/consuming_streaming_data.html#Disconnections)
+* _[DONE]_ Reconnect using an exponential backoff pattern.
+* _[DONE]_ Support Backfill
+* Support Replay
+* Reconnect when there's a GNIP server issue signaled by the 503 HTTP response status
+
+## Other features
+
+* _[DONE]_ Support test and development streams
+* Support status dashboard
+* Support Historical Powertrack

data/lib/core_ext/class.rb

@@ -0,0 +1,7 @@
+# See http://stackoverflow.com/questions/2393697/look-up-all-descendants-of-a-class-in-ruby
+class Class
+  # Returns the descendants of the class.
+  def descendants
+    ObjectSpace.each_object(::Class).select {|klass| klass < self }
+  end
+end

data/lib/powertrack.rb

@@ -0,0 +1,10 @@
+require 'core_ext/class.rb'
+
+require 'powertrack/version'
+require 'powertrack/errors'
+
+require 'powertrack/rules/rule'
+require 'powertrack/rules/string_extension'
+
+require 'powertrack/streaming/api'
+require 'powertrack/streaming/stream'

data/lib/powertrack/errors.rb

@@ -0,0 +1,110 @@
+module PowerTrack
+  # Base PowerTrack error, capable of wrapping another
+  class BasePowerTrackError < StandardError
+    attr_reader :status, :body
+
+    def initialize(status, msg, body=nil)
+      msg ||= body
+      _status = "#{status}".strip
+      _msg = "#{msg}".strip
+      err = [ _status, _msg ].select { |part| !part.empty? }.join(': ')
+      super(err)
+      @status = status
+      @body = body
+    end
+  end
+
+  # Base class for PowerTrack errors without a precise status
+  class NoStatusPowerTrackError < BasePowerTrackError
+    def initialize(message, body)
+      super(nil, message, body)
+    end
+  end
+
+  # An error which is raised when there is a connection issue with the PowerTrack
+  # endpoint
+  class ConnectionError < NoStatusPowerTrackError
+    def initialize(message)
+      super(message, nil)
+    end
+  end
+
+  # Base class for PowerTrack errors with a precise status
+  class WithStatusPowerTrackError < BasePowerTrackError
+    # Factory method which returns an error instance based on a given status.
+    def self.build(status, message, body)
+      @@status_to_error_class ||= Hash[*self.descendants.map { |desc|
+        [ desc.new(nil, nil).status, desc ] }.flatten ]
+      if @@status_to_error_class.key?(status)
+        @@status_to_error_class[status].new(message, body)
+      else
+        # default to unknown status error
+        UnknownStatusError.new(status, message, body)
+      end
+    end
+  end
+
+  # An exception which is raised when the response received from PowerTrack is
+  # invalid, poorly formatted in most cases.
+  class InvalidResponseError < WithStatusPowerTrackError
+  end
+
+  # An exception which is raised when PowerTrack returns an unknown HTTP status code.
+  class UnknownStatusError < WithStatusPowerTrackError
+  end
+
+  # Base class for errors which match a well-defined HTTP status code as
+  # documented in the PowerTrack API reference.
+  class PredefinedStatusPowerTrackError < WithStatusPowerTrackError
+  end
+
+  # Generally relates to poorly formatted JSON, and includes an "Invalid JSON"
+  # message in the response.
+  class BadRequestError < PredefinedStatusPowerTrackError
+    def initialize(message, body)
+      super(400, message, body)
+    end
+  end
+
+  # HTTP authentication failed due to invalid credentials.
+  class UnauthorizedError < PredefinedStatusPowerTrackError
+    def initialize(message, body)
+      super(401, message, body)
+    end
+  end
+
+  # Generally, this occurs where your client fails to properly include the
+  # headers to accept gzip encoding from the stream, but can occur in other
+  # circumstances as well.
+  #
+  # Will contain a JSON message similar to "This connection requires
+  # compression. To enable compression, send an 'Accept-Encoding: gzip' header
+  # in your request and be ready to uncompress the stream as it is read on
+  # the client end."
+  class NotAcceptableError < PredefinedStatusPowerTrackError
+    def initialize(message, body)
+      super(406, message, body)
+    end
+  end
+
+  class UnprocessableEntityError < PredefinedStatusPowerTrackError
+    def initialize(message, body)
+      super(422, message, body)
+    end
+  end
+
+  # Your app has exceeded the limit on connection requests.
+  class RateLimitedError < PredefinedStatusPowerTrackError
+    def initialize(message, body)
+      super(429, message, body)
+    end
+  end
+
+  # Gnip server issue. If no notice about this issue has been posted on
+  # status.gnip.com, email support@gnip.com.
+  class ServiceUnavailableError < PredefinedStatusPowerTrackError
+    def initialize(message, body)
+      super(503, message, body)
+    end
+  end
+end