datasift 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1fe731a05bf2f78cee04bd0d37a053991ee0b08
-  data.tar.gz: 119a97e10c022290c0a239c0fd192b63a13eca3d
+  metadata.gz: 231dfbf3fbb478090969896e459179651d9aab7d
+  data.tar.gz: dc955abee233f6cb82a5765065796563a89ef268
 SHA512:
-  metadata.gz: 660b017e290d8218413938369e93f5bdf3c784971d954497d6b8d14de9049dca430b4a78f13867acf458fe10085815357d7d7b0a0016ec4a962e2eaf87c0890c
-  data.tar.gz: d99b43dc735b245c684fffde08cdb65ee4fa47c892c91330e6ee67def6a634df015ddeb46c16b6ba7c4ebfcac07c917610c6cc52cb391c3f375f74a74e76dd7c
+  metadata.gz: 5adc6aa0777edfef9835b08e1e133a63805cf10e926ab67fda130cca85ca79509e466d91b1c7eb21affa44f6829dd7fdc1c2807e0ae66666581c44aa532b3161
+  data.tar.gz: 7074591477540122765b3c91bd3dec1dbd1f2d65371277fb59b24a523455e791f44c21d2fa2572fa4abe04ce21f20a52203485bab10957bc349b8d1ee39ac8b3

data/.rubocop.yml

@@ -0,0 +1 @@
+inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -0,0 +1,300 @@
+# This configuration was generated by `rubocop --auto-gen-config`
+# on 2015-08-14 10:29:23 +0100 using RuboCop version 0.32.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Lint/ParenthesesAsGroupedExpression:
+  Enabled: false
+
+# Offense count: 1
+Lint/RescueException:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Lint/SpaceBeforeFirstArg:
+  Enabled: false
+
+# Offense count: 5
+# Cop supports --auto-correct.
+Lint/UnusedBlockArgument:
+  Enabled: false
+
+# Offense count: 2
+Lint/UselessAssignment:
+  Enabled: false
+
+# Offense count: 27
+Metrics/AbcSize:
+  Max: 106
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 108
+
+# Offense count: 13
+Metrics/CyclomaticComplexity:
+  Max: 15
+
+# Offense count: 118
+# Configuration parameters: AllowURI, URISchemes.
+LineLength:
+  Max: 90
+
+# Offense count: 33
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 91
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/ModuleLength:
+  Max: 232
+
+# Offense count: 8
+# Configuration parameters: CountKeywordArgs.
+Metrics/ParameterLists:
+  Max: 8
+
+# Offense count: 8
+Metrics/PerceivedComplexity:
+  Max: 16
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/AlignArray:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle, SupportedLastArgumentHashStyles.
+Style/AlignHash:
+  Enabled: false
+
+# Offense count: 6
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/AlignParameters:
+  Enabled: false
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles, ProceduralMethods, FunctionalMethods, IgnoredMethods.
+Style/BlockDelimiters:
+  Enabled: false
+
+# Offense count: 20
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/BracesAroundHashParameters:
+  Enabled: false
+
+# Offense count: 20
+# Cop supports --auto-correct.
+# Configuration parameters: IndentWhenRelativeTo, SupportedStyles, IndentOneStep.
+Style/CaseIndentation:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/ColonMethodCall:
+  Enabled: false
+
+# Offense count: 7
+# Cop supports --auto-correct.
+Style/CommentIndentation:
+  Enabled: false
+
+# Offense count: 6
+# Cop supports --auto-correct.
+Style/DeprecatedHashMethods:
+  Enabled: false
+
+# Offense count: 14
+Style/Documentation:
+  Enabled: false
+
+# Offense count: 5
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/EmptyElse:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/EmptyLinesAroundBlockBody:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/EmptyLinesAroundClassBody:
+  Enabled: false
+
+# Offense count: 1
+# Configuration parameters: MinBodyLength.
+Style/GuardClause:
+  Enabled: false
+
+# Offense count: 181
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles, UseHashRocketsWithSymbolValues.
+Style/HashSyntax:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: MaxLineLength.
+Style/IfUnlessModifier:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/IndentHash:
+  Enabled: false
+
+# Offense count: 3
+# Cop supports --auto-correct.
+Style/Lambda:
+  Enabled: false
+
+# Offense count: 12
+# Cop supports --auto-correct.
+Style/LeadingCommentSpace:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/LineEndConcatenation:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/MethodCallParentheses:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/MethodDefParentheses:
+  Enabled: false
+
+# Offense count: 2
+Style/MultilineTernaryOperator:
+  Enabled: false
+
+# Offense count: 4
+# Cop supports --auto-correct.
+Style/NegatedIf:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/NilComparison:
+  Enabled: false
+
+# Offense count: 7
+# Cop supports --auto-correct.
+Style/NumericLiterals:
+  MinDigits: 10
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: PreferredDelimiters.
+Style/PercentLiteralDelimiters:
+  Enabled: false
+
+# Offense count: 1
+# Configuration parameters: NamePrefix, NamePrefixBlacklist.
+Style/PredicateName:
+  Enabled: false
+
+# Offense count: 14
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/RaiseArgs:
+  Enabled: false
+
+# Offense count: 11
+# Cop supports --auto-correct.
+Style/RedundantBegin:
+  Enabled: false
+
+# Offense count: 19
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/SignalException:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/SpaceAfterComma:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/SpaceAfterMethodName:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/SpaceAroundEqualsInParameterDefault:
+  Enabled: false
+
+# Offense count: 7
+# Cop supports --auto-correct.
+# Configuration parameters: MultiSpaceAllowedForOperators.
+Style/SpaceAroundOperators:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/SpaceBeforeBlockBraces:
+  Enabled: false
+
+# Offense count: 15
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SupportedStyles.
+Style/SpaceInsideHashLiteralBraces:
+  Enabled: false
+
+# Offense count: 3
+# Cop supports --auto-correct.
+Style/SpaceInsideParens:
+  Enabled: false
+
+# Offense count: 23
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/StringLiterals:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/TrailingBlankLines:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyleForMultiline, SupportedStyles.
+Style/TrailingComma:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/UnneededPercentQ:
+  Enabled: false
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: WordRegex.
+Style/WordArray:
+  MinSize: 4

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 CHANGELOG
 ================================
+## v.3.4.0 (2015-08-20)
+### Added
+* Support for [Open Data Processing](https://datasift.com/products/open-data-processing-for-twitter/) batch uploads (Thanks @giovannelli)
+* Explicit supprot for 413 and 422 errors from API
+* Ability to get at API response headers using the ```object.response``` accessor. (Thanks again @giovannelli)
+
+### Changed
+* Bumped [rest-client](https://github.com/rest-client/rest-client) dependency to ~> 1.8
+* The OpenSSL version you select within the DataSift gem no longer changes the default OpenSSL version in your wider application; it only affects API calls made by the DataSift gem to DataSift's API
+
 ## v.3.3.0 (2015-08-05)
 ### Added
 * Explicit support for 429 errors from the API

data/VERSION

@@ -1 +1 @@
-3.3.0
+3.4.0

data/datasift.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test}/*`.split("\n")
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency('rest-client', '~> 1.6')
+  s.add_runtime_dependency('rest-client', '~> 1.8')
   s.add_runtime_dependency('multi_json', '~> 1.7')
   s.add_runtime_dependency('websocket-td', '~> 0.0.5')
   s.add_development_dependency('bundler', '~> 1.0')

data/examples/ingestion/twitter_gnip_batch_eg.rb

@@ -0,0 +1,61 @@
+require './../auth'
+class TwitterGnipBatchEg < DataSiftExample
+  def initialize
+    super
+    @datasift = DataSift::Client.new(@config)
+    run
+  end
+
+  def run
+    begin
+      # "gnip_1" is the default mapping used to map Tweets ingested from Gnip into a
+      #   format DataSift can use. Using this mapping will altomatically define the
+      #   IDML you need to use for this source
+      resource = [{
+        parameters: {
+          mapping: "gnip_1"
+        }
+      }]
+      puts "Creating Managed Source for ODP Ingestion\n"
+      source = @datasift.managed_source.create('twitter_gnip', 'Ruby ODP API', {}, resource)
+      puts "Manage Source with ID #{source[:data][:id]} created"
+
+      lines = 0
+      payload = ''
+      # Read interactions out of the fake Tweets fixture file
+      File.readlines('./../../test/fixtures/data/fake_gnip_tweets.json').each do |line|
+        lines += 1
+        payload += line
+
+        # Upload interactions as batches of five
+        if lines % 5 == 0
+          puts "\nUploading a batch of five interactions\n"
+          puts @datasift.odp.ingest(source[:data][:id], payload)[:data].to_json
+          payload = ''
+        end
+      end
+
+      puts "\nCleanup after this test and delete the Managed Source"
+      @datasift.managed_source.delete(source[:data][:id])
+
+    rescue DataSiftError => dse
+      puts dse.message
+      # Then match specific one to take action - All errors thrown by the client extend
+      #   DataSiftError
+      case dse
+        when ConnectionError
+          # some connection error
+        when AuthError
+        when BadRequestError
+        when PayloadTooLargeError
+          # You've probably hit your x_ingestion_data_ratelimit_limit rate limit; back off
+          #   and try again in a bit. Check your x_ingestion_request_ratelimit_reset header
+          #   to see when you can retry this request successfully
+        else
+          # do something else...
+      end
+    end
+  end
+end
+
+TwitterGnipBatchEg.new

data/lib/api/api_resource.rb

@@ -11,18 +11,16 @@ module DataSift
       @config = config
       config[:api_host] = 'api.datasift.com' unless config.has_key?(:api_host)
       config[:stream_host] = 'websocket.datasift.com' unless config.has_key?(:stream_host)
+      config[:ingestion_host] = 'in.datasift.com' unless config.has_key?(:ingestion_host)
       config[:api_version] = 'v1.2' unless config.has_key?(:api_version)
       config[:enable_ssl] = true unless config.has_key?(:enable_ssl)
-      # Only SSLv3, TLSv1 and TLSv1.2 currently supported, TLSv1.2 preferred
-      #   this is fixed in REST client and is scheduled for the 1.7.0 release
-      #   see https://github.com/rest-client/rest-client/pull/123
 
       ssl_default = "TLSv1_2"
       if RUBY_VERSION.to_i == 1
+        # Ruby 1.x does not seem to support > TLSv1
         ssl_default = "TLSv1"
       end
-      OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ssl_version] = config[:ssl_version] ||
-        ssl_default
+      config[:ssl_version] = config[:ssl_version] || ssl_default
 
       # max 320 seconds retry - http://dev.datasift.com/docs/streaming-api/reconnecting
       config[:max_retry_time] = 320 unless config.key?(:max_retry_time)

data/lib/datasift.rb

@@ -21,6 +21,7 @@ require dir + '/account'
 require dir + '/account_identity'
 require dir + '/account_identity_token'
 require dir + '/account_identity_limit'
+require dir + '/odp'
 #
 require 'rbconfig'
 
@@ -51,7 +52,7 @@ module DataSift
       raise InvalidConfigError.new('Config cannot be nil') if config.nil?
       if !config.key?(:username) || !config.key?(:api_key)
         raise InvalidConfigError.new('A valid username and API key are required. ' +
-          'You can check your API credentials at https://datasift.com/settings')
+          'You can check your API credentials at https://app.datasift.com/settings')
       end
 
       @config                   = config
@@ -66,11 +67,12 @@ module DataSift
       @account_identity         = DataSift::AccountIdentity.new(config)
       @account_identity_token   = DataSift::AccountIdentityToken.new(config)
       @account_identity_limit   = DataSift::AccountIdentityLimit.new(config)
+      @odp                      = DataSift::Odp.new(config)
     end
 
     attr_reader :historics, :push, :managed_source, :managed_source_resource,
       :managed_source_auth, :historics_preview, :pylon, :account,
-      :account_identity, :account_identity_token, :account_identity_limit
+      :account_identity, :account_identity_token, :account_identity_limit, :odp
 
     # Checks if the syntax of the given CSDL is valid
     #
@@ -167,7 +169,7 @@ module DataSift
       url += "#{URI.parse(url).query ? '&' : '?'}#{encode params}"
       payload = nil
     else
-      payload = MultiJson.dump(params)
+      payload = params.is_a?(String) ? params : MultiJson.dump(params)
       headers.update({ :content_type => 'application/json' })
     end
 
@@ -178,10 +180,11 @@ module DataSift
       :timeout      => timeout,
       :payload      => payload,
       :url          => url,
-      :ssl_version  => OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ssl_version],
+      :ssl_version  => config[:ssl_version],
       :verify_ssl   => OpenSSL::SSL::VERIFY_PEER
     )
 
+    response = nil
     begin
       response = RestClient::Request.execute options
       if !response.nil? && response.length > 0
@@ -223,7 +226,19 @@ module DataSift
         body = e.http_body
         if code && body
           error = MultiJson.load(body)
-          handle_api_error(e.http_code, (error['error'] ? error['error'] : '') + " for URL #{url}")
+          response_on_error = {
+            :data => nil,
+            :datasift => {
+              :x_ratelimit_limit     => e.response.headers[:x_ratelimit_limit],
+              :x_ratelimit_remaining => e.response.headers[:x_ratelimit_remaining],
+              :x_ratelimit_cost      => e.response.headers[:x_ratelimit_cost]
+            },
+            :http => {
+              :status  => e.response.code,
+              :headers => e.response.headers
+            }
+          }
+          handle_api_error(e.http_code, (error['error'] ? error['error'] : '') + " for URL #{url}", response_on_error)
         else
           process_client_error(e)
         end
@@ -238,8 +253,12 @@ module DataSift
   private
 
   def self.build_url(path, config)
-    'http' + (config[:enable_ssl] ? 's' : '') + '://' + config[:api_host] +
-      '/' + config[:api_version] + '/' + path
+    url = 'http' + (config[:enable_ssl] ? 's' : '') + '://' + config[:api_host]
+    if !config[:api_version].nil?
+      url += '/' + config[:api_version] + '/' + path
+    else
+      url += '/' + path
+    end
   end
 
   # Returns true if username or api key are not set
@@ -257,22 +276,26 @@ module DataSift
     params.collect { |param, value| [param, CGI.escape(value.to_s)].join('=') }.join('&')
   end
 
-  def self.handle_api_error(code, body)
+  def self.handle_api_error(code, body, response)
     case code
     when 400
-      raise BadRequestError.new(code, body)
+      raise BadRequestError.new(code, body, response)
     when 401
-      raise AuthError.new(code, body)
+      raise AuthError.new(code, body, response)
     when 404
-      raise ApiResourceNotFoundError.new(code, body)
+      raise ApiResourceNotFoundError.new(code, body, response)
     when 409
-      raise ConflictError.new(code, body)
+      raise ConflictError.new(code, body, response)
     when 410
-      raise GoneError.new(code, body)
+      raise GoneError.new(code, body, response)
+    when 413
+      raise PayloadTooLargeError.new(code, body, response)
+    when 422
+      raise UnprocessableEntityError.new(code, body, response)
     when 429
-      raise TooManyRequestsError.new(code, body)
+      raise TooManyRequestsError.new(code, body, response)
     else
-      raise DataSiftError.new(code, body)
+      raise DataSiftError.new(code, body, response)
     end
   end
 
@@ -310,7 +333,7 @@ module DataSift
       raise BadParametersError.new('on_close - 2 parameter required') unless on_close.arity == 2
     end
     begin
-      stream                    = WebsocketTD::Websocket.new('websocket.datasift.com', '/multi', "username=#{config[:username]}&api_key=#{config[:api_key]}")
+      stream                    = WebsocketTD::Websocket.new(config[:stream_host], '/multi', "username=#{config[:username]}&api_key=#{config[:api_key]}")
       connection                = LiveStream.new(config, stream)
       KNOWN_SOCKETS[connection] = Time.new.to_i
       stream.on_ping            = lambda { |data|
@@ -362,12 +385,12 @@ module DataSift
       if use_closed && !on_close.nil?
         on_close.call(connection, message)
       else
-        on_error.call(connection, ReconnectTimeoutError.new("Connecting to DataSift has failed, re-connection was attempted but
-                                         multiple consecutive failures where encountered. As a result no further
-                                         re-connection will be automatically attempted. Manually invoke connect() after
-                                          investigating the cause of the failure, be sure to observe DataSift's
-                                          re-connect policies available at http://dev.datasift.com/docs/streaming-api/reconnecting
-                                          - Error { #{message}}"))
+        on_error.call(connection, ReconnectTimeoutError.new("Connecting to DataSift has " \
+          "failed, re-connection was attempted but multiple consecutive failures where " \
+          "encountered. As a result no further re-connection will be automatically " \
+          "attempted. Manually invoke connect() after investigating the cause of the " \
+          "failure, be sure to observe DataSift's re-connect policies available at " \
+          "http://dev.datasift.com/docs/streaming-api/reconnecting - Error {#{message}}"))
       end
     else
       sleep config[:retry_timeout]

data/lib/errors.rb

@@ -1,11 +1,11 @@
 # Custom error class for rescuing DataSift errors
 class DataSiftError < StandardError
-  attr_reader :status
-  attr_reader :body
+  attr_reader :status, :body, :response
 
-  def initialize(http_status = nil, http_body = nil)
+  def initialize(http_status = nil, http_body = nil, response_on_error = nil)
     @status = http_status
     @body   = http_body
+    @response = response_on_error
   end
 
   def message
@@ -48,6 +48,14 @@ end
 class GoneError < DataSiftError
 end
 
+# Standard error returned when receiving a 413 response from the API
+class PayloadTooLargeError < DataSiftError
+end
+
+# Standard error returned when receiving a 422 response from the API
+class UnprocessableEntityError < DataSiftError
+end
+
 # Standard error returned when receiving a 429 response from the API
 class TooManyRequestsError < DataSiftError
 end

data/lib/odp.rb

@@ -0,0 +1,14 @@
+module DataSift
+  ##
+  # Use DataSift's Open Data Processing (ODP) to upload your own data to
+  #   DataSift for processing
+  class Odp < DataSift::ApiResource
+    def ingest(source_id, data)
+      fail ArgumentError, 'source_id is required' if source_id.nil?
+      fail ArgumentError, 'data payload is required' if data.nil?
+
+      config = @config.merge(api_host: @config[:ingestion_host], api_version: nil)
+      DataSift.request(:POST, source_id, config, data)
+    end
+  end
+end

data/test/datasift/odp/batch_upload_test.rb

@@ -0,0 +1,52 @@
+require File.expand_path('../../../test_helper', __FILE__)
+
+describe 'DataSift::Odp' do
+  before do
+    auth      = DataSiftExample.new
+    @datasift = auth.datasift
+    @data     = OpenStruct.new
+  end
+
+  ##
+  # Batch Upload
+  #
+  describe '#ingest (success)' do
+    before do
+      VCR.use_cassette('odp/batch/before_upload') do
+        resource = [{ parameters: { mapping: "gnip_1" } }]
+        @source = @datasift.managed_source.create('twitter_gnip', 'Ruby ODP API', {}, resource)
+      end
+    end
+
+    after do
+      VCR.use_cassette('odp/batch/after_upload') do
+        @datasift.managed_source.delete @source[:data][:id]
+      end
+    end
+
+    it 'can batch upload gnip twitter data' do
+      VCR.use_cassette('odp/batch/upload_success') do
+        payload = File.open(File.expand_path('../../fixtures/data/fake_gnip_tweets.json', File.dirname(__FILE__))).read
+        response = @datasift.odp.ingest(@source[:data][:id], payload)
+        assert_equal STATUS.valid, response[:http][:status]
+      end
+    end
+  end
+
+  describe '#ingest (failure)' do
+    it 'handles 404 when Managed Source can not be found' do
+      VCR.use_cassette('odp/batch/upload_failure_no_source') do
+        payload = File.open(File.expand_path('../../fixtures/data/fake_gnip_tweets.json', File.dirname(__FILE__))).read
+        assert_raises ApiResourceNotFoundError do
+          response = @datasift.odp.ingest('invalid_source_id', payload)
+        end
+      end
+    end
+
+    it 'raises BadParametersError when payload is missing' do
+      assert_raises ArgumentError do
+        @datasift.odp.ingest('invalid_source_id')
+      end
+    end
+  end
+end

data/test/fixtures/cassettes/odp/batch/after_upload.json

@@ -0,0 +1 @@
+{"http_interactions":[{"request":{"method":"delete","uri":"https://api.datasift.com/v1.2/source/delete?id=3446d92fd7274adeaf8b27f4e99f3b60","body":{"encoding":"US-ASCII","base64_string":""},"headers":{"Accept":["*/*; q=0.5, application/xml"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["DataSift/v1.2 Ruby/v3.4.0"],"Authorization":["<USERNAME:API_KEY>"],"Content-Type":["application/x-www-form-urlencoded"]}},"response":{"status":{"code":204,"message":"No Content"},"headers":{"Server":["nginx"],"Date":["Fri, 14 Aug 2015 19:27:56 GMT"],"Content-Type":["text/html"],"Connection":["close"],"X-Api-Version":["1.2"],"P3p":["CP=\"CAO PSA\""],"X-Ratelimit-Limit":["10000"],"X-Ratelimit-Remaining":["9816"],"X-Ratelimit-Cost":["25"],"X-Ratelimit-Reset":["1439582455"],"X-Ratelimit-Reset-Ttl":["1979"],"Access-Control-Allow-Origin":["*"],"Access-Control-Allow-Headers":["Authorization,Content-Type"],"X-Served-By":["ded2584"],"X-Frame-Options":["SAMEORIGIN"]},"body":{"encoding":"UTF-8","base64_string":""},"http_version":null},"recorded_at":"Fri, 14 Aug 2015 19:27:56 GMT"}],"recorded_with":"VCR 2.9.3"}

data/test/fixtures/cassettes/odp/batch/before_upload.json

@@ -0,0 +1 @@
+{"http_interactions":[{"request":{"method":"post","uri":"https://api.datasift.com/v1.2/source/create","body":{"encoding":"UTF-8","base64_string":"PEJBU0U2NF9TVFJJTkc+\n"},"headers":{"Accept":["*/*; q=0.5, application/xml"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["DataSift/v1.2 Ruby/v3.4.0"],"Authorization":["<USERNAME:API_KEY>"],"Content-Type":["application/json"],"Content-Length":["110"]}},"response":{"status":{"code":201,"message":"Created"},"headers":{"Server":["nginx"],"Date":["Fri, 14 Aug 2015 19:27:36 GMT"],"Content-Type":["application/json"],"Transfer-Encoding":["chunked"],"Connection":["close"],"X-Api-Version":["1.2"],"P3p":["CP=\"CAO PSA\""],"X-Ratelimit-Limit":["10000"],"X-Ratelimit-Remaining":["9842"],"X-Ratelimit-Cost":["25"],"X-Ratelimit-Reset":["1439582455"],"X-Ratelimit-Reset-Ttl":["2000"],"Access-Control-Allow-Origin":["*"],"Access-Control-Allow-Headers":["Authorization,Content-Type"],"X-Served-By":["ded2587"],"X-Frame-Options":["SAMEORIGIN"]},"body":{"encoding":"UTF-8","base64_string":"eyJpZCI6IjM0NDZkOTJmZDcyNzRhZGVhZjhiMjdmNGU5OWYzYjYwIiwic3Rh\ndHVzIjoic3RvcHBlZCIsInNvdXJjZV90eXBlIjoidHdpdHRlcl9nbmlwIiwi\nbmFtZSI6IlJ1YnkgT0RQIEFQSSIsImF1dGgiOltdLCJyZXNvdXJjZXMiOlt7\nInJlc291cmNlX2lkIjoiMjUyM2FhMDUzNWIzNGYzNmFmMzlhYmEzZTJkY2Rj\nYmIiLCJwYXJhbWV0ZXJzIjp7InVuaXF1ZV9pZF9sb2NhdGlvbiI6ImlkIiwi\naW50ZXJhY3Rpb25fdHlwZSI6InR3aXR0ZXJfZ25pcCIsImF1Z21lbnRhdGlv\nbnMiOlsiZ2VuZGVyIiwia2xvdXQiLCJsYW5ndWFnZWRldGVjdGlvbiIsInNh\nbGllbmNlX2NvbnRlbnQiLCJzYWxpZW5jZV90aXRsZSIsInRyZW5kcyIsInpl\nbnlhX2NvbnRlbnQiLCJ6ZW55YV90aXRsZSJdLCJpZG1sIjoiIyBNYXBzIHRo\nZSBHbmlwIEFjdGl2aXR5U3RyZWFtIGZvcm1hdCB0byBhIFR3aXR0ZXIgb2Jq\nZWN0XG5bbWFpbl1cbnR3aXR0ZXIgPSAoYXBwbHkoXCJjcmVhdGVfcG9zdFwi\nKSBbIHJvb3QudmVyYiA9PSBcInBvc3RcIiBdIHxcbiAgICAgICAgICAgYXBw\nbHkoXCJjcmVhdGVfc2hhcmVcIikgWyByb290LnZlcmIgPT0gXCJzaGFyZVwi\nXSB8XG4gICAgICAgICAgIGFwcGx5KFwiY3JlYXRlX2RlbGV0ZVwiKSBbIHJv\nb3QudmVyYiA9PSBcImRlbGV0ZVwiIF0gfFxuICAgICAgICAgICBhcHBseShc\nImNyZWF0ZV9zdGF0dXNfd2l0aGhlbGRcIikgWyByb290LnZlcmIgPT0gXCJz\ndGF0dXNfd2l0aGhlbGRcIiBdIHxcbiAgICAgICAgICAgYXBwbHkoXCJjcmVh\ndGVfc2NydWJfZ2VvXCIpIFsgcm9vdC52ZXJiID09IFwic2NydWJfZ2VvXCIg\nXSB8XG4gICAgICAgICAgIGFwcGx5KFwiY3JlYXRlX290aGVyX2NvbXBsaWFu\nY2VcIilcbiAgICAgICAgICApXG5cbiMgZGVsZXRlcyBoYXZlIGFuIGV4dHJh\nIHRvcC1sZXZlbCBmaWVsZFxuZGVsZXRlZCA9IHRydWUgW3Jvb3QudmVyYiA9\nPSBcImRlbGV0ZVwiXVxuXG5bY3JlYXRlX2RlbGV0ZV1cbmlkICAgICAgICAg\nID0gb2JqZWN0LmlkLnNwbGl0KFwiOlwiKVsyXVxuY3JlYXRlZF9hdCAgPSB0\naW1lc3RhbXBNcy5kYXRlKClcbnVzZXIgICAgICAgID0gYWN0b3IuYXBwbHko\nXCJjcmVhdGVfYWN0b3JcIilcblxuW2NyZWF0ZV9zdGF0dXNfd2l0aGhlbGRd\nXG5jcmVhdGVkX2F0ICAgICAgICAgPSB0aW1lc3RhbXBNcy5kYXRlKClcbnN0\nYXR1cyAgICAgICAgICAgICA9IHZlcmJcbndpdGhoZWxkLmNvdW50cmllcyA9\nIG9iamVjdC53aXRoaGVsZF9pbl9jb3VudHJpZXNcblxuIyB0aGUgb2JqZWN0\nIHJlZmVycyB0byB0aGUgc3RhdHVzIG1lc3NhZ2UgYW5kIHRoZXJlJ3MgYSB1\nc2VyX2lkIGZpZWxkIGluc3RlYWQgb2YgYW4gYWN0b3JcbmlkICAgICAgICAg\nID0gb2JqZWN0LmlkLm1hdGNoKFwiLio/KFxcXFxkKylcIilbMF1cbnVzZXIu\naWQgICAgID0gb2JqZWN0LnVzZXJfaWQubWF0Y2goXCIuKj8oXFxcXGQrKVwi\nKVswXS5pbnQoKVxudXNlci5pZF9zdHIgPSBvYmplY3QudXNlcl9pZC5tYXRj\naChcIi4qPyhcXFxcZCspXCIpWzBdXG5cbltjcmVhdGVfc2NydWJfZ2VvXVxu\nY3JlYXRlZF9hdCAgPSB0aW1lc3RhbXBNcy5kYXRlKClcbnN0YXR1cyAgICAg\nID0gdmVyYlxudXNlciAgICAgICAgPSBhY3Rvci5hcHBseShcImNyZWF0ZV9h\nY3RvclwiKVxuc2NydWJfZ2VvLnVwX3RvX3N0YXR1c19pZCA9IHRhcmdldC51\ncF90b19pZC5tYXRjaChcIi4qPyhcXFxcZCspXCIpWzBdXG5cbltjcmVhdGVf\nb3RoZXJfY29tcGxpYW5jZV1cbmNyZWF0ZWRfYXQgID0gdGltZXN0YW1wTXMu\nZGF0ZSgpXG5zdGF0dXMgICAgICA9IHZlcmJcbnVzZXIgICAgICAgID0gb2Jq\nZWN0LmFwcGx5KFwiY3JlYXRlX2FjdG9yXCIpXG53aXRoaGVsZC5jb3VudHJp\nZXMgPSB3aXRoaGVsZF9pbl9jb3VudHJpZXNcblxuW2NyZWF0ZV9nZW9fZnJv\nbV9wb2x5XVxubGF0aXR1ZGUgICAgPSBnZW8uY29vcmRpbmF0ZXNbMF0uZXh0\ncmFjdCh0aGlzWzFdKS5hdmVyYWdlKClcbmxvbmdpdHVkZSAgID0gZ2VvLmNv\nb3JkaW5hdGVzWzBdLmV4dHJhY3QodGhpc1swXSkuYXZlcmFnZSgpXG5cbltj\ncmVhdGVfZ2VvX2Zyb21fcG9pbnRdXG5sYXRpdHVkZSAgICA9IGdlby5jb29y\nZGluYXRlc1swXVxubG9uZ2l0dWRlICAgPSBnZW8uY29vcmRpbmF0ZXNbMV1c\nblxuW2NyZWF0ZV9sb2NfZ2VvX2Zyb21fcG9seV1cbmxhdGl0dWRlICAgID0g\nbG9jYXRpb24uZ2VvLmNvb3JkaW5hdGVzWzBdLmV4dHJhY3QodGhpc1sxXSku\nYXZlcmFnZSgpXG5sb25naXR1ZGUgICA9IGxvY2F0aW9uLmdlby5jb29yZGlu\nYXRlc1swXS5leHRyYWN0KHRoaXNbMF0pLmF2ZXJhZ2UoKVxuXG5bY3JlYXRl\nX2xvY19nZW9fZnJvbV9wb2ludF1cbmxhdGl0dWRlICAgID0gbG9jYXRpb24u\nZ2VvLmNvb3JkaW5hdGVzWzBdXG5sb25naXR1ZGUgICA9IGxvY2F0aW9uLmdl\nby5jb29yZGluYXRlc1sxXVxuXG5bY3JlYXRlX3Bvc3RdXG5pZCAgICAgICAg\nICAgICAgPSBpZC5zcGxpdChcIjpcIilbMl1cbmxpbmsgICAgICAgICAgICA9\nIGxpbmtcbnVzZXIgICAgICAgICAgICA9IGFjdG9yLmFwcGx5KFwiY3JlYXRl\nX2FjdG9yXCIpXG5sYW5nICAgICAgICAgICAgPSB0d2l0dGVyX2xhbmdcbnRl\neHQgICAgICAgICAgICA9IGJvZHkuc2l6ZSgxMDAwKVxuY3JlYXRlZF9hdCAg\nICAgID0gcG9zdGVkVGltZS5kYXRlKClcbmZpbHRlcl9sZXZlbCAgICA9IHR3\naXR0ZXJfZmlsdGVyX2xldmVsXG5zb3VyY2UgICAgICAgICAgPSBnZW5lcmF0\nb3IuZGlzcGxheU5hbWVcbmhhc2h0YWdzICAgICAgICA9IHR3aXR0ZXJfZW50\naXRpZXMuaGFzaHRhZ3MgWyB0cnVuY2F0ZWQgIT0gdHJ1ZSBdIC5leHRyYWN0\nKHRleHQpXG5tZW50aW9ucyAgICAgICAgPSB0d2l0dGVyX2VudGl0aWVzLnVz\nZXJfbWVudGlvbnMuZXh0cmFjdChzY3JlZW5fbmFtZSlcbm1lbnRpb25faWRz\nICAgICA9IHR3aXR0ZXJfZW50aXRpZXMudXNlcl9tZW50aW9ucy5leHRyYWN0\nKGlkKVxubWVkaWEgICAgICAgICAgID0gdHdpdHRlcl9lbnRpdGllcy5tZWRp\nYS5leHRyYWN0KGJsYWNrbGlzdChcImluZGljZXNcIikpXG5wbGFjZSAgICAg\nICAgICAgPSBsb2NhdGlvbi5hcHBseShcImNyZWF0ZV9wbGFjZVwiKVxuc3lt\nYm9scyAgICAgICAgID0gc3ltYm9scy5leHRyYWN0KHRleHQpXG5nZW8gICAg\nICAgICAgICAgPSAoYXBwbHkoXCJjcmVhdGVfZ2VvX2Zyb21fcG9pbnRcIikg\nICAgIFsgcm9vdC5nZW8udHlwZSA9PSBcIlBvaW50XCIgXSAgICAgICAgICAg\nfCAjIHRyeSB0aGUgYWNjdXJhdGUgcG9pbnRcbiAgICAgICAgICAgICAgICAg\nICAgICAgYXBwbHkoXCJjcmVhdGVfZ2VvX2Zyb21fcG9seVwiKSAgICAgIFsg\ncm9vdC5nZW8udHlwZSA9PSBcIlBvbHlnb25cIiBdICAgICAgICAgfCAjIHRy\neSBhbiBhY2N1cmF0ZSBwb2x5XG4gICAgICAgICAgICAgICAgICAgICAgICAg\nICAgICAgICAgICAgICBhcHBseShcImNyZWF0ZV9sb2NfZ2VvX2Zyb21fcG9p\nbnRcIikgWyByb290LmxvY2F0aW9uLmdlby50eXBlID09IFwiUG9pbnRcIiBd\nICB8ICMgdHJ5IHRoZSBsb2NhdGlvbiBwb2ludFxuICAgICAgICAgICAgICAg\nICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGFwcGx5\nKFwiY3JlYXRlX2xvY19nZW9fZnJvbV9wb2x5XCIpICBbIHJvb3QubG9jYXRp\nb24uZ2VvLnR5cGUgPT0gXCJQb2x5Z29uXCIgXSkgIyB0cnkgdGhlIGxvY2F0\naW9uIHBvbHlcblxuIyBzcGxpdCBsaW5rcyBsaWtlIGh0dHA6XC9cL3R3aXR0\nZXIuY29tXC9zY3JlZW5fbmFtZVwvc3RhdHVzZXNcL3N0YXR1c19pZFxuaW5f\ncmVwbHlfdG9fc2NyZWVuX25hbWUgICA9IGluUmVwbHlUby5saW5rLnNwbGl0\nKFwiXFxcL1wiKSBbM11cbmluX3JlcGx5X3RvX3N0YXR1c19pZCAgICAgPSBp\nblJlcGx5VG8ubGluay5zcGxpdChcIlxcXC9cIikgWzVdXG5cbiMgZXh0cmFj\ndCBsaW5rcyBmcm9tIHR3aXR0ZXIgZW50aXRpZXNcbmxpbmtzICAgICAgICAg\nICA9IHR3aXR0ZXJfZW50aXRpZXMudXJscy5leHRyYWN0KChleHBhbmRlZF91\ncmwgfCB1cmwpKVswOjVdXG5kb21haW5zICAgICAgICAgPSB0d2l0dGVyX2Vu\ndGl0aWVzLnVybHMuZXh0cmFjdCgoZXhwYW5kZWRfdXJsLnVybCgpLmhvc3Qg\nfCB1cmwudXJsKCkuaG9zdCkpWzA6NV1cbmRpc3BsYXlfdXJscyAgICA9IHR3\naXR0ZXJfZW50aXRpZXMudXJscy5leHRyYWN0KGRpc3BsYXlfdXJsKVswOjVd\nXG5cbltjcmVhdGVfc2hhcmVdXG5yZXR3ZWV0ICAgICA9IGFwcGx5KFwiY3Jl\nYXRlX3Bvc3RcIilcbnJldHdlZXRlZCAgID0gb2JqZWN0LmFwcGx5KFwiY3Jl\nYXRlX3Bvc3RcIilcbmlkICAgICAgICAgID0gaWQuc3BsaXQoXCI6XCIpWzJd\nXG5cbiMgd2UgY2FuIGdldCB0aGUgb3JkZXIgdGhlIHJldHdlZXQgd2FzIG1h\nZGUgaW4gYnkgZ2V0dGluZyB0aGUgb3JpZ2luYWwgdHdlZXRzIHJldHdlZXQg\nY291bnRcbnJldHdlZXQuY291bnQgPSByZXR3ZWV0Q291bnQuaW50KCkuZGVm\nYXVsdCgwKVxuXG4jIHRoZSByZXR3ZWV0IGlzIG9mdGVuIHRydW5jYXRlZCBi\neSB0d2l0dGVyLiBsZXQncyBmb3JtYXQgaXQgb3Vyc2VsZlxucmV0d2VldC50\nZXh0ID0gXCJSVCBAJXM6ICVzXCIuZm9ybWF0KG9iamVjdC5hY3Rvci5wcmVm\nZXJyZWRVc2VybmFtZSwgb2JqZWN0LmJvZHkpXG5cbiMgd2UgZG9uJ3QgbWFw\nIHRoZSBmYXZvcml0ZXMgY291bnQgb2YgYSBuZXcgdHdlZXQgc2luY2UgaXQn\nbGwgbGlrZWx5IGJlIHplcm9cbnJldHdlZXRlZC5mYXZvdXJpdGVzX2NvdW50\nID0gb2JqZWN0LmZhdm9yaXRlc0NvdW50XG5cbiMgcHJlcGVuZCB0aGUgcmV0\nd2VldGVkIHVzZXIgc2NyZWVuIG5hbWUgYW5kIGlkIHRvIHRoZSBtZW50aW9u\ncyBsaXN0XG5yZXR3ZWV0Lm1lbnRpb25zID0gQHJldHdlZXQubWVudGlvbnMu\ncHJlcGVuZChAcmV0d2VldGVkLnVzZXIuc2NyZWVuX25hbWUpXG5yZXR3ZWV0\nLm1lbnRpb25faWRzID0gQHJldHdlZXQubWVudGlvbl9pZHMucHJlcGVuZChA\ncmV0d2VldGVkLnVzZXIuaWQpXG5cbltjcmVhdGVfcGxhY2VdXG5jb3VudHJ5\nICAgICAgICAgPSBjb3VudHJ5X2NvZGVcbmNvdW50cnlfY29kZSAgICA9IHR3\naXR0ZXJfY291bnRyeV9jb2RlXG5mdWxsX25hbWUgICAgICAgPSBkaXNwbGF5\nTmFtZVxubmFtZSAgICAgICAgICAgID0gbmFtZVxucGxhY2VfdHlwZSAgICAg\nID0gdHdpdHRlcl9wbGFjZV90eXBlXG51cmwgICAgICAgICAgICAgPSBsaW5r\nXG5cbltjcmVhdGVfYWN0b3JdXG5pZCAgICAgICAgICAgICAgICAgICAgICA9\nIGlkLm1hdGNoKFwiLio/KFxcXFxkKylcIilbMF0uaW50KClcbmlkX3N0ciAg\nICAgICAgICAgICAgICAgID0gaWQubWF0Y2goXCIuKj8oXFxcXGQrKVwiKVsw\nXVxudXJsICAgICAgICAgICAgICAgICAgICAgPSBsaW5rXG5uYW1lICAgICAg\nICAgICAgICAgICAgICA9IGRpc3BsYXlOYW1lXG5wcm9maWxlX2ltYWdlX3Vy\nbCAgICAgICA9IGltYWdlXG5wcm9maWxlX2ltYWdlX3VybF9odHRwcyA9IGlt\nYWdlXG5kZXNjcmlwdGlvbiAgICAgICAgICAgICA9IHN1bW1hcnlcbmNyZWF0\nZWRfYXQgICAgICAgICAgICAgID0gcG9zdGVkVGltZS5kYXRlKClcbmxvY2F0\naW9uICAgICAgICAgICAgICAgID0gbG9jYXRpb24uZGlzcGxheU5hbWVcbnV0\nY19vZmZzZXQgICAgICAgICAgICAgID0gdXRjT2Zmc2V0LmludCgpXG5zY3Jl\nZW5fbmFtZSAgICAgICAgICAgICA9IHByZWZlcnJlZFVzZXJuYW1lXG5sYW5n\nICAgICAgICAgICAgICAgICAgICA9IGxhbmd1YWdlc1swXVxudGltZV96b25l\nICAgICAgICAgICAgICAgPSB0d2l0dGVyVGltZVpvbmVcbmZyaWVuZHNfY291\nbnQgICAgICAgICAgID0gZnJpZW5kc0NvdW50XG5mb2xsb3dlcnNfY291bnQg\nICAgICAgICA9IGZvbGxvd2Vyc0NvdW50XG5saXN0ZWRfY291bnQgICAgICAg\nICAgICA9IGxpc3RlZENvdW50XG5zdGF0dXNlc19jb3VudCAgICAgICAgICA9\nIHN0YXR1c2VzQ291bnRcbnZlcmlmaWVkICAgICAgICAgICAgICAgID0gdmVy\naWZpZWRcbmZhdm91cml0ZXNfY291bnQgICAgICAgID0gZmF2b3JpdGVzQ291\nbnQiLCJjYXRlZ29yeSI6InR3aXR0ZXJjYXRlZ29yeV8xIn19XSwicGFyYW1l\ndGVycyI6W10sImNyZWF0ZWRfYXQiOjE0Mzk1ODA0NTZ9\n"},"http_version":null},"recorded_at":"Fri, 14 Aug 2015 19:27:36 GMT"}],"recorded_with":"VCR 2.9.3"}

data/test/fixtures/cassettes/odp/batch/upload_failure_no_source.json

@@ -0,0 +1 @@
+{"http_interactions":[{"request":{"method":"post","uri":"https://in.datasift.com/invalid_source_id","body":{"encoding":"UTF-8","base64_string":"PEJBU0U2NF9TVFJJTkc+\n"},"headers":{"Accept":["*/*; q=0.5, application/xml"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["DataSift/ Ruby/v3.4.0"],"Authorization":["<USERNAME:API_KEY>"],"Content-Type":["application/json"],"Content-Length":["27255"]}},"response":{"status":{"code":404,"message":"Not Found"},"headers":{"Cache-Control":["private"],"Content-Type":["application/json"],"Server":["DataSift Ingestion/1.0"],"X-Ingestion-Data-Ratelimit-Reset":["0"],"X-Ingestion-Request-Ratelimit-Limit":["10000"],"X-Ingestion-Request-Ratelimit-Reset":["60"],"X-Ingestion-Data-Ratelimit-Remaining":["36864000"],"X-Ingestion-Request-Ratelimit-Reset-Ttl":["1439580312"],"X-Ingestion-Data-Ratelimit-Reset-Ttl":["1439580252"],"X-Ingestion-Data-Ratelimit-Limit":["36864000"],"X-Ingestion-Request-Ratelimit-Remaining":["9999"],"Transfer-Encoding":["chunked"],"Strict-Transport-Security":["max-age=31536000"]},"body":{"encoding":"ASCII-8BIT","base64_string":"eyJlcnJvciI6IlVua25vd24gdXNlciBzb3VyY2UifQ==\n"},"http_version":null},"recorded_at":"Fri, 14 Aug 2015 19:24:12 GMT"}],"recorded_with":"VCR 2.9.3"}

data/test/fixtures/cassettes/odp/batch/upload_success.json

@@ -0,0 +1 @@
+{"http_interactions":[{"request":{"method":"post","uri":"https://in.datasift.com/3446d92fd7274adeaf8b27f4e99f3b60","body":{"encoding":"UTF-8","base64_string":"PEJBU0U2NF9TVFJJTkc+\n"},"headers":{"Accept":["*/*; q=0.5, application/xml"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["DataSift/ Ruby/v3.4.0"],"Authorization":["<USERNAME:API_KEY>"],"Content-Type":["application/json"],"Content-Length":["27255"]}},"response":{"status":{"code":200,"message":"OK"},"headers":{"Cache-Control":["private"],"Content-Type":["application/json"],"Server":["DataSift Ingestion/1.0"],"X-Ingestion-Data-Ratelimit-Reset":["60"],"X-Ingestion-Request-Ratelimit-Limit":["10000"],"X-Ingestion-Request-Ratelimit-Reset":["60"],"X-Ingestion-Data-Ratelimit-Remaining":["36836754"],"X-Ingestion-Request-Ratelimit-Reset-Ttl":["1439580536"],"X-Ingestion-Data-Ratelimit-Reset-Ttl":["1439580536"],"X-Ingestion-Data-Ratelimit-Limit":["36864000"],"X-Ingestion-Request-Ratelimit-Remaining":["9999"],"Transfer-Encoding":["chunked"],"Strict-Transport-Security":["max-age=31536000"]},"body":{"encoding":"ASCII-8BIT","base64_string":"eyJhY2NlcHRlZCI6MTAsInRvdGFsX21lc3NhZ2VfYnl0ZXMiOjI3MjQ2fQ==\n"},"http_version":null},"recorded_at":"Fri, 14 Aug 2015 19:27:56 GMT"}],"recorded_with":"VCR 2.9.3"}

data/test/fixtures/data/fake_gnip_tweets.json

@@ -0,0 +1,10 @@
+{"id":"tag:search.twitter.com,2005:587648815764746240","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:36136637","link":"http://www.twitter.com/davidmarkwales","displayName":"Dave Thomas","postedTime":"2009-04-28T18:14:12.000Z","image":"https://pbs.twimg.com/profile_images/1173129755/image_normal.jpg","summary":"Instagram: davidmarkwales","links":[{"href":null,"rel":"me"}],"friendsCount":340,"followersCount":122,"listedCount":3,"statusesCount":499,"twitterTimeZone":"London","verified":false,"utcOffset":"3600","preferredUsername":"davidmarkwales","languages":["en"],"location":{"objectType":"place","displayName":"Liverpool"},"favoritesCount":33},"verb":"post","postedTime":"2015-04-13T16:09:30.000Z","generator":{"displayName":"Twitter for Android","link":"http://twitter.com/download/android"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/davidmarkwales/statuses/587648815764746240","body":"@asda has cancelled my home delivery today, no reason given, not even a courtesy call or email to let me know. Any idea why they would?","object":{"objectType":"note","id":"object:search.twitter.com,2005:587648815764746240","summary":"@asda has cancelled my home delivery today, no reason given, not even a courtesy call or email to let me know. Any idea why they would?","link":"http://twitter.com/davidmarkwales/statuses/587648815764746240","postedTime":"2015-04-13T16:09:30.000Z"},"favoritesCount":0,"location":{"objectType":"place","displayName":"Garston, England","name":"Garston","country_code":"United Kingdom","twitter_country_code":"GB","link":"https://api.twitter.com/1.1/geo/id/12f69ad404352073.json","geo":{"type":"Polygon","coordinates":[[[-2.940659,53.331287],[-2.940659,53.38523],[-2.818002,53.38523],[-2.818002,53.331287]]]}},"geo":{"type":"Point","coordinates":[53.345841,-2.87278]},"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"asda","name":"Asda","id":20123366,"id_str":"20123366","indices":[0,5]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"}],"language":{"value":"en"}}}
+{"id":"tag:search.twitter.com,2005:587648834689445889","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:3157665856","link":"http://www.twitter.com/emilywlittle","displayName":"Emily ","postedTime":"2015-04-11T20:20:40.000Z","image":"https://pbs.twimg.com/profile_images/586987964225445912/ubKhVysy_normal.jpg","summary":null,"links":[{"href":null,"rel":"me"}],"friendsCount":61,"followersCount":20767,"listedCount":0,"statusesCount":128,"twitterTimeZone":"Amsterdam","verified":false,"utcOffset":"7200","preferredUsername":"emilywlittle","languages":["en"],"favoritesCount":0},"verb":"post","postedTime":"2015-04-13T16:09:34.000Z","generator":{"displayName":"Yappem","link":"http://www.yappem.com/"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/emilywlittle/statuses/587648834689445889","body":"@amazon online bookstore http://t.co/Cz3XUWDvva http://t.co/6lNnsbABii","object":{"objectType":"note","id":"object:search.twitter.com,2005:587648834689445889","summary":"@amazon online bookstore http://t.co/Cz3XUWDvva http://t.co/6lNnsbABii","link":"http://twitter.com/emilywlittle/statuses/587648834689445889","postedTime":"2015-04-13T16:09:34.000Z"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[{"url":"http://t.co/Cz3XUWDvva","expanded_url":"http://goo.gl/iXPwB4","display_url":"goo.gl/iXPwB4","indices":[25,47]}],"user_mentions":[{"screen_name":"amazon","name":"Amazon","id":20793816,"id_str":"20793816","indices":[0,7]}],"symbols":[],"media":[{"id":587648834580455424,"id_str":"587648834580455424","indices":[48,70],"media_url":"http://pbs.twimg.com/media/CCe_gGXXIAAWYU6.jpg","media_url_https":"https://pbs.twimg.com/media/CCe_gGXXIAAWYU6.jpg","url":"http://t.co/6lNnsbABii","display_url":"pic.twitter.com/6lNnsbABii","expanded_url":"http://twitter.com/emilywlittle/status/587648834689445889/photo/1","type":"photo","sizes":{"small":{"w":340,"h":340,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":600,"h":600,"resize":"fit"},"large":{"w":600,"h":600,"resize":"fit"}}}]},"twitter_extended_entities":{"media":[{"id":587648834580455424,"id_str":"587648834580455424","indices":[48,70],"media_url":"http://pbs.twimg.com/media/CCe_gGXXIAAWYU6.jpg","media_url_https":"https://pbs.twimg.com/media/CCe_gGXXIAAWYU6.jpg","url":"http://t.co/6lNnsbABii","display_url":"pic.twitter.com/6lNnsbABii","expanded_url":"http://twitter.com/emilywlittle/status/587648834689445889/photo/1","type":"photo","sizes":{"small":{"w":340,"h":340,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":600,"h":600,"resize":"fit"},"large":{"w":600,"h":600,"resize":"fit"}}}]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"urls":[{"url":"http://t.co/Cz3XUWDvva","expanded_url":"https://www.yappem.com/redirect/?pid=782008","expanded_status":403},{"url":"http://t.co/6lNnsbABii","expanded_url":"http://twitter.com/emilywlittle/status/587648834689445889/photo/1","expanded_status":200}],"language":{"value":"en"}}}
+{"id":"tag:search.twitter.com,2005:587648869485379585","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:265357386","link":"http://www.twitter.com/dr_acula365","displayName":"Andy","postedTime":"2011-03-13T12:16:17.000Z","image":"https://pbs.twimg.com/profile_images/2438808933/to47tp1pbrww1gqwbe9k_normal.jpeg","summary":"Just a guy who likes sport, music and food. You'll find me in the LGBT thread on f365 forum. Please don't follow me if you are under 18. You will be blocked.","links":[{"href":null,"rel":"me"}],"friendsCount":185,"followersCount":128,"listedCount":3,"statusesCount":916,"twitterTimeZone":"Casablanca","verified":false,"utcOffset":"3600","preferredUsername":"dr_acula365","languages":["en"],"location":{"objectType":"place","displayName":"Newcastle"},"favoritesCount":0},"verb":"post","postedTime":"2015-04-13T16:09:43.000Z","generator":{"displayName":"Tweetbot for iΟS","link":"http://tapbots.com/tweetbot"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/dr_acula365/statuses/587648869485379585","body":"@Tesco when your website says next day click and collect and the email says pick up from 3pm, when do you actually mean I can pick it up?","object":{"objectType":"note","id":"object:search.twitter.com,2005:587648869485379585","summary":"@Tesco when your website says next day click and collect and the email says pick up from 3pm, when do you actually mean I can pick it up?","link":"http://twitter.com/dr_acula365/statuses/587648869485379585","postedTime":"2015-04-13T16:09:43.000Z"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"Tesco","name":"Tesco","id":271986064,"id_str":"271986064","indices":[0,6]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}
+{"id":"tag:search.twitter.com,2005:587648932337098752","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:2642049675","link":"http://www.twitter.com/michaeljohnmcr","displayName":"Michael Delaney","postedTime":"2014-06-25T22:59:21.000Z","image":"https://pbs.twimg.com/profile_images/544998090609291264/W-4vCwC7_normal.jpeg","summary":"邁克爾 ❌❌","links":[{"href":null,"rel":"me"}],"friendsCount":48,"followersCount":19,"listedCount":0,"statusesCount":302,"twitterTimeZone":"Azores","verified":false,"utcOffset":"0","preferredUsername":"michaeljohnmcr","languages":["en"],"favoritesCount":10},"verb":"post","postedTime":"2015-04-13T16:09:58.000Z","generator":{"displayName":"Twitter for iPhone","link":"http://twitter.com/download/iphone"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/michaeljohnmcr/statuses/587648932337098752","body":"@asda I need a number to make a complaint with in the uk can't find it online","object":{"objectType":"note","id":"object:search.twitter.com,2005:587648932337098752","summary":"@asda I need a number to make a complaint with in the uk can't find it online","link":"http://twitter.com/michaeljohnmcr/statuses/587648932337098752","postedTime":"2015-04-13T16:09:58.000Z"},"favoritesCount":0,"location":{"objectType":"place","displayName":"Manchester, England","name":"Manchester","country_code":"United Kingdom","twitter_country_code":"GB","link":"https://api.twitter.com/1.1/geo/id/315b740b108481f6.json","geo":{"type":"Polygon","coordinates":[[[-2.319934,53.343623],[-2.319934,53.5702824],[-2.147026,53.5702824],[-2.147026,53.343623]]]}},"geo":{"type":"Point","coordinates":[53.38604,-2.251666]},"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"asda","name":"Asda","id":20123366,"id_str":"20123366","indices":[0,5]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"}],"language":{"value":"en"}}}
+{"id":"tag:search.twitter.com,2005:587649014415278080","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:14577165","link":"http://www.twitter.com/marinmaven","displayName":"marinmaven","postedTime":"2008-04-28T23:11:01.000Z","image":"https://pbs.twimg.com/profile_images/489833602222661632/dKsJXZaX_normal.jpeg","summary":"I am a San Francisco native who grew up on the Peninsula. I muse about politics, economics, culture, and living in Marin and beyond.","links":[{"href":"http://www.marinmaven.com/","rel":"me"}],"friendsCount":2316,"followersCount":2360,"listedCount":123,"statusesCount":19661,"twitterTimeZone":"Pacific Time (US & Canada)","verified":false,"utcOffset":"-25200","preferredUsername":"marinmaven","languages":["en"],"location":{"objectType":"place","displayName":"ÜT: 38.006898,-122.531684"},"favoritesCount":355},"verb":"post","postedTime":"2015-04-13T16:10:17.000Z","generator":{"displayName":"Twitter for iPhone","link":"http://twitter.com/download/iphone"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/marinmaven/statuses/587649014415278080","body":"@Macys I have sent you an email detailing what has happened.","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649014415278080","summary":"@Macys I have sent you an email detailing what has happened.","link":"http://twitter.com/marinmaven/statuses/587649014415278080","postedTime":"2015-04-13T16:10:17.000Z"},"inReplyTo":{"link":"http://twitter.com/Macys/statuses/587641764682403841"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"Macys","name":"Macy's","id":50687788,"id_str":"50687788","indices":[0,6]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}
+{"id":"tag:search.twitter.com,2005:587649047697215488","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:265357386","link":"http://www.twitter.com/dr_acula365","displayName":"Andy","postedTime":"2011-03-13T12:16:17.000Z","image":"https://pbs.twimg.com/profile_images/2438808933/to47tp1pbrww1gqwbe9k_normal.jpeg","summary":"Just a guy who likes sport, music and food. You'll find me in the LGBT thread on f365 forum. Please don't follow me if you are under 18. You will be blocked.","links":[{"href":null,"rel":"me"}],"friendsCount":185,"followersCount":128,"listedCount":3,"statusesCount":917,"twitterTimeZone":"Casablanca","verified":false,"utcOffset":"3600","preferredUsername":"dr_acula365","languages":["en"],"location":{"objectType":"place","displayName":"Newcastle"},"favoritesCount":0},"verb":"post","postedTime":"2015-04-13T16:10:25.000Z","generator":{"displayName":"Tweetbot for iΟS","link":"http://tapbots.com/tweetbot"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/dr_acula365/statuses/587649047697215488","body":"@Tesco the store has no idea where my order will be and suggests I go back tomorrow as it \"might\" be with them by then.","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649047697215488","summary":"@Tesco the store has no idea where my order will be and suggests I go back tomorrow as it \"might\" be with them by then.","link":"http://twitter.com/dr_acula365/statuses/587649047697215488","postedTime":"2015-04-13T16:10:25.000Z"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"Tesco","name":"Tesco","id":271986064,"id_str":"271986064","indices":[0,6]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}
+{"id":"tag:search.twitter.com,2005:587649057914490880","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:509474447","link":"http://www.twitter.com/DebbieMeldrum","displayName":"Debstweeted","postedTime":"2012-03-01T00:24:28.000Z","image":"https://pbs.twimg.com/profile_images/483673950602682368/dsKmu4AK_normal.jpeg","summary":"Professional. Linguist. Fashion follower. Juice addict. Day dreamer. Number 11.","links":[{"href":null,"rel":"me"}],"friendsCount":677,"followersCount":132,"listedCount":3,"statusesCount":323,"twitterTimeZone":null,"verified":false,"utcOffset":null,"preferredUsername":"DebbieMeldrum","languages":["en"],"location":{"objectType":"place","displayName":"Aberdeen"},"favoritesCount":337},"verb":"post","postedTime":"2015-04-13T16:10:28.000Z","generator":{"displayName":"Twitter for iPad","link":"http://twitter.com/#!/download/ipad"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/DebbieMeldrum/statuses/587649057914490880","body":"@Topshop really want to but unfortunately my local Topshop in #Aberdeen doesn't stock it 😟","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649057914490880","summary":"@Topshop really want to but unfortunately my local Topshop in #Aberdeen doesn't stock it 😟","link":"http://twitter.com/DebbieMeldrum/statuses/587649057914490880","postedTime":"2015-04-13T16:10:28.000Z"},"inReplyTo":{"link":"http://twitter.com/Topshop/statuses/587648638467366912"},"favoritesCount":0,"location":{"objectType":"place","displayName":"Aberdeen, Scotland","name":"Aberdeen","country_code":"United Kingdom","twitter_country_code":"GB","link":"https://api.twitter.com/1.1/geo/id/73cc26d418860ddd.json","geo":{"type":"Polygon","coordinates":[[[-2.1899763,57.107695],[-2.1899763,57.2134174],[-2.0641046,57.2134174],[-2.0641046,57.107695]]]}},"geo":{"type":"Point","coordinates":[57.170311,-2.094876]},"twitter_entities":{"hashtags":[{"text":"Aberdeen","indices":[62,71]}],"trends":[],"urls":[],"user_mentions":[{"screen_name":"Topshop","name":"Topshop","id":20908109,"id_str":"20908109","indices":[0,8]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"}],"language":{"value":"en"}}}
+{"id":"tag:search.twitter.com,2005:587649092089733120","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:274108611","link":"http://www.twitter.com/IAmKatieDevs","displayName":"Katie Devlin","postedTime":"2011-03-29T19:02:59.000Z","image":"https://pbs.twimg.com/profile_images/454566612080033793/du2a4BhY_normal.jpeg","summary":"Boston bred, fashion fanatic, music enthusiast, IBMer. {{MY tweets are MY own}}","links":[{"href":null,"rel":"me"}],"friendsCount":382,"followersCount":234,"listedCount":10,"statusesCount":1673,"twitterTimeZone":"Quito","verified":false,"utcOffset":"-18000","preferredUsername":"IAmKatieDevs","languages":["en"],"location":{"objectType":"place","displayName":"Boston, MA"},"favoritesCount":263},"verb":"post","postedTime":"2015-04-13T16:10:36.000Z","generator":{"displayName":"Hootsuite","link":"http://www.hootsuite.com"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/IAmKatieDevs/statuses/587649092089733120","body":"@BedBathBeyond marries wedding registry with augmented reality http://t.co/0bojF4HY12 #mobile #mcommerce","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649092089733120","summary":"@BedBathBeyond marries wedding registry with augmented reality http://t.co/0bojF4HY12 #mobile #mcommerce","link":"http://twitter.com/IAmKatieDevs/statuses/587649092089733120","postedTime":"2015-04-13T16:10:36.000Z"},"favoritesCount":0,"twitter_entities":{"hashtags":[{"text":"mobile","indices":[86,93]},{"text":"mcommerce","indices":[94,104]}],"trends":[],"urls":[{"url":"http://t.co/0bojF4HY12","expanded_url":"http://www.mobilecommercedaily.com/bed-bath-and-beyond-marries-wedding-registry-with-augmented-reality","display_url":"mobilecommercedaily.com/bed-bath-and-b…","indices":[63,85]}],"user_mentions":[{"screen_name":"BedBathBeyond","name":"Bed Bath & Beyond","id":372817561,"id_str":"372817561","indices":[0,14]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"}],"urls":[{"url":"http://t.co/0bojF4HY12","expanded_url":"http://www.mobilecommercedaily.com/bed-bath-and-beyond-marries-wedding-registry-with-augmented-reality","expanded_status":200}],"language":{"value":"en"}}}
+{"id":"tag:search.twitter.com,2005:587649207844081664","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:944015658","link":"http://www.twitter.com/dianekendall716","displayName":"diane kendall","postedTime":"2012-11-12T16:59:18.000Z","image":"https://pbs.twimg.com/profile_images/534077775309975553/b0Z9SWCF_normal.jpeg","summary":null,"links":[{"href":null,"rel":"me"}],"friendsCount":24,"followersCount":10,"listedCount":0,"statusesCount":34,"twitterTimeZone":null,"verified":false,"utcOffset":null,"preferredUsername":"dianekendall716","languages":["en"],"favoritesCount":63},"verb":"post","postedTime":"2015-04-13T16:11:03.000Z","generator":{"displayName":"Mobile Web (M5)","link":"https://mobile.twitter.com"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/dianekendall716/statuses/587649207844081664","body":"@marksandspencer black please","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649207844081664","summary":"@marksandspencer black please","link":"http://twitter.com/dianekendall716/statuses/587649207844081664","postedTime":"2015-04-13T16:11:03.000Z"},"inReplyTo":{"link":"http://twitter.com/marksandspencer/statuses/587541076187090944"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"marksandspencer","name":"M&S","id":37916167,"id_str":"37916167","indices":[0,16]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"},{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}
+{"id":"tag:search.twitter.com,2005:587649264156811264","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:920532930","link":"http://www.twitter.com/karengodfrey06","displayName":"Karen Godfrey-Young","postedTime":"2012-11-02T07:26:39.000Z","image":"https://pbs.twimg.com/profile_images/579378574119051264/NutpKf5S_normal.jpg","summary":"Wife/ Mother / Grandma","links":[{"href":null,"rel":"me"}],"friendsCount":89,"followersCount":61,"listedCount":0,"statusesCount":1771,"twitterTimeZone":null,"verified":false,"utcOffset":null,"preferredUsername":"karengodfrey06","languages":["en"],"location":{"objectType":"place","displayName":" Oxfordshire"},"favoritesCount":1383},"verb":"post","postedTime":"2015-04-13T16:11:17.000Z","generator":{"displayName":"Twitter for Android","link":"http://twitter.com/download/android"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/karengodfrey06/statuses/587649264156811264","body":"@AldiUK nothing is happening. Opens to blank white page? \nI  shop with you every week so would love to enter comp xx","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649264156811264","summary":"@AldiUK nothing is happening. Opens to blank white page? \nI  shop with you every week so would love to enter comp xx","link":"http://twitter.com/karengodfrey06/statuses/587649264156811264","postedTime":"2015-04-13T16:11:17.000Z"},"inReplyTo":{"link":"http://twitter.com/AldiUK/statuses/587623827787493376"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"AldiUK","name":"Aldi Stores UK","id":28596803,"id_str":"28596803","indices":[0,7]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datasift
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - DataSift
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-05 00:00:00.000000000 Z
+date: 2015-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -18,14 +18,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -77,6 +77,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
 - ".travis.yml"
 - ".yardopts"
 - CHANGELOG.md
@@ -95,6 +97,7 @@ files:
 - examples/core_api_eg.rb
 - examples/historics_eg.rb
 - examples/historics_preview_eg.rb
+- examples/ingestion/twitter_gnip_batch_eg.rb
 - examples/live_stream_eg.rb
 - examples/managed_source_fb_eg.rb
 - examples/managed_source_ig_eg.rb
@@ -115,10 +118,12 @@ files:
 - lib/managed_source.rb
 - lib/managed_source_auth.rb
 - lib/managed_source_resource.rb
+- lib/odp.rb
 - lib/push.rb
 - lib/pylon.rb
 - test/datasift/core_api_test.rb
 - test/datasift/historics_preview_api_test.rb
+- test/datasift/odp/batch_upload_test.rb
 - test/datasift/push_api_test.rb
 - test/fixtures/cassettes/core/after_historic_dpu.json
 - test/fixtures/cassettes/core/balance_get.json
@@ -132,6 +137,10 @@ files:
 - test/fixtures/cassettes/core/validate_invalid_hash.json
 - test/fixtures/cassettes/core/validate_success_bool.json
 - test/fixtures/cassettes/core/validate_success_hash.json
+- test/fixtures/cassettes/odp/batch/after_upload.json
+- test/fixtures/cassettes/odp/batch/before_upload.json
+- test/fixtures/cassettes/odp/batch/upload_failure_no_source.json
+- test/fixtures/cassettes/odp/batch/upload_success.json
 - test/fixtures/cassettes/preview/before_preview_create.json
 - test/fixtures/cassettes/preview/before_preview_get.json
 - test/fixtures/cassettes/preview/preview_create_success.json
@@ -160,6 +169,7 @@ files:
 - test/fixtures/cassettes/push/push_stop.json
 - test/fixtures/cassettes/push/push_update.json
 - test/fixtures/cassettes/push/push_validate.json
+- test/fixtures/data/fake_gnip_tweets.json
 - test/test_helper.rb
 homepage: https://github.com/datasift/datasift-ruby
 licenses:
@@ -181,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.5
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: DataSift is a simple wrapper for the DataSift API.