datasift 2.1.1 → 3.0.0.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93c090e18483741e93ccb9a66e0eb36950012ac7
-  data.tar.gz: 8357792052e374f5e73953b5d55993582c031ed7
+  metadata.gz: 2edfce8b73a255a0b9cce1bcf3d8834bf1e0b028
+  data.tar.gz: 4e34361963da7c4c5e06ebd2b7bf6d13c80b04fa
 SHA512:
-  metadata.gz: f356d57e9d9d37dbcbf5122ad2cacf9d67638703f6560d228e70c3eb5aaa442d64331976672c335f2b7888c6da967808b0dae4cedf3d31639eb700571aab1331
-  data.tar.gz: a95b4318c787396045e1db54c7a6bd0bfe6f74e87ac1ca9f1a4f944565b537fda4a5430eb97b3b9cbbe2ab4879bb850d358d98cc91077998462d026ed310c34b
+  metadata.gz: b9b2ef5c9c081a46ea7dc343dc33faad8e0cba822ef84f7abb7ba755221dc6e334f02ee08e915b6bf059eb678e73004184b17de39c939814285bb33b96febc41
+  data.tar.gz: 7cfe773014a1b0c719501f8b7b0160606dde21525bb2cb03d41d551837e4020f7f9aa64bca1d660ebbdbf0119d14c23462dc93b09b0913da3d0d2ce046167263

data/.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
 sftp-config.json
+*.iml
 .idea/
 rdoc/

data/CHANGELOG.md

@@ -0,0 +1,100 @@
+CHANGELOG
+================================
+
+v.3.0.0.beta (2013-11-07)
+-------------------------
+
+Total rewrite of the DataSift client library for Ruby.
+
+This release does break backwards compatibility with previous versions. New features include:
+* Support for DataSift Historics Preview
+* Support for multi-streaming via WebSockets
+
+v.2.1.0 (2013-09-13)
+--------------------
+
+Final fixes for v.2.1.0
+
+* Adds /source/log call
+* Updated /source/create call to comply with latest version of the API
+* Removed JSON gem - now dependant on Yajl-Ruby for JSON parsing
+* Added license info to gemspec
+* Fixed edge case when passing refresh token to authentication with G+ API
+* Fixed /source/get call
+
+v.2.1.0.beta (2013-08-05)
+-------------------------
+
+Adding Managed Sources support.
+Thanks to [giovannelli](https://github.com/giovannelli) for the contribution
+
+v.2.0.4 (2013-03-18)
+--------------------
+
+Bug fix to handle HTTP 202 response codes
+
+v.2.0.3 (2013-03-04)
+--------------------
+
+Stability improvement and bug fix
+
+* Removed references to deprecated Historic output field 'volume_info'.
+* Added 65s timeout on live streaming to handle 'silent' server disconnects.
+* Minor changes to ensure Ruby 2.0 compatibility.
+
+v.2.0.2 (2012-12-03)
+--------------------
+
+Added missing Historic sample size into historic/prepare requests
+
+v.2.0.1 (2012-09-03)
+--------------------
+
+Fixed a bug that was preventing streaming connections from being established
+
+v.2.0.0 (2012-08-31)
+--------------------
+
+Added support for Historics queries and Push delivery
+
+v.1.5.0 (2012-05-24)
+--------------------
+
+Added getBalance to the User class [joncooper](https://github.com/joncooper)
+
+v.1.4.1 (2012-05-15)
+--------------------
+
+Fixed a minor bug in the SSL support
+
+v.1.4.0 (2012-05-15)
+--------------------
+
+Added SSL support
+
+This is enabled by default and can be disabled by passing false as the third
+parameter to the User constructor, or calling enableSSL(false) on the User
+object.
+
+v.1.3.1 (2012-04-20)
+--------------------
+
+Exposed compile failures when getting the stream hash
+
+v.1.3.0 (2012-03-08)
+--------------------
+
+Improved error handling
+
+* Added onError and onWarning events - see examples/consume-stream.rb for an example.
+* Stopped the HTTP consumer from attempting to reconnect when it receives a 4xx response from the server.
+
+v.1.2.0 (2012-02-28)
+--------------------
+
+Twitter Compliance
+
+* The consumer now has an onDeleted method to which you can assign a block that will be called to handle DELETE requests from Twitter. See delete.rb in the examples folder for a sample implementation. (@see http://dev.datasift.com/docs/twitter-deletes)
+
+NB: if you are storing tweets you must implement this method in your code
+and take appropriate action to maintain compliance with the Twitter license.

data/Gemfile.lock

@@ -0,0 +1,32 @@
+PATH
+  remote: .
+  specs:
+    datasift (3.0.0.beta)
+      rest-client (~> 1.6.3)
+      websocket-td (~> 0.0.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    json (1.8.0)
+    mime-types (1.25)
+    multi_json (1.8.0)
+    rdoc (4.0.1)
+      json (~> 1.4)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    shoulda (2.11.3)
+    test-unit (2.5.5)
+    websocket (1.1.1)
+    websocket-td (0.0.3)
+      websocket (~> 1.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  datasift!
+  multi_json (~> 1.8.0)
+  rdoc (> 0)
+  shoulda (~> 2.11.3)
+  test-unit (>= 2.5.5)

data/README.md

@@ -4,8 +4,6 @@ DataSift
 The official Ruby library for accessing the DataSift API. See
 http://datasift.com/ for full details and to sign up for an account.
 
-The examples use the username and API key in config.yml unless otherwise noted.
-
 Install Instructions
 --------------------
 
@@ -16,34 +14,56 @@ Dependencies
 
 If you're using the source you'll need to install the dependencies.
 
-sudo gem install yajl-ruby rest-client
+sudo gem install rest-client multi_json websocket-td
 
 The library will use SSL connections by default. While we recommend using SSL
-you may disable it if required by passing false as the third parameter when
-creating a user, or by calling user.enableSSL(false) on the user object.
+you may disable it if required by passing ':enable_ssl => false' as the third
+parameter when creating your @config object.
 
 Simple example
 --------------
 
-This example looks for anything that contains the word "datasift" and simply
-prints the content to the screen as they come in.
+This example looks for anything that contains the word "football" in real-time,
+and simply prints the content to the screen as they come in.
 
 ```ruby
-require 'rubygems'
 require 'datasift'
-user = DataSift::User.new("your username", "your api_key")
-definition = user.createDefinition('interaction.content contains "football"')
-consumer = definition.getConsumer(DataSift::StreamConsumer::TYPE_HTTP)
-consumer.consume(true) do |interaction|
-	if interaction
-		puts interaction['interaction']['content']
-	end
+@config = {:username => 'DATASIFT_USERNAME', :api_key => 'DATASIFT_API_KEY', :enable_ssl => true}
+@datasift = DataSift::Client.new(@config)
+csdl = 'interaction.content contains "football"'
+filter = @datasift.compile csdl
+receivedCount = 0
+
+on_delete = lambda { |stream, m| puts 'We must delete this to be compliant ==> ' + m }
+on_error = lambda { |stream, e| puts "A serious error has occurred: #{e.message}" }
+on_message = lambda do |message, stream, hash|
+  receivedCount += 1
+  puts "Received interaction: #{message}"
+
+  if receivedCount >= 5
+    puts "Unsubscribing from hash #{hash}"
+    stream.unsubscribe hash
+  end
+end
+
+on_connect = lambda do |stream|
+  stream.subscribe(filter[:data][:hash], on_message)
+  puts 'Subscribed to '+ filter[:data][:hash]
 end
+
+on_datasift_message = lambda do |stream, message, hash|
+  #not all messages have a hash
+  puts "is_success =  #{message[:is_success]}, is_failure =  #{message[:is_failure]}, is_warning =  #{message[:is_warning]}, is_tick =  #{message[:is_tick]}"
+  puts "DataSift Message #{hash} ==> #{message}"
+end
+
+conn = DataSift::new_stream(@config, on_delete, on_error, on_connect)
+conn.on_datasift_message = on_datasift_message
+conn.stream.read_thread.join
 ```
 
-See the DataSift documentation for full details of the data contained within
-each interaction. See this page on our developer site for an example tweet:
-http://dev.datasift.com/docs/targets/twitter/tweet-output-format
+See the [Understanding the Output Data](http://dev.datasift.com/docs/getting-started/data) page on the DataSift Developer site for
+full details of the data contained within each interaction.
 
 License
 -------
@@ -52,64 +72,3 @@ All code contained in this repository is Copyright 2011-2013 MediaSift Ltd.
 
 This code is released under the BSD license. Please see the LICENSE file for
 more details.
-
-Changelog
----------
-* v.2.1.1 (2014-05-16)
-  - Allow use of more recent versions of required gems
-  - Minor bug fixes
-
-* v.2.1.0 Final fixes for v.2.1.0  (2013-09-13)
-
-  - Adds /source/log call
-  - Updated /source/create call to comply with latest version of the API
-  - Removed JSON gem - now dependant on Yajl-Ruby for JSON parsing
-  - Added license info to gemspec
-  - Fixed edge case when passing refresh token to authentication with G+ API
-  - Fixed /source/get call
-
-* v.2.1.0.beta Adding Managed Sources support. Thanks to [giovannelli](https://github.com/giovannelli) for the contribution (2013-08-05)
-
-* v.2.0.4 Bug fix to handle HTTP 202 response codes (2013-03-18)
-
-* v.2.0.3 Stability improvement and bug fix (2013-03-04)
-
-  Removed references to deprecated Historic output field 'volume_info'.
-  Added 65s timeout on live streaming to handle 'silent' server disconnects.
-  Minor changes to ensure Ruby 2.0 compatibility.
-
-* v.2.0.2 Added missing Historic sample size into historic/prepare requests (2012-12-03)
-
-* v.2.0.1 Fixed a bug that was preventing streaming connections from being established (2012-09-03)
-
-* v.2.0.0 Added support for Historics queries and Push delivery (2012-08-31)
-
-* v.1.5.0 Added getBalance to the User class [joncooper](https://github.com/joncooper) (2012-05-24)
-
-* v.1.4.1 Fixed a minor bug in the SSL support (2012-05-15)
-
-* v.1.4.0 Added SSL support (2012-05-15)
-
-  - This is enabled by default and can be disabled by passing false as the third
-  parameter to the User constructor, or calling enableSSL(false) on the User
-  object.
-
-* v.1.3.1 Exposed compile failures when getting the stream hash (2012-04-20)
-
-* v.1.3.0 Improved error handling (2012-03-08)
-
-  - Added onError and onWarning events - see examples/consume-stream.rb for an
-  example.
-
-  - Stopped the HTTP consumer from attempting to reconnect when it receives a
-  4xx response from the server.
-
-* v.1.2.0 Twitter Compliance (2012-02-28)
-
-  - The consumer now has an onDeleted method to which you can assign a block
-  that will be called to handle DELETE requests from Twitter. See delete.rb
-  in the examples folder for a sample implementation.
-  (@see http://dev.datasift.com/docs/twitter-deletes)
-
-  - NB: if you are storing tweets you must implement this method in your code
-  and take appropriate action to maintain compliance with the Twitter license.

data/VERSION

@@ -1 +1 @@
-2.1.1
+3.0.0.beta

data/datasift.gemspec

@@ -1,28 +1,25 @@
-# -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-
 Gem::Specification.new do |s|
-	s.name    = 'datasift'
-	s.version = File.open('VERSION').first
-
-	s.authors     = ['DataSift']
-	s.email       = ['support@datasift.com']
-	s.description = %q{The official Ruby library for accessing the DataSift API. See http://datasift.com/ for full details and to sign up for an account.}
-	s.summary     = %q{DataSift is a simple wrapper for the DataSift API.}
-	s.homepage    = 'http://github.com/datasift/datasift-ruby'
-	s.license     = 'BSD'
+  s.name        = 'datasift'
+  s.version     = File.open('VERSION').first
+  s.authors     = ['DataSift']
+  s.email       = ['support@datasift.com']
+  s.description = %q{The official Ruby library for accessing the DataSift API. See http://datasift.com/ for full details and to sign up for an account.}
+  s.summary     = %q{DataSift is a simple wrapper for the DataSift API.}
+  s.homepage    = 'https://github.com/datasift/datasift-ruby'
+  s.license     = 'BSD'
 
-	s.platform                  = Gem::Platform::RUBY
-	s.rubygems_version          = %q{1.3.6}
-	s.required_rubygems_version	= Gem::Requirement.new(">= 1.3.6") if s.respond_to? :required_rubygems_version=
+  s.platform         = Gem::Platform::RUBY
+  s.rubygems_version = %q{1.3.6}
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.3.6") if s.respond_to? :required_rubygems_version=
 
-	s.add_runtime_dependency('rest-client', '~> 1.6')
-	s.add_runtime_dependency('yajl-ruby', '~> 1.1')
-	s.add_development_dependency('rdoc', '> 0')
-	s.add_development_dependency('shoulda', '>= 2.11')
-	s.add_development_dependency('rspec', '~> 2.6')
+  s.add_runtime_dependency('rest-client', '~> 1.6.3')
+  s.add_development_dependency('multi_json', '~> 1.8.0')
+  s.add_runtime_dependency('websocket-td', '~> 0.0.3')
+  s.add_development_dependency('rdoc', '> 0')
+  s.add_development_dependency('shoulda', '~> 2.11.3')
+  s.add_development_dependency('test-unit', '>= 2.5.5')
 
-	s.files         = `git ls-files`.split("\n")
-	s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-	s.require_paths = ["lib"]
-end
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ["lib"]
+end

data/examples/auth.rb

@@ -0,0 +1,44 @@
+class DataSiftExample
+  require '../lib/datasift'
+
+  def initialize
+    @username = 'DATASIFT_USERNAME'
+    @api_key  = 'DATASIFT_API_KEY'
+    @config   ={:username => @username, :api_key => @api_key, :enable_ssl => false}
+    @params   = {
+        :output_type                       => 's3',
+        'output_params.bucket'             => 'apitests',
+        'output_params.directory'          => 'ruby',
+        'output_params.acl'                => 'private',
+        'output_params.auth.access_key'    => 'AKIAIINK5C4FH75RSWNA',
+        'output_params.auth.secret_key'    => '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',
+        'output_params.delivery_frequency' => 0,
+        'output_params.max_size'           => 10485760,
+        'output_params.file_prefix'        => 'DataSift',
+    }
+    @datasift = DataSift::Client.new(@config)
+  end
+
+  attr_reader :datasift
+
+  def create_push(hash, is_historics_id = false)
+    create_params = @params.merge ({
+        #hash or playback_id can be used but not both
+        :name           => 'My awesome push subscription',
+        :initial_status => 'active', # or 'paused' or 'waiting_for_start'
+    })
+    if is_historics_id
+      create_params.merge!({:playback_id => hash})
+    else
+      create_params.merge!({:hash  => hash,
+                            #start and end are not valid for historics
+                            :start => Time.now.to_i,
+                            :end   => Time.now.to_i + 320
+                           })
+    end
+    puts 'Creating subscription'
+    subscription = @datasift.push.create create_params
+    puts 'Create push => ' + subscription.to_s
+    subscription
+  end
+end

data/examples/core_api_eg.rb

@@ -0,0 +1,46 @@
+require './auth'
+class CoreApiEg < DataSiftExample
+  def initialize
+    super
+    @datasift = DataSift::Client.new(@config)
+    run
+  end
+
+  def run
+    begin
+      csdl = 'interaction.content contains "test"'
+      # see docs at http://dev.datasift.com/docs/rest-api/validate
+      puts @datasift.valid? csdl
+
+      # http://dev.datasift.com/docs/rest-api/compile
+      stream = @datasift.compile csdl
+      puts stream[:data][:hash]
+
+      # http://dev.datasift.com/docs/rest-api/dpu
+      dpu = @datasift.dpu stream[:data][:hash]
+      puts dpu[:data][:dpu]
+
+      # http://dev.datasift.com/docs/rest-api/balance
+      balance = @datasift.balance
+      puts balance[:data]
+
+      #http://dev.datasift.com/docs/rest-api/usage
+      usage = @datasift.usage
+      puts usage[:data]
+
+        #rescue DataSiftError
+    rescue DataSiftError => dse
+      puts dse.message
+      # Then match specific one to take action - All errors thrown by the client extend DataSiftError
+      case dse
+        when ConnectionError
+          # some connection error
+        when AuthError
+        when BadRequestError
+        else
+          # do something else...
+      end
+    end
+  end
+end
+CoreApiEg.new

data/examples/historics_eg.rb

@@ -0,0 +1,50 @@
+require './auth'
+class HistoricsApi < DataSiftExample
+  def initialize
+    super
+    run_historics
+  end
+
+  def run_historics
+    begin
+      stream   = @datasift.compile 'interaction.content contains "datasift"'
+      hash     = stream[:data][:hash]
+      start    = Time.now.to_i - 7200
+      end_time = start + 3600
+
+      puts 'Check the data coverage for the last two hours'
+      puts @datasift.historics.status(start, end_time)
+
+      puts 'Preparing'
+      #prepare a historics to start from two hours ago and run for 1 hour using twitter as a data source and using only a 10% sample
+      historics = @datasift.historics.prepare(hash, start, end_time, 'My ruby historics')
+      puts historics
+
+      id = historics[:data][:id]
+
+      puts 'Creating push subscription for historics'
+      puts create_push(id, true)
+
+      puts "Starting historics #{id}"
+      puts @datasift.historics.start id
+
+      puts 'Updating historics'
+      puts @datasift.historics.update(id, 'The new name of my historics')
+
+      puts 'Get info for the historics'
+      puts @datasift.historics.get_by_id id
+
+      puts 'Getting info for all my historics'
+      puts @datasift.historics.get
+
+      puts 'Stopping historics'
+      puts @datasift.historics.stop id
+
+      puts 'Deleting historics'
+      puts @datasift.historics.delete id
+    rescue DataSiftError => dse
+      puts dse.message
+    end
+  end
+end
+HistoricsApi.new