datasift 1.1.0 → 1.2.0

data/.gitignore

@@ -0,0 +1 @@
+sftp-config.json

data/README.md

@@ -1,7 +1,8 @@
 DataSift
 ========
 
-The official Ruby library for accessing the DataSift API. See http://datasift.net for full details and to sign up for an account.
+The official Ruby library for accessing the DataSift API. See
+http://datasift.net for full details and to sign up for an account.
 
 The examples and tests use the username and API key in config.yml.
 
@@ -15,12 +16,13 @@ Dependencies
 
 If you're using the source you'll need to install the dependencies.
 
-sudo gem install yajl-ruby json rest-client
+sudo gem install yajl-ruby rest-client
 
 Simple example
 --------------
 
-This example looks for anything that contains the word "datasift" and simply prints the content to the screen as they come in.
+This example looks for anything that contains the word "datasift" and simply
+prints the content to the screen as they come in.
 
 ```ruby
 require 'rubygems'
@@ -35,11 +37,27 @@ consumer.consume(true) do |interaction|
 end
 ```
 
-See the DataSift documentation for full details of the data contained within each interaction. See this page on our developer site for an example tweet: http://dev.datasift.com/docs/targets/twitter/twitter-output-format
+See the DataSift documentation for full details of the data contained within
+each interaction. See this page on our developer site for an example tweet:
+http://dev.datasift.com/docs/targets/twitter/twitter-output-format
 
 License
 -------
 
 All code contained in this repository is Copyright 2011-2012 MediaSift Ltd.
 
-This code is released under the BSD license. Please see the LICENSE file for more details.
+This code is released under the BSD license. Please see the LICENSE file for
+more details.
+
+Changelog
+---------
+
+* v.1.2.0 Twitter Compliance (2012-02-28)
+
+  The consumer now has an onDeleted method to which you can assign a block
+  that will be called to handle DELETE requests from Twitter. See delete.rb
+  in the examples folder for a sample implementation.
+  (@see http://dev.datasift.com/docs/twitter-deletes)
+
+  NB: if you are storing tweets you must implement this method in your code
+  and take appropriate action to maintain compliance with the Twitter license.

data/VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0

data/datasift.gemspec

@@ -16,7 +16,6 @@ Gem::Specification.new do |s|
 	s.required_rubygems_version	= Gem::Requirement.new(">= 1.3.6") if s.respond_to? :required_rubygems_version=
 
 	s.add_runtime_dependency('rest-client', '~> 1.6.3')
-	s.add_runtime_dependency('json', '~> 1')
 	s.add_runtime_dependency('yajl-ruby', '~> 0.8.2')
 	s.add_development_dependency('rdoc', '~> 0')
 	s.add_development_dependency('shoulda', '~> 2.11.3')

data/examples/deletes.rb

@@ -0,0 +1,52 @@
+# This example consumes 1% of tweets, displaying a . for each interaction
+# received, and an X for each delete notification.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the DataSift library
+require File.dirname(__FILE__) + '/../lib/datasift'
+
+# Include the configuration - put your username and API key in this file
+require 'yaml'
+config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
+
+# Authenticate
+puts 'Creating user...'
+user = DataSift::User.new(config['username'], config['api_key'])
+
+# Create the definition
+csdl = 'interaction.type == "twitter" AND interaction.sample < 1.0'
+puts 'Creating definition...'
+puts '  ' + csdl
+definition = user.createDefinition(csdl)
+
+# Create the consumer
+puts 'Getting the consumer...'
+consumer = definition.getConsumer(DataSift::StreamConsumer::TYPE_HTTP)
+
+# Set up the delete event handler. Refer to the documentation for details of
+# what the interaction variable will contain:
+# http://dev.datasift.com/docs/twitter-deletes
+consumer.onDeleted do |interaction|
+	print 'X'
+	$stdout.flush
+end
+
+# And start consuming
+puts 'Consuming...'
+puts '--'
+consumer.consume(true) do |interaction|
+	if interaction
+		print '.'
+		$stdout.flush
+	end
+end
+
+# This example will not stop unless it gets disconnected
+puts
+puts 'Consumer stopped'
+puts

data/lib/DataSift/apiclient.rb

@@ -9,7 +9,7 @@
 # DataSift API.
 
 require 'rest_client'
-require 'json'
+require 'yajl'
 
 module DataSift
 	# ApiCLient class.
@@ -29,7 +29,7 @@ module DataSift
 		# * +api_key+ - The API key for the Auth header
 		def call(username, api_key, endpoint, params = {}, user_agent = 'DataSiftPHP/0.0')
 			# Build the full endpoint URL
-			url = 'http://' + User::API_BASE_URL + endpoint + '.json?' + hashToQuerystring(params)
+			url = 'http://' + User::API_BASE_URL + endpoint
 
 			retval = {
 				'response_code' => 500,
@@ -40,14 +40,13 @@ module DataSift
 
 			begin
 				# Make the call
-				res = RestClient.get(url, { 'Auth' => username + ':' + api_key, 'User-Agent' => user_agent })
+				res = RestClient.post(url, params, { 'Auth' => username + ':' + api_key, 'User-Agent' => user_agent })
 
 				# Success
 				retval['response_code'] = 200
 
 				# Parse the JSON response
-				retval['data'] = JSON.parse(res)
-
+				retval['data'] = Yajl::Parser.parse(res)
 				# Rate limit headers
 				if (res.headers[:x_ratelimit_limit])
 					retval['rate_limit'] = res.headers[:x_ratelimit_limit]
@@ -61,7 +60,7 @@ module DataSift
 				retval['response_code'] = err.http_code
 
 				# And set the data
-				retval['data'] = JSON.parse(err.response)
+				retval['data'] = Yajl::Parser.parse(err.response)
 			end
 
 			retval

data/lib/DataSift/stream_consumer.rb

@@ -61,6 +61,20 @@ module DataSift
 			@definition.hash
 		end
 
+		# This is called when a deletion notification is received.
+		# === Parameters
+		#
+		# * +interaction+ - Minimal details about the interaction that was deleted.
+		#
+		def onDeleted(&block)
+			if block_given?
+				@on_deleted = block
+				self
+			else
+				@on_deleted
+			end
+		end
+
 		# This is called when the consumer is stopped.
 		# === Parameters
 		#
@@ -88,7 +102,13 @@ module DataSift
 
 			# Start consuming
 			@state = STATE_STARTING
-			onStart(&block)
+			onStart do |interaction|
+				if interaction.has_key?('deleted') and interaction['deleted']
+					onDeleted.call(interaction) unless onDeleted.nil?
+				else
+					block.call(interaction) unless block.nil?
+				end
+			end
 		end
 
 		# Called when the consumer should start consuming the stream.

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: datasift
 version: !ruby/object:Gem::Version 
-  prerelease: false
+  hash: 31
+  prerelease: 
   segments: 
   - 1
-  - 1
+  - 2
   - 0
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors: 
 - DataSift
@@ -14,16 +15,17 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-24 00:00:00 +00:00
-default_executable: 
+date: 2012-02-29 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rest-client
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
+        hash: 9
         segments: 
         - 1
         - 6
@@ -31,72 +33,68 @@ dependencies:
         version: 1.6.3
   type: :runtime
   version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: json
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        version: "1"
-  type: :runtime
-  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: yajl-ruby
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
+        hash: 59
         segments: 
         - 0
         - 8
         - 2
         version: 0.8.2
   type: :runtime
-  version_requirements: *id003
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: rdoc
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
   type: :development
-  version_requirements: *id004
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: shoulda
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
+        hash: 37
         segments: 
         - 2
         - 11
         - 3
         version: 2.11.3
   type: :development
-  version_requirements: *id005
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: rspec
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
+        hash: 23
         segments: 
         - 2
         - 6
         - 0
         version: 2.6.0
   type: :development
-  version_requirements: *id006
+  version_requirements: *id005
 description: The official Ruby library for accessing the DataSift API. See http://datasift.com/ for full details and to sign up for an account.
 email: 
 - support@datasift.net
@@ -107,6 +105,7 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- .gitignore
 - LICENSE
 - README.md
 - Rakefile
@@ -114,6 +113,7 @@ files:
 - config.yml
 - datasift.gemspec
 - examples/consume-stream.rb
+- examples/deletes.rb
 - examples/dpu.rb
 - examples/football-buffered.rb
 - examples/football.rb
@@ -131,7 +131,6 @@ files:
 - test/test_live_api.rb
 - test/test_user.rb
 - test/testdata.yml
-has_rdoc: true
 homepage: http://github.com/datasift/datasift-ruby
 licenses: []
 
@@ -141,16 +140,20 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 23
       segments: 
       - 1
       - 3
@@ -159,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: DataSift is a simple wrapper for the DataSift API.