datasift 0.2.0

data/LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2011 MediaSift Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+

data/README.md

@@ -0,0 +1,38 @@
+DataSift
+========
+
+The official Ruby library for accessing the DataSift API. See http://datasift.net for full details and to sign up for an account.
+
+The examples and tests use the username and API key in config.yml.
+
+Install Instructions
+--------------------
+
+sudo gem install datasift
+
+Simple example
+--------------
+
+This example looks for anything that contains the word "datasift" and simply prints the content to the screen as they come in.
+
+```ruby
+require 'rubygems'
+require 'datasift'
+user = DataSift::User.new("your username", "your api_key")
+definition = user.createDefinition('interaction.content contains "football"')
+consumer = definition.getConsumer(DataSift::StreamConsumer::TYPE_HTTP)
+consumer.consume(true) do |interaction|
+	if interaction
+		puts interaction['interaction']['content']
+	end
+end
+```
+
+See the DataSift documentation for full details of the data contained within each interaction: http://support.datasift.net/help/kb/rest-api/return-objects
+
+License
+-------
+
+All code contained in this repository is Copyright 2011 MediaSift Ltd.
+
+This code is released under the BSD license. Please see the LICENSE file for more details.

data/Rakefile

@@ -0,0 +1,36 @@
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+Rake::TestTask.new(:test) do |test|
+	test.libs << 'lib' << 'test'
+	test.pattern = 'test/**/test_*.rb'
+	test.verbose = true
+end
+
+begin
+	require 'rcov/rcovtask'
+	Rcov::RcovTask.new do |test|
+		test.libs << 'test'
+		test.pattern = 'test/**/test_*.rb'
+		test.verbose = true
+	end
+rescue LoadError
+	task :rcov do
+		abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+	end
+end
+
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
+	version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+	rdoc.rdoc_dir = 'rdoc'
+	rdoc.title = "datasift #{version}"
+	rdoc.rdoc_files.include('README*')
+	rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/config.yml

@@ -0,0 +1,2 @@
+username: YOUR_USERNAME_HERE
+api_key: YOUR_API_KEY_HERE

data/datasift.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+	s.name				= 'datasift'
+	s.version			= File.open('VERSION').first
+
+	s.authors			= ['MediaSift']
+	s.email				= ['support@datasift.net']
+	s.description = %q{The official Ruby library for accessing the DataSift API. See http://datasift.net/ for full details and to sign up for an account.}
+	s.summary			= %q{DataSit is a simple wrapper for the DataSift API.}
+	s.homepage		= 'http://github.com/mediasift/datasift-ruby'
+
+	s.platform									= Gem::Platform::RUBY
+	s.rubygems_version					= %q{1.3.6}
+	s.required_rubygems_version = Gem::Requirement.new(">= 1.3.6") if s.respond_to? :required_rubygems_version=
+
+	s.add_runtime_dependency('rest-client', '~> 1.6.3')
+	s.add_runtime_dependency('crack', '~> 0')
+	s.add_runtime_dependency('yajl-ruby', '~> 0.8.2')
+	s.add_development_dependency('rdoc', '~> 0')
+	s.add_development_dependency('shoulda', '~> 2.11.3')
+	s.add_development_dependency('rspec', '~> 2.6.0')
+
+	s.files	        = `git ls-files`.split("\n")
+	s.test_files		= `git ls-files -- {test,spec,features}/*`.split("\n")
+	s.require_paths = ["lib"]
+end

data/examples/cost.rb

@@ -0,0 +1,123 @@
+# This example gets the cost associated with the stream given on the command
+# line or piped/typed into STDIN. It  presents it in a nice ASCII table.]
+# Note that the CSDL must be enclosed in quotes if given on the command line.
+#
+# ruby cost.rb 'interaction.content contains "football"'
+#  or
+# cat football.csdl | ruby cost.rb
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the DataSift library
+require File.dirname(__FILE__) + '/../lib/datasift'
+
+# Function to format a number with commas
+def number_with_delimiter(number, delimiter=',')
+  number.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
+end
+
+# Include the configuration - put your username and API key in this file
+config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
+
+# Get the CSDL from the command line or STDIN
+if ARGV.size > 0
+	csdl = ARGV[0]
+else
+	csdl = ARGF.read
+end
+
+# Authenticate
+puts 'Creating user...'
+user = DataSift::User.new(config['username'], config['api_key'])
+
+# Create the definition
+puts 'Creating definition...'
+definition = user.createDefinition(csdl)
+
+# Getting cost
+puts 'Getting cost...'
+begin
+	cost = definition.getCostBreakdown()
+rescue DataSift::CompileFailedError => e
+	puts 'CSDL compilation failed: ' + e
+	puts
+	exit!
+end
+
+costtable = []
+maxlength = {'target' => 'Target'.length, 'times used' => 'Times used'.length, 'complexity' => 'Complexity'.length};
+cost['costs'].each do |tgt,c|
+	maxlength['target'] = [maxlength['target'], tgt.length].max()
+	maxlength['times used'] = [maxlength['times used'], number_with_delimiter(c['count']).length].max()
+	maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(c['cost']).length].max()
+
+	costtable.push({
+			'target' => tgt,
+			'times used' => number_with_delimiter(c['count']),
+			'complexity' => number_with_delimiter(c['cost']),
+		})
+
+	c['targets'].each do |tgt2,d|
+		maxlength['target'] = [maxlength['target'], 2 + tgt2.length].max()
+		maxlength['times used'] = [maxlength['times used'], number_with_delimiter(d['count']).length].max()
+		maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(d['cost']).length].max()
+
+		costtable.push({
+				'target' => '  ' + tgt2,
+				'times used' => number_with_delimiter(d['count']),
+				'complexity' => number_with_delimiter(d['cost']),
+			})
+	end
+end
+
+maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(cost['total']).length].max()
+
+puts
+print '/-' + ('-' * maxlength['target']) + '---'
+print ('-' * maxlength['times used']) + '---'
+puts ('-' * maxlength['complexity']) + '-\\'
+
+print '| ' + 'Target'.ljust(maxlength['target']) + ' | '
+print 'Times Used'.ljust(maxlength['times used']) + ' | '
+puts 'Complexity'.ljust(maxlength['complexity']) + ' |'
+
+print '|-' + ('-' * maxlength['target']) + '-+-'
+print ('-' * maxlength['times used']) + '-+-'
+puts ('-' * maxlength['complexity']) + '-|'
+
+costtable.each do |row|
+	print '| ' + row['target'].ljust(maxlength['target']) + ' | '
+	print row['times used'].rjust(maxlength['times used']) + ' | '
+	puts row['complexity'].rjust(maxlength['complexity']) + ' |'
+end
+
+print '|-' + ('-' * maxlength['target']) + '-+-'
+print ('-' * maxlength['times used']) + '-+-'
+puts ('-' * maxlength['complexity']) + '-|'
+
+print '| ' + 'Total'.rjust(maxlength['target'] + 3 + maxlength['times used']) + ' = '
+puts cost['total'].to_s.rjust(maxlength['complexity']) + ' |'
+
+print '\\-' + ('-' * maxlength['target']) + '---'
+print ('-' * maxlength['times used']) + '---'
+puts ('-' * maxlength['complexity']) + '-/'
+
+puts
+
+if cost['total'] > 1000
+	tiernum = 3;
+	tierdesc = 'high complexity';
+elsif cost['total'] > 100
+	tiernum = 2;
+	tierdesc = 'medium complexity';
+else
+	tiernum = 1;
+	tierdesc = 'simple complexity';
+end
+
+puts 'A total cost of ' + number_with_delimiter(cost['total']) + ' puts this stream in tier ' + tiernum.to_s + ', ' + tierdesc
+puts

data/examples/football-buffered.rb

@@ -0,0 +1,50 @@
+# This example constructs a DataSift_Definition object with CSDL that looks
+# for anything containing the word "football". It then sits in a loop,
+# getting buffered interactions once every 10 seconds until it's retrieved
+# 10.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the DataSift library
+require File.dirname(__FILE__) + '/../lib/datasift'
+
+# Include the configuration - put your username and API key in this file
+config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
+
+# Authenticate
+puts 'Creating user...'
+user = DataSift::User.new(config['username'], config['api_key'])
+
+# Create the definition
+csdl = 'interaction.content contains "football"'
+puts 'Creating definition...'
+puts '  ' + csdl
+definition = user.createDefinition(csdl)
+
+# Get buffered interactions until we've had 10
+puts 'Getting buffered interactions...'
+puts '--'
+num = 10
+from_id = false
+begin
+	interactions = definition.getBuffered(num, from_id)
+	interactions.each do |interaction|
+		puts 'Type: ' + interaction['interaction']['type']
+		puts 'Content: ' + interaction['interaction']['content']
+		puts '--'
+		num -= 1
+		from_id = interaction['interaction']['id']
+	end
+
+	if num > 0
+		sleep(10)
+	end
+end while num > 0
+
+puts
+puts 'Fetched 10 interactions, we\'re done.'
+puts

data/examples/football.rb

@@ -0,0 +1,52 @@
+# This example constructs a DataSift_Definition object with CSDL that looks
+# for anything containing the word "football". It then gets an HTTP
+# consumer for that definition and displays matching interactions to the
+# screen as they come in. It will display 10 interactions and then stop.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the DataSift library
+require File.dirname(__FILE__) + '/../lib/datasift'
+
+# Include the configuration - put your username and API key in this file
+config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
+
+# Authenticate
+puts 'Creating user...'
+user = DataSift::User.new(config['username'], config['api_key'])
+
+# Create the definition
+csdl = 'interaction.content contains "football"'
+puts 'Creating definition...'
+puts '  ' + csdl
+definition = user.createDefinition(csdl)
+
+# Create the consumer
+puts 'Getting the consumer...'
+consumer = definition.getConsumer(DataSift::StreamConsumer::TYPE_HTTP)
+
+# And start consuming
+puts 'Consuming...'
+puts '--'
+count = 10
+consumer.consume(true) do |interaction|
+	if interaction
+		puts 'Type: ' + interaction['interaction']['type']
+		puts 'Content: ' + interaction['interaction']['content']
+		puts '--'
+
+		count -= 1
+		if count == 0
+			puts 'Stopping consumer...'
+			consumer.stop()
+		end
+	end
+end
+
+puts
+puts 'Finished consuming'
+puts

data/examples/twitter-track.rb

@@ -0,0 +1,60 @@
+# This example constructs a DataSift_Definition object with CSDL that looks
+# for anything containing the word "football". It then gets an HTTP
+# consumer for that definition and displays matching interactions to the
+# screen as they come in. It will display 10 interactions and then stop.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Make sure we have some arguments
+if ARGV.size == 0
+	puts 'ERR: Please specify the words and/or phrases to track!'
+	puts
+	puts
+	exit!
+end
+
+# Include the DataSift library
+require File.dirname(__FILE__) + '/../lib/datasift'
+
+# Include the configuration - put your username and API key in this file
+config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
+
+# Authenticate
+puts 'Creating user...'
+user = DataSift::User.new(config['username'], config['api_key'])
+
+# Create the definition
+csdl = 'interaction.type == "twitter" and (interaction.content contains "' + ARGV.join('" or interaction.content contains "') + '")'
+puts 'Creating definition...'
+puts '  ' + csdl
+definition = user.createDefinition(csdl)
+
+# Create the consumer
+puts 'Getting the consumer...'
+consumer = definition.getConsumer(DataSift::StreamConsumer::TYPE_HTTP)
+
+# Setting up the onStopped handler
+consumer.onStopped do |reason|
+	puts
+	puts 'Stopped: ' + reason
+	puts
+end
+
+# And start consuming
+puts 'Consuming...'
+puts '--'
+consumer.consume(true) do |interaction|
+	if interaction
+		puts 'Type: ' + interaction['interaction']['type']
+		puts 'Content: ' + interaction['interaction']['content']
+		puts '--'
+	end
+end
+
+puts
+puts 'Finished consuming'
+puts

data/lib/DataSift/apiclient.rb

@@ -0,0 +1,79 @@
+#
+# apiclient.rb - This file contains the ApiClient class.
+#
+# Copyright (C) 2011 MediaSift Ltd
+#
+# == Overview
+#
+# The ApiClient class wraps the functionality that makes calls to the
+# DataSift API.
+
+require 'rest_client'
+require 'crack'
+
+module DataSift
+	# ApiCLient class.
+	#
+	# == Introduction
+	#
+	# The ApiClient class wraps the functionality that makes calls to the
+	# DataSift API.
+	#
+	class ApiClient
+		# Make a call to a DataSift API endpoint.
+		# === Parameters
+		#
+		# * +endpoint+ - The endpoint of the API call.
+		# * +params+ - The parameters to be passed along with the request.
+		# * +username+ - The username for the Auth header
+		# * +api_key+ - The API key for the Auth header
+		def call(username, api_key, endpoint, params = {}, user_agent = 'DataSiftPHP/0.0')
+			# Build the full endpoint URL
+			url = 'http://' + User::API_BASE_URL + endpoint + '.json?' + hashToQuerystring(params)
+
+			retval = {
+				'response_code' => 500,
+				'data' => { 'error' => 'Unknown error' },
+				'rate_limit' => -1,
+				'rate_limit_remaining' => -1,
+			}
+
+			begin
+				# Make the call
+				res = RestClient.get(url, { 'Auth' => username + ':' + api_key, 'User-Agent' => user_agent })
+
+				# Success
+				retval['response_code'] = 200
+
+				# Parse the JSON response
+				retval['data'] = Crack::JSON.parse(res)
+
+				# Rate limit headers
+				if (res.headers[:x_ratelimit_limit])
+					retval['rate_limit'] = res.headers[:x_ratelimit_limit]
+				end
+
+				if (res.headers[:x_ratelimit_remaining])
+					retval['rate_limit_remaining'] = res.headers[:x_ratelimit_remaining]
+				end
+			rescue RestClient::ExceptionWithResponse => err
+				# Set the response code
+				retval['response_code'] = err.http_code
+
+				# And set the data
+				retval['data'] = Crack::JSON.parse(err.response)
+			end
+
+			retval
+		end
+
+	private
+
+		def hashToQuerystring(hash)
+			hash.keys.inject('') do |query_string, key|
+				query_string << '&' unless key == hash.keys.first
+				query_string << "#{URI.encode(key.to_s)}=#{URI.encode(hash[key].to_s)}"
+			end
+		end
+	end
+end