datasift 3.0.0.beta4 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5a8e4b6cddffbd3b5608c4fe831bdf87213c772
-  data.tar.gz: e05742fa5831660e60c8395b812025d40bbdf0dd
+  metadata.gz: 92d16d17339dcc890c88002c7f23576e2a6adf3d
+  data.tar.gz: 3759590fac485d84ec8e7cddc4a6fec799972afa
 SHA512:
-  metadata.gz: 0a03568fe8b5bc5a650b8694f3763154ffd0c10fd13924221dc29cadfcc3befdd5a68b4ae325817c39fa84aea1b84946111230f3f73363d4682ae45f4e1445f9
-  data.tar.gz: f626e022353a188ba41d76d68680bacb60c92bd716921362c5c51400efbbe876ac6313b93caf7e7d9b1463bc53f4a4a339b11167ee781af34ca3cb7840b7c5a1
+  metadata.gz: c4afb552f393f501c954f0f148a5bb95ff070a52656838d1bf49a3b5188f679a0b38c8a51dde3d413c6d2ab3fc8c54a251c20fd7ec9855446ffb294326965577
+  data.tar.gz: ed4fb2feef7cabfc0534a35e2b5fb475c74be79b319bb7f6d6a6e1c443c8405654ccc06e1c6c596e0a73250518da7c45023a2657ebe65d8e449cd748d2c1c23a

data/.gitignore

@@ -5,3 +5,5 @@ sftp-config.json
 rdoc/
 .bundle/
 Gemfile.lock
+coverage/*
+TODO

data/CHANGELOG.md

@@ -1,6 +1,18 @@
 CHANGELOG
 ================================
 
+v.3.0.0 (2014-07-04)
+--------------------
+
+Final fixes for v.3.0.0!
+This release does break backwards compatibility with version 2.x and earlier. A [migration guide](https://github.com/datasift/datasift-ruby/blob/3.0.0/MIGRATING_TO_V.3.0.0.md) is available.
+* Adds support for the [Pull Push Destination](http://dev.datasift.com/docs/push/connectors/pull)
+* Adds support for DataSift [Historics Preview](http://dev.datasift.com/docs/historics/preview)
+* Adds support for the new Dynamic Lists feature
+* Adds support for multi-streaming via WebSockets
+* Added a CLI to the library
+* Updated test suite
+
 v.3.0.0.beta (2013-11-07)
 -------------------------
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2011-2012 MediaSift Ltd
+Copyright (c) 2011-2014 MediaSift Ltd
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/MIGRATING_TO_V.3.0.0.md

@@ -109,7 +109,7 @@ on_message is called when we receive [user status messages](http://dev.datasift.
 * @datasift.push.update @params.merge({:id => subscription_id, :name => 'Updated name'})
 * @datasift.push.stop subscription_id
 * @datasift.push.delete subscription_id
-* @datasift.push.logs
+* @datasift.push.log
 * @datasift.push.get_by_subscription subscription_id
 * @datasift.push.get
 * @datasift.pull

data/README.md

@@ -20,6 +20,14 @@ The library will use SSL connections by default. While we recommend using SSL
 you may disable it if required by passing ':enable_ssl => false' as the third
 parameter when creating your @config object.
 
+Getting Started
+---------------
+Before you can begin using this library, you will need to have an active [DataSift](http://datasift.com) account - you can sign up for a new account at [datasift.com/get-started](http://datasift.com/get-started/).
+
+Many of the examples and API endpoints used in this library require you have enabled certain data sources before you can receive any data (you should do this at [datasift.com/source](https://datasift.com/source)). Certain API features, such as [Historics](http://datasift.com/platform/historics/) and [Managed Sources](http://datasift.com/platform/datasources/) will require you have signed up to a monthly subscription before you can access them. 
+
+If you are interested in using these features, or would like more information about DataSift, please [get in touch](http://datasift.com/contact-us/)!
+
 Simple example
 --------------
 
@@ -65,10 +73,23 @@ conn.stream.read_thread.join
 See the [Understanding the Output Data](http://dev.datasift.com/docs/getting-started/data) page on the DataSift Developer site for
 full details of the data contained within each interaction.
 
+Supported Operating Environment
+-------------------------------
+This version of the client library has been tested, and is known to work against the following language versions and Operating Systems:
+
+### Language Versions
+* Ruby 2.0.0
+* Ruby 2.1.0
+
+### Operating Systems
+* Linux
+* OS X
+* Windows 7/8
+
 License
 -------
 
-All code contained in this repository is Copyright 2011-2013 MediaSift Ltd.
+All code contained in this repository is Copyright 2011-2014 MediaSift Ltd.
 
-This code is released under the BSD license. Please see the LICENSE file for
+This code is released under the BSD license. Please see the [LICENSE](https://github.com/datasift/datasift-ruby/blob/master/LICENSE) file for
 more details.

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/VERSION

@@ -1 +1 @@
-3.0.0.beta4
+3.0.0

data/datasift.gemspec

@@ -8,19 +8,19 @@ Gem::Specification.new do |s|
   s.homepage    = 'https://github.com/datasift/datasift-ruby'
   s.license     = 'BSD'
 
-  s.platform         = Gem::Platform::RUBY
-  s.rubygems_version = %q{1.3.6}
+  s.platform    = Gem::Platform::RUBY
   s.required_rubygems_version = Gem::Requirement.new(">= 1.3.6") if s.respond_to? :required_rubygems_version=
 
   s.add_runtime_dependency('rest-client', '~> 1.6.7')
   s.add_runtime_dependency('multi_json', '~> 1.8.0')
   s.add_runtime_dependency('websocket-td', '~> 0.0.4')
+
   s.add_development_dependency('rdoc', '> 0')
   s.add_development_dependency('webmock', '~> 1.17.1')
   s.add_development_dependency('shoulda', '~> 2.11.3')
-  s.add_development_dependency('test-unit', '>= 2.5.5')
+  s.add_development_dependency('minitest', '~> 5.0')
 
   s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.test_files    = `git ls-files -- {test}/*`.split("\n")
   s.require_paths = ["lib"]
 end

data/examples/auth.rb

@@ -1,25 +1,29 @@
 class DataSiftExample
-  require '../lib/datasift'
+  require 'datasift'
 
   def initialize
     #only SSLv3 and TLSv1 currently supported, TLSv1 preferred
     # this is fixed in REST client and is scheduled for the 1.7.0 release
     # see https://github.com/rest-client/rest-client/pull/123
     OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ssl_version] = 'TLSv1'
-    @username = 'zcourts'
-    @api_key  ='9dc2e16075de005b1a23444fbc6dd269'
-    @config   ={:username => @username, :api_key => @api_key, :enable_ssl => true}
+    @username = 'DATASIFT_USERNAME'
+    @api_key  = 'DATASIFT_API_KEY'
+    @config   = {:username => @username, :api_key => @api_key, :enable_ssl => true}
     @params   = {
         :output_type                       => 's3',
-        'output_params.bucket'             => 'apitests',
+        'output_params.bucket'             => 'YOUR_BUCKET_NAME',
         'output_params.directory'          => 'ruby',
         'output_params.acl'                => 'private',
-        'output_params.auth.access_key'    => 'AKIAIINK5C4FH75RSWNA',
-        'output_params.auth.secret_key'    => 'F9mLnLoFFGuCNgbMUhdhHmm5YCcNAt/OG32SUhPy',
+        'output_params.auth.access_key'    => 'ADD_YOUR_ACCESS_KEY',
+        'output_params.auth.secret_key'    => 'ADD_YOUR_SECRET_KEY',
         'output_params.delivery_frequency' => 0,
-        'output_params.max_size'           => 10485760,
+        'output_params.max_size'           => 104857600,
         'output_params.file_prefix'        => 'DataSift',
     }
+    @pull_params = {
+      :output_type => 'pull',
+      'output_params.max_size' => 52428800
+    }
     @datasift = DataSift::Client.new(@config)
   end
 
@@ -27,12 +31,12 @@ class DataSiftExample
 
   def create_push(hash, is_historics_id = false)
     create_params = @params.merge ({
-        #hash or playback_id can be used but not both
+        #hash or historics_id can be used but not both
         :name           => 'My awesome push subscription',
         :initial_status => 'active', # or 'paused' or 'waiting_for_start'
     })
     if is_historics_id
-      create_params.merge!({:playback_id => hash})
+      create_params.merge!({:historics_id => hash})
     else
       create_params.merge!({:hash  => hash,
                             #start and end are not valid for historics

data/examples/cli.sh

@@ -48,7 +48,7 @@ historic_id=$(echo ${historic} | jq -r .body.id)
 echo "Historic created with ID $historic_id"
 
 echo 'Validating Push subscription'
-push_v=$(ds -e push -c validate -p playback_id ${historic_id} -p name "Playback CLI @ $start" -p output_type http \
+push_v=$(ds -e push -c validate -p historics_id ${historic_id} -p name "Playback CLI @ $start" -p output_type http \
 -p output_params.method post -p output_params.url 'http://ec2-50-19-63-138.compute-1.amazonaws.com:80' \
 -p output_params.delivery_frequency 0 -p output_params.max_size 102400 -p output_params.auth.type none \
 -p output_params.verify_ssl false -p output_params.use_gzip true)
@@ -61,7 +61,7 @@ if [ ${push_status} != 200 ]; then
 fi
 
 echo 'Creating Push from Historic'
-push=$(ds -e push -c create -p playback_id ${historic_id} -p name "Playback CLI @ $start" -p output_type http \
+push=$(ds -e push -c create -p historics_id ${historic_id} -p name "Playback CLI @ $start" -p output_type http \
 -p output_params.method post -p output_params.url 'http://ec2-50-19-63-138.compute-1.amazonaws.com:80' \
 -p output_params.delivery_frequency 0 -p output_params.max_size 102400 -p output_params.auth.type none \
 -p output_params.verify_ssl false -p output_params.use_gzip true)

data/examples/core_api_eg.rb

@@ -10,21 +10,26 @@ class CoreApiEg < DataSiftExample
     begin
       csdl = 'interaction.content contains "test"'
       # see docs at http://dev.datasift.com/docs/rest-api/validate
+      puts "Is the following CSDL valid? #{csdl}"
       puts @datasift.valid? csdl
 
       # http://dev.datasift.com/docs/rest-api/compile
+      puts "\nCompile the CSDL and get a stream hash"
       stream = @datasift.compile csdl
       puts stream[:data][:hash]
 
       # http://dev.datasift.com/docs/rest-api/dpu
+      puts "\nGet the DPU cost of the compiled CSDL"
       dpu = @datasift.dpu stream[:data][:hash]
       puts dpu[:data][:dpu]
 
       # http://dev.datasift.com/docs/rest-api/balance
+      puts "\nGet the remaining balance for my account"
       balance = @datasift.balance
       puts balance[:data]
 
       #http://dev.datasift.com/docs/rest-api/usage
+      puts "\nGet my recent account usage"
       usage = @datasift.usage
       puts usage[:data]
 
@@ -43,4 +48,4 @@ class CoreApiEg < DataSiftExample
     end
   end
 end
-CoreApiEg.new
+CoreApiEg.new

data/examples/dynamic_list_eg.rb

@@ -0,0 +1,74 @@
+require './auth'
+
+class DynamicListApi < DataSiftExample
+  def initialize
+    super
+    run
+  end
+
+  def run
+    begin
+
+      ##
+      # Text Dynamic Lists
+      puts "\nCreating a Dynamic List containing text items"
+      puts 'Creating a dynamic list: /list/create'
+      list = @datasift.dynamic_list.create('text', 'My dynamic list')
+      puts list
+
+      id = list[:data][:id]
+      items = ['keyword1', 'keyword2']
+
+      puts "\nGet a list of your Dynamic Lists: /list/get"
+      puts @datasift.dynamic_list.get
+
+      puts "\nAdding items to the dynamic list: #{items}: /list/add"
+      puts @datasift.dynamic_list.add(id, items)
+
+      csdl = "interaction.content list_any \"#{id}\""
+      puts "\nCompile the following CSDL using a Dynamic List:"
+      puts csdl
+      puts @datasift.compile csdl
+
+      remove_items = ['keyword1']
+      puts "\nRemoving item #{remove_items} from the dynamic list: /list/remove"
+      puts @datasift.dynamic_list.remove(id, remove_items)
+
+      puts "\nCheck #{remove_items} has been removed from the list: /list/exists"
+      puts @datasift.dynamic_list.exists(id, remove_items)
+
+      remaining_item = ['keyword2']
+      puts "\nCheck #{remaining_item} still exists in the list: /list/exists"
+      puts @datasift.dynamic_list.exists(id, remaining_item)
+
+      puts "\nDeleting the list: /list/delete"
+      puts @datasift.dynamic_list.delete id
+
+      ##
+      # Integer Dynamic Lists
+      puts "\n --- \nCreating a Dynamic List containing integers"
+      puts 'Creating a dynamic list: /list/create'
+      list = @datasift.dynamic_list.create('integer', 'My dynamic integer list')
+      puts list
+
+      id = list[:data][:id]
+      items = [11111, 22222, 33333, 44444]
+
+      puts "\nAdding integers to the dynamic list: #{items}: /list/add"
+      puts @datasift.dynamic_list.add(id, items)
+
+      csdl = "interaction.author.id list_in \"#{id}\""
+      puts "\nCompile the following CSDL using a Dynamic List:"
+      puts csdl
+      puts @datasift.compile csdl
+
+      puts "\nDeleting the list: /list/delete"
+      puts @datasift.dynamic_list.delete id
+
+    rescue DataSiftError => dse
+      puts dse.message
+    end
+  end
+end
+
+DynamicListApi.new

data/examples/dynamic_list_replace_eg.rb

@@ -0,0 +1,45 @@
+require './auth'
+
+class DynamicListReplaceApi < DataSiftExample
+  def initialize
+    super
+    run
+  end
+
+  def run
+    begin
+
+      puts "\nCreating a dynamic list: /list/create"
+      list = @datasift.dynamic_list.create('text', 'My dynamic list')
+      puts list
+
+      id = list[:data][:id]
+
+      puts "\nStart a new replace list: /list/replace/start"
+      replace = @datasift.dynamic_list_replace.start id
+      puts replace
+
+      puts "\nAbort the list replace: /list/replace/abort"
+      puts @datasift.dynamic_list_replace.abort replace[:data][:id]
+
+      puts "\nStart a new replace list (again): /list/replace/start"
+      replace = @datasift.dynamic_list_replace.start id
+      puts replace
+
+      items = ['keyword1', 'keyword2']
+      puts "\nAdd items #{items} to replacement list: /list/replace/add"
+      puts @datasift.dynamic_list_replace.add(replace[:data][:id], items)
+
+      puts "\nCommit the replacement list: /list/replace/commit"
+      puts @datasift.dynamic_list_replace.commit replace[:data][:id]
+
+      puts "\nCleanup: Delete the list: /list/delete"
+      puts @datasift.dynamic_list.delete id
+
+    rescue DataSiftError => dse
+      puts dse.message
+    end
+  end
+end
+
+DynamicListReplaceApi.new

data/examples/historics_eg.rb

@@ -9,42 +9,49 @@ class HistoricsApi < DataSiftExample
     begin
       stream   = @datasift.compile 'interaction.content contains "datasift"'
       hash     = stream[:data][:hash]
-      start    = Time.now.to_i - 7200
+      start    = Time.now.to_i - 10800
       end_time = start + 3600
 
-      puts 'Check the data coverage for the last two hours'
+      puts 'Check the data coverage for this Historic period'
       puts @datasift.historics.status(start, end_time)
 
-      puts 'Preparing'
-      #prepare a historics to start from two hours ago and run for 1 hour using twitter as a data source and using only a 10% sample
-      historics = @datasift.historics.prepare(hash, start, end_time, 'My ruby historics')
+      puts "\nPreparing"
+      #prepare a historics to start from three hours ago and run for 1 hour using twitter as a data source and using only a 10% sample
+      historics = @datasift.historics.prepare(hash, start, end_time, 'My ruby historics', 'twitter', 10)
       puts historics
 
       id = historics[:data][:id]
 
-      puts 'Creating push subscription for historics'
+      puts "\nCreating push subscription for historics"
       puts create_push(id, true)
 
-      puts "Starting historics #{id}"
+      puts "\nStarting historics #{id}"
       puts @datasift.historics.start id
 
-      puts 'Updating historics'
+      puts "\nPausing historics #{id}"
+      puts @datasift.historics.pause id
+
+      puts "\nResuming historics #{id}"
+      puts @datasift.historics.resume id
+
+      puts "\nUpdating historics"
       puts @datasift.historics.update(id, 'The new name of my historics')
 
-      puts 'Get info for the historics'
+      puts "\nGet info for the historics"
       puts @datasift.historics.get_by_id id
 
-      puts 'Getting info for all my historics'
+      puts "\nGetting info for all my historics"
       puts @datasift.historics.get
 
-      puts 'Stopping historics'
+      puts "\nStopping historics"
       puts @datasift.historics.stop id
 
-      puts 'Deleting historics'
+      puts "\nDeleting historics"
       puts @datasift.historics.delete id
     rescue DataSiftError => dse
       puts dse.message
     end
   end
 end
-HistoricsApi.new
+
+HistoricsApi.new

data/examples/historics_preview_eg.rb

@@ -11,14 +11,15 @@ class HistoricsPreviewApi < DataSiftExample
       stream = @datasift.compile 'interaction.content contains "datasift"'
       hash   = stream[:data][:hash]
 
-      puts 'Creating a preview'
+      puts "\nCreating a preview"
       # see http://dev.datasift.com/docs/rest-api/previewcreate for docs
+      sources    = 'facebook,twitter'
       parameters = 'interaction.author.link,targetVol,hour;interaction.type,freqDist,10'
       start      = Time.now.to_i - (3600 * 48) # 48hrs ago
-      source     = @datasift.historics_preview.create(hash, parameters, start)
+      source     = @datasift.historics_preview.create(hash, sources, parameters, start)
       puts source
 
-      puts 'Getting preview data'
+      puts "\nGetting preview data"
       puts @datasift.historics_preview.get source[:data][:id]
 
     rescue DataSiftError => dse
@@ -27,4 +28,4 @@ class HistoricsPreviewApi < DataSiftExample
   end
 end
 
-HistoricsPreviewApi.new
+HistoricsPreviewApi.new