datasift 3.7.2 → 3.8.0

data/examples/account_identity_token_eg.rb

@@ -1,73 +0,0 @@
-require './auth'
-class AccountIdentityTokenEg < DataSiftExample
-  def initialize
-    super
-    @datasift = DataSift::Client.new(@config)
-    run
-  end
-
-  def run
-    begin
-      puts "Create a new identity to create tokens for"
-      identity = @datasift.account_identity.create(
-        "Ruby Identity for Tokena",
-        "active",
-        false
-      )
-      identity_id = identity[:data][:id]
-      puts identity[:data].to_json
-
-      puts "\nCreate a Token for our Identity"
-      puts @datasift.account_identity_token.create(
-        identity_id,
-        'facebook',
-        'YOUR_TOKEN'
-      )[:data].to_json
-
-      puts "\nList all existing Tokens for this Identity"
-      puts @datasift.account_identity_token.list(
-        identity_id
-      )[:data].to_json
-
-      puts "\nGet existing Token by Identity and Service"
-      puts @datasift.account_identity_token.get(
-        identity_id,
-        'facebook'
-      )[:data].to_json
-
-      puts "\nUpdate a Token for a given Identity"
-      puts @datasift.account_identity_token.update(
-        identity_id,
-        'facebook',
-        'YOUR_NEW_TOKEN'
-      )[:data].to_json
-
-      puts "\nDelete an Token for a given Identity and Service"
-      puts @datasift.account_identity_token.delete(
-        identity_id,
-        'facebook'
-      )[:data].to_json
-
-      puts "\nCleanup and remove the Identity"
-      @datasift.account_identity.delete(identity_id)
-
-    rescue DataSiftError => dse
-      puts dse.inspect
-      # Then match specific error to take action;
-      #   All errors thrown by the client extend DataSiftError
-      case dse
-        when ConnectionError
-          # some connection error
-        when AuthError
-        when BadRequestError
-          puts '[WARNING] You will need to use a valid token to run through this example'
-        else
-          # do something else...
-      end
-      puts "\nCleanup and remove the Identity"
-      @datasift.account_identity.delete(identity_id)
-    end
-  end
-end
-
-AccountIdentityTokenEg.new

data/examples/auth.rb

@@ -1,55 +0,0 @@
-class DataSiftExample
-  require 'datasift'
-
-  def initialize
-    @config = {
-      username: 'DATASIFT_USERNAME',
-      api_key: 'DATASIFT_API_KEY',
-      api_version: 'v1.3'
-    }
-    @params = {
-      output_type: 's3',
-      output_params: {
-        bucket: 'YOUR_BUCKET_NAME',
-        directory: 'ruby',
-        acl: 'private',
-        delivery_frequency: 0,
-        max_size: 104857600,
-        file_prefix: 'DataSift',
-        auth: {
-          access_key: 'ADD_YOUR_ACCESS_KEY',
-          secret_key: 'ADD_YOUR_SECRET_KEY',
-        }
-      }
-    }
-    @pull_params = {
-      output_type: 'pull',
-      output_params: {
-        max_size: 52428800
-      }
-    }
-    @datasift = DataSift::Client.new(@config)
-  end
-
-  attr_reader :datasift, :params
-
-  def create_push(hash, is_historics_id = false)
-    create_params = @params.merge ({
-        # Hash or historics_id can be used, but not both
-        name: 'My awesome push subscription',
-        initial_status: 'active', # or 'paused' or 'waiting_for_start'
-    })
-    if is_historics_id
-      create_params.merge!(historics_id: hash)
-    else
-      # Start and end are not valid for historics
-      create_params.merge!(
-        hash: hash,
-        start: Time.now.to_i,
-        end: Time.now.to_i + 320
-      )
-    end
-    puts 'Creating subscription'
-    @datasift.push.create create_params
-  end
-end

data/examples/cli.sh

@@ -1,155 +0,0 @@
-#!/bin/bash
-command -v jq >/dev/null 2>&1 || { echo >&2 "jq command must be available. See http://stedolan.github.io/jq/.  Aborting."; exit 1; }
-
-set -e
-
-DEFAULT_API='api.datasift.com'
-
-CC=${1:$CC}
-DU=${2:-$DU}
-DK=${3:-$DK}
-API=${4:-$DEFAULT_API}
-
-function ds(){
-    ${CC} -a ${DU} ${DK} --u ${API} "$@" #| jq .
-}
-
-# core API - validate our hash, compile it, check our usage, dpu and balance
-echo 'Validating CSDL'
-csdl='interaction.content contains "music"'
-valid=$(ds -e core -c validate -p csdl "$csdl" | jq .status)
-
-if [ ${valid} != 200 ]; then
-    echo "Validating CSDL failed"
-    echo ${valid}
-    exit -1
-fi
-
-echo 'Compiling'
-hash=$(ds -e core -c compile -p csdl "$csdl" | jq -r .body.hash)
-echo "Compiled and got $hash"
-
-echo 'Usage :'
-ds -e core -c usage | jq .
-
-echo 'DPU :'
-ds -e core -c dpu -p hash $hash | jq .
-
-echo 'Balance :'
-ds -e core -c usage | jq .
-
-echo 'Preparing Historic query'
-end=`expr $(date +%s) - 7200`
-start=`expr $end - 3600`
-
-historic=$(ds -e historics -c prepare -p start ${start} -p end ${end} -p name "Historics CLI @ $start" -p hash ${hash})
-echo ${historic} | jq .
-historic_id=$(echo ${historic} | jq -r .body.id)
-echo "Historic created with ID $historic_id"
-
-echo 'Validating Push subscription'
-push_v=$(ds -e push -c validate -p historics_id ${historic_id} -p name "Playback CLI @ $start" -p output_type http \
--p output_params.method post -p output_params.url 'http://ec2-50-19-63-138.compute-1.amazonaws.com:80' \
--p output_params.delivery_frequency 0 -p output_params.max_size 102400 -p output_params.auth.type none \
--p output_params.verify_ssl false -p output_params.use_gzip true)
-push_status=$(echo ${push_v} | jq .status)
-echo ${push_v} | jq .
-
-if [ ${push_status} != 200 ]; then
-    echo "Validating Push subscription failed"
-    exit -1
-fi
-
-echo 'Creating Push from Historic'
-push=$(ds -e push -c create -p historics_id ${historic_id} -p name "Playback CLI @ $start" -p output_type http \
--p output_params.method post -p output_params.url 'http://ec2-50-19-63-138.compute-1.amazonaws.com:80' \
--p output_params.delivery_frequency 0 -p output_params.max_size 102400 -p output_params.auth.type none \
--p output_params.verify_ssl false -p output_params.use_gzip true)
-
-echo "Created push subscription for historic"
-echo ${push} | jq .
-push_id=$(echo ${push} | jq -r .body.id)
-
-echo 'Starting Historic query'
-ds -e historics -c start -p id ${historic_id} | jq .
-
-echo 'Getting Historic status'
-ds -e historics -c status -p start ${start} -p end ${end} | jq .
-
-echo 'Getting Historics'
-ds -e historics -c get -p id ${historic_id} | jq .
-
-echo 'Updating historic'
-ds -e historics -c update -p id ${historic_id} -p name "Some name @ $start - CLI" | jq .
-
-echo 'Getting push'
-ds -e push -c get -p id ${push_id} | jq .
-
-echo 'Getting push logs'
-ds -e push -c log -p id ${push_id} | jq .
-
-echo 'Pausing push'
-ds -e push -c pause -p id ${push_id} | jq .
-
-echo 'Resuming push'
-ds -e push -c resume -p id ${push_id} | jq .
-
-echo 'Stopping Historic'
-ds -e historics -c stop -p id ${historic_id} | jq .
-
-echo 'Deleting Historic'
-ds -e historics -c delete -p id ${historic_id} | jq .
-
-echo 'Stopping push'
-ds -e push -c stop -p id ${push_id} | jq .
-
-echo 'Deleting push'
-ds -e push -c delete -p id ${push_id} | jq .
-#todo update push, pull
-
-echo "Attempting to create a Historics preview"
-preview=$(ds -e preview -c create -p start ${start} -p end ${end} -p hash ${hash} -p sources tumblr \
--p parameters 'interaction.author.link,targetVol,hour;interaction.type,freqDist,10')
-
-echo ${preview} | jq .
-preview_id=$(echo ${preview} | jq -r .body.id)
-
-
-echo "Getting the preview we created"
-ds -e preview -c get -p id ${preview_id} | jq .
-
-echo "Creating a managed source"
-source=$(ds -e managed_sources -c create -p source_type instagram -p name api \
-     -p auth "[{\"parameters\":{\"value\":\"$start$end\"}}]" \
-     -p resources '[{"parameters":{"value":"cats","type":"tag"}}]' \
-     -p parameters '{"comments":true,"likes":false}')
-echo ${source}
-source_id=$(echo ${source}| jq -r .body.id)
-echo ${source_id}
-
-echo "Starting managed source"
-ds -e managed_sources -c start -p source_id ${source_id} | jq .
-
-echo "Getting managed sources"
-ds -e managed_sources -c get | jq .
-
-echo "Getting Instagram sources"
-ds -e managed_sources -c get -p source_type instagram | jq .
-
-echo "Getting Facebook page sources"
-ds -e managed_sources -c get -p source_type facebook_page | jq .
-
-echo "Getting page 2 of instagram sources"
-ds -e managed_sources -c get -p source_type instagram -p page 2 | jq .
-
-echo "Getting source for $source_id"
-ds -e managed_sources -c get -p source_id ${source_id} | jq .
-
-echo "Getting logs for source $source_id"
-ds -e managed_sources -c log -p source_id ${source_id} -p page 2 -p per_page 1 | jq .
-
-echo "Stopping managed source"
-ds -e managed_sources -c stop -p source_id ${source_id} | jq .
-
-echo "Deleting managed source $source_id"
-ds -e managed_sources -c delete -p source_id ${source_id} | jq .

data/examples/core_api_eg.rb

@@ -1,50 +0,0 @@
-require './auth'
-class CoreApiEg < DataSiftExample
-  def initialize
-    super
-    @datasift = DataSift::Client.new(@config)
-    run
-  end
-
-  def run
-    begin
-      csdl = 'interaction.content contains "test"'
-      # see docs at http://dev.datasift.com/docs/rest-api/validate
-      puts "Is the following CSDL valid? #{csdl}"
-      puts @datasift.valid? csdl
-
-      # http://dev.datasift.com/docs/rest-api/compile
-      puts "\nCompile the CSDL and get a stream hash"
-      stream = @datasift.compile csdl
-      puts stream[:data]
-
-      # http://dev.datasift.com/docs/rest-api/dpu
-      puts "\nGet the DPU cost of the compiled CSDL"
-      dpu = @datasift.dpu stream[:data][:hash]
-      puts dpu[:data][:dpu]
-
-      # http://dev.datasift.com/docs/rest-api/balance
-      puts "\nGet the remaining balance for my account"
-      balance = @datasift.balance
-      puts balance[:data]
-
-      #http://dev.datasift.com/docs/rest-api/usage
-      puts "\nGet my recent account usage"
-      usage = @datasift.usage
-      puts usage[:data]
-
-    rescue DataSiftError => dse
-      puts dse.message
-      # Then match specific one to take action; All errors thrown by the client extend DataSiftError
-      case dse
-        when ConnectionError
-          # some connection error
-        when AuthError
-        when BadRequestError
-        else
-          # do something else...
-      end
-    end
-  end
-end
-CoreApiEg.new

data/examples/historics_eg.rb

@@ -1,66 +0,0 @@
-require './auth'
-class HistoricsApi < DataSiftExample
-  def initialize
-    super
-    run_historics
-  end
-
-  def run_historics
-    begin
-      stream   = @datasift.compile 'interaction.content contains "datasift"'
-      hash     = stream[:data][:hash]
-      start    = Time.now.to_i - 10800
-      end_time = start + 3600
-
-      puts 'Check the data coverage for this Historic period'
-      puts @datasift.historics.status(start, end_time)
-
-      puts "\nPreparing"
-      # Prepare an Historics query to start from three hours ago and run for 1
-      #   hour using Tumblr as a data source and using only a 10% sample
-      historics = @datasift.historics.prepare(
-        hash, start, end_time, 'My ruby historics', 'tumblr', 10
-      )
-      puts historics
-
-      id = historics[:data][:id]
-
-      puts "\nCheck the DPU cost of this Historics query"
-      puts "#{@datasift.dpu('', id)[:data][:dpu]} DPUs"
-
-      puts "\nCreating push subscription for historics"
-      subscription = create_push(id, true)
-
-      puts "\nStarting historics #{id}"
-      puts @datasift.historics.start id
-
-      puts "\nPausing historics #{id}"
-      puts @datasift.historics.pause id
-
-      puts "\nResuming historics #{id}"
-      puts @datasift.historics.resume id
-
-      puts "\nUpdating historics"
-      puts @datasift.historics.update(id, 'The new name of my historics')
-
-      puts "\nGet info for the historics"
-      puts @datasift.historics.get_by_id id
-
-      puts "\nGetting info for all my historics"
-      puts @datasift.historics.get
-
-      puts "\nStopping historics"
-      puts @datasift.historics.stop id
-
-      puts "\nDeleting historics"
-      puts @datasift.historics.delete id
-
-      puts "\nCleanup and delete Push subscription"
-      puts @datasift.push.delete subscription[:data][:id]
-    rescue DataSiftError => dse
-      puts dse.message
-    end
-  end
-end
-
-HistoricsApi.new

data/examples/historics_preview_eg.rb

@@ -1,31 +0,0 @@
-require './auth'
-class HistoricsPreviewApi < DataSiftExample
-  def initialize
-    super
-    run
-  end
-
-  def run
-    begin
-      puts 'Creating hash'
-      stream = @datasift.compile 'interaction.content contains "datasift"'
-      hash   = stream[:data][:hash]
-
-      puts "\nCreating a preview"
-      # see http://dev.datasift.com/docs/rest-api/previewcreate for docs
-      sources    = 'tumblr'
-      parameters = 'interaction.author.link,targetVol,hour;interaction.type,freqDist,10'
-      start      = Time.now.to_i - (3600 * 48) # 48hrs ago
-      source     = @datasift.historics_preview.create(hash, sources, parameters, start)
-      puts source
-
-      puts "\nGetting preview data"
-      puts @datasift.historics_preview.get source[:data][:id]
-
-    rescue DataSiftError => dse
-      puts dse.message
-    end
-  end
-end
-
-HistoricsPreviewApi.new

data/examples/ingestion/twitter_gnip_batch_eg.rb

@@ -1,61 +0,0 @@
-require './../auth'
-class TwitterGnipBatchEg < DataSiftExample
-  def initialize
-    super
-    @datasift = DataSift::Client.new(@config)
-    run
-  end
-
-  def run
-    begin
-      # "gnip_1" is the default mapping used to map Tweets ingested from Gnip into a
-      #   format DataSift can use. Using this mapping will altomatically define the
-      #   IDML you need to use for this source
-      resource = [{
-        parameters: {
-          mapping: "gnip_1"
-        }
-      }]
-      puts "Creating Managed Source for ODP Ingestion\n"
-      source = @datasift.managed_source.create('twitter_gnip', 'Ruby ODP API', {}, resource)
-      puts "Manage Source with ID #{source[:data][:id]} created"
-
-      lines = 0
-      payload = ''
-      # Read interactions out of the fake Tweets fixture file
-      File.readlines('./../../test/fixtures/data/fake_gnip_tweets.json').each do |line|
-        lines += 1
-        payload += line
-
-        # Upload interactions as batches of five
-        if lines % 5 == 0
-          puts "\nUploading a batch of five interactions\n"
-          puts @datasift.odp.ingest(source[:data][:id], payload)[:data].to_json
-          payload = ''
-        end
-      end
-
-      puts "\nCleanup after this test and delete the Managed Source"
-      @datasift.managed_source.delete(source[:data][:id])
-
-    rescue DataSiftError => dse
-      puts dse.message
-      # Then match specific one to take action - All errors thrown by the client extend
-      #   DataSiftError
-      case dse
-        when ConnectionError
-          # some connection error
-        when AuthError
-        when BadRequestError
-        when PayloadTooLargeError
-          # You've probably hit your x_ingestion_data_ratelimit_limit rate limit; back off
-          #   and try again in a bit. Check your x_ingestion_request_ratelimit_reset header
-          #   to see when you can retry this request successfully
-        else
-          # do something else...
-      end
-    end
-  end
-end
-
-TwitterGnipBatchEg.new