dscli 0.0.1 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +17 -8
- data/bin/dscli +4 -1
- data/lib/dscli/api.rb +92 -11
- data/lib/dscli/cli.rb +53 -0
- data/lib/dscli/connectors.rb +97 -0
- data/lib/dscli/core.rb +7 -0
- data/lib/dscli/historics.rb +68 -0
- data/lib/dscli/outputParams.rb +159 -0
- data/lib/dscli/push.rb +80 -0
- data/lib/dscli/source.rb +110 -0
- data/lib/dscli/storage.rb +0 -23
- data/lib/dscli/version.rb +1 -1
- data/lib/dscli.rb +5 -1
- metadata +43 -23
- data/lib/dscli/commands.rb +0 -69
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9742d9c0950b2e48c61b676a8373d27fa5b13c6c
|
4
|
+
data.tar.gz: c6cd518e48efd480348a3ecadaa14815151e2ea5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cf49bbb677cc4a28debf4c23af619ccd2d40e66b80cfbaebd8d4157e1e8d40b5236af1046195c4bbc80ec210e09e976c2dfae3158c246a4ac7e61bad1bf4543a
|
7
|
+
data.tar.gz: 21f94e1c8156ddb1a87b920f049765075738403c48c997cd45b40436ba79c9fbe6db3bcbd48d3c7f45b6fd29b4a33a75502e7317a68ae1bd3c8b6af8ad5661b0
|
data/README.md
CHANGED
@@ -2,7 +2,6 @@
|
|
2
2
|
|
3
3
|
DSCLI is a command line wrapper around the DataSift API Client Library for Ruby.
|
4
4
|
It was built by the DataSift Support Team to help speed up tasks we found ourselves frequently repeating.
|
5
|
-
Version 0.0.1 offers support for only some of the core API functions. Support for the Push and Historics APIs is on it's way.
|
6
5
|
|
7
6
|
## Installation
|
8
7
|
|
@@ -12,15 +11,25 @@ This tool was intended for use as a CLI tool only:
|
|
12
11
|
|
13
12
|
## Usage
|
14
13
|
|
15
|
-
First, call ``` dscli config ``` to enter your DataSift API credentials. This will store your username and API key in a new file called ~/.datasiftcli
|
14
|
+
First, call ``` dscli config ``` to enter your DataSift API credentials. This will store your username and API key in a new file called ~/.datasiftcli.
|
16
15
|
You should then be free to use any of following commands:
|
17
16
|
|
18
|
-
dscli
|
19
|
-
dscli
|
20
|
-
dscli
|
21
|
-
dscli
|
22
|
-
dscli
|
23
|
-
dscli
|
17
|
+
dscli balance # Gets your current plan and balance
|
18
|
+
dscli compile (csdl) # Compile CSDL
|
19
|
+
dscli config # Configure your DataSift API credentials
|
20
|
+
dscli dpu (hash) # Find the DPU cost of a given stream hash
|
21
|
+
dscli help [COMMAND] # Describe available commands or one specific command
|
22
|
+
dscli historics <command> # Run commands relating to historics
|
23
|
+
dscli push <command> # Run commands relating to push subscriptions
|
24
|
+
dscli source <command> # Run commands relating to managed sources
|
25
|
+
dscli stream (hash) # Stream interactions from a DataSift stream to the command line
|
26
|
+
dscli usage [period] # Find your stream and license fee usage
|
27
|
+
|
28
|
+
|
29
|
+
## Example Usage
|
30
|
+
|
31
|
+
$ dscli compile 'interaction.content contains "coffee"'
|
32
|
+
=> ce9d18e45a53dfa2dfde7747f8c5d98b
|
24
33
|
|
25
34
|
## Contributing
|
26
35
|
|
data/bin/dscli
CHANGED
data/lib/dscli/api.rb
CHANGED
@@ -8,7 +8,7 @@ module Dscli
|
|
8
8
|
def initialize
|
9
9
|
storage = Dscli::Storage.new
|
10
10
|
config = storage.get_auth
|
11
|
-
@config = {:username => config[:auth][:username], :api_key => config[:auth][:api_key], :enable_ssl =>
|
11
|
+
@config = {:username => config[:auth][:username], :api_key => config[:auth][:api_key], :enable_ssl => true}
|
12
12
|
@datasift = DataSift::Client.new(@config)
|
13
13
|
end
|
14
14
|
|
@@ -32,12 +32,17 @@ module Dscli
|
|
32
32
|
dpu = response[:data]
|
33
33
|
end
|
34
34
|
|
35
|
+
def balance
|
36
|
+
response = @datasift.balance
|
37
|
+
return response[:data]
|
38
|
+
end
|
39
|
+
|
35
40
|
def stream(hash)
|
36
41
|
on_delete = lambda { |stream, m| puts m }
|
37
42
|
on_error = lambda { |stream, e| puts 'A serious error has occurred: ' + e.message.to_s }
|
38
43
|
on_message = lambda { |message, stream, hash| puts Yajl::Encoder.encode(message) }
|
39
44
|
on_connect = lambda { |stream| stream.subscribe(hash, on_message) }
|
40
|
-
on_close = lambda { |stream| puts
|
45
|
+
on_close = lambda { |stream, message| puts "Closed #{stream}: #{message}" }
|
41
46
|
|
42
47
|
on_datasift_message = lambda do |stream, message, hash|
|
43
48
|
puts "DataSift Message #{hash} ==> #{message}"
|
@@ -57,16 +62,92 @@ module Dscli
|
|
57
62
|
end
|
58
63
|
end
|
59
64
|
|
60
|
-
|
61
|
-
#
|
62
|
-
|
63
|
-
|
65
|
+
#######################################################
|
66
|
+
# PUSH
|
67
|
+
#######################################################
|
68
|
+
|
69
|
+
def push_list(page)
|
70
|
+
response = @datasift.push.get(page)
|
71
|
+
response[:data]
|
72
|
+
end
|
73
|
+
|
74
|
+
def push_get(id)
|
75
|
+
response = @datasift.push.get_by_subscription(id)
|
76
|
+
return response
|
77
|
+
end
|
78
|
+
|
79
|
+
def push_stop(id)
|
80
|
+
response = @datasift.push.stop(id)
|
81
|
+
puts response
|
82
|
+
end
|
83
|
+
|
84
|
+
def push_delete(id)
|
85
|
+
response = @datasift.push.delete(id)
|
86
|
+
return response
|
87
|
+
end
|
88
|
+
|
89
|
+
def push_log(id)
|
90
|
+
|
91
|
+
if id.nil?
|
92
|
+
response = @datasift.push.log
|
93
|
+
else
|
94
|
+
response = @datasift.push.log_for(id)
|
95
|
+
end
|
96
|
+
|
97
|
+
return response[:data]
|
98
|
+
|
99
|
+
end
|
100
|
+
|
101
|
+
#######################################################
|
102
|
+
# HISTORICS
|
103
|
+
#######################################################
|
64
104
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
105
|
+
def historics_list(page)
|
106
|
+
response = @datasift.historics.get(20,page)
|
107
|
+
response[:data]
|
108
|
+
end
|
109
|
+
|
110
|
+
def historics_get(id)
|
111
|
+
return @datasift.historics.get_by_id(id)
|
112
|
+
end
|
113
|
+
|
114
|
+
def historics_stop(id)
|
115
|
+
return @datasift.historics.stop(id)
|
116
|
+
end
|
117
|
+
|
118
|
+
def historics_delete(id)
|
119
|
+
return @datasift.historics.delete(id)
|
120
|
+
end
|
121
|
+
|
122
|
+
#######################################################
|
123
|
+
# MANAGED SOURCES
|
124
|
+
#######################################################
|
125
|
+
|
126
|
+
def source_list(page)
|
127
|
+
response = @datasift.managed_source.get(nil,nil,page,20)
|
128
|
+
response[:data]
|
129
|
+
end
|
130
|
+
|
131
|
+
def source_get(id)
|
132
|
+
return @datasift.managed_source.get(id)
|
133
|
+
end
|
134
|
+
|
135
|
+
def source_start(id)
|
136
|
+
return @datasift.managed_source.start(id)
|
137
|
+
end
|
138
|
+
|
139
|
+
def source_stop(id)
|
140
|
+
return @datasift.managed_source.stop(id)
|
141
|
+
end
|
142
|
+
|
143
|
+
def source_delete(id)
|
144
|
+
return @datasift.managed_source.delete(id)
|
145
|
+
end
|
146
|
+
|
147
|
+
def source_log(id)
|
148
|
+
response = @datasift.managed_source.log(id,1,20)
|
149
|
+
response[:data]
|
150
|
+
end
|
70
151
|
|
71
152
|
end
|
72
153
|
|
data/lib/dscli/cli.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
class CLI < Thor
|
2
|
+
|
3
|
+
#######################################################
|
4
|
+
# Top-level commands
|
5
|
+
#######################################################
|
6
|
+
desc 'config', 'Configure your DataSift API credentials'
|
7
|
+
def config
|
8
|
+
input = Dscli::Parameters.new.user_config(options)
|
9
|
+
puts input.to_yaml
|
10
|
+
open(ENV['HOME'] + '/.datasiftcli', 'w') do |f|
|
11
|
+
f.write input.to_yaml
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
desc 'compile (csdl)', 'Compile CSDL'
|
16
|
+
def compile(csdl)
|
17
|
+
api = Dscli::API.new
|
18
|
+
definition = api.compile(csdl)
|
19
|
+
puts definition
|
20
|
+
end
|
21
|
+
|
22
|
+
desc 'usage [period]', 'Find your stream and license fee usage'
|
23
|
+
def usage(period = 'day')
|
24
|
+
api = Dscli::API.new
|
25
|
+
puts Yajl::Encoder.encode(api.usage(period), :pretty => true)
|
26
|
+
end
|
27
|
+
|
28
|
+
desc 'dpu (hash)', 'Find the DPU cost of a given stream hash'
|
29
|
+
def dpu(hash)
|
30
|
+
api = Dscli::API.new
|
31
|
+
puts Yajl::Encoder.encode(api.dpu(hash), :pretty => true)
|
32
|
+
end
|
33
|
+
|
34
|
+
desc 'stream (hash)', 'Stream interactions from a DataSift stream to the command line'
|
35
|
+
def stream(hash)
|
36
|
+
api = Dscli::API.new
|
37
|
+
api.stream(hash)
|
38
|
+
end
|
39
|
+
|
40
|
+
desc 'balance', 'Gets your current plan and balance'
|
41
|
+
def balance
|
42
|
+
api = Dscli::API.new
|
43
|
+
puts api.balance
|
44
|
+
end
|
45
|
+
|
46
|
+
#######################################################
|
47
|
+
# Second-level commands
|
48
|
+
#######################################################
|
49
|
+
register(Push, 'push', 'push <command>', 'Run commands relating to push subscriptions')
|
50
|
+
register(Historics, 'historics', 'historics <command>', 'Run commands relating to historics')
|
51
|
+
register(Source, 'source', 'source <command>', 'Run commands relating to managed sources')
|
52
|
+
|
53
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module Dscli
|
2
|
+
class Connectors
|
3
|
+
|
4
|
+
CONNECTORS = [["HTTP", "http"],
|
5
|
+
["CouchDB", "couchdb"],
|
6
|
+
["DynamoDB", "dynamodb"],
|
7
|
+
["ElasticSearch", "elasticsearch"],
|
8
|
+
["FTP", "ftp"],
|
9
|
+
["Google BigQuery", "bigquery"],
|
10
|
+
["MongoDB", "mongodb"],
|
11
|
+
["MySQL", "mysql"],
|
12
|
+
["Pull", "pull"],
|
13
|
+
["Redis", "redis"],
|
14
|
+
["S3", "s3"],
|
15
|
+
["SFTP", "sftp"],
|
16
|
+
["Splunk Storm REST", "splunkstormrest"],
|
17
|
+
["Splunk Enterprise", "splunkenterprise"],
|
18
|
+
["Zoomdata", "zoomdata"]]
|
19
|
+
|
20
|
+
def list
|
21
|
+
conenctors = CONNECTORS.map { |c| c[1] }
|
22
|
+
end
|
23
|
+
|
24
|
+
def setOutputParams(connector)
|
25
|
+
@op = Dscli::OutputParams.new
|
26
|
+
@push = {}
|
27
|
+
@push[:auth] = {}
|
28
|
+
@push[:server_level_auth] = {}
|
29
|
+
self.send(connector)
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
#
|
34
|
+
# Method to create each Push Destination
|
35
|
+
#
|
36
|
+
|
37
|
+
def http
|
38
|
+
end
|
39
|
+
|
40
|
+
def couchdb
|
41
|
+
end
|
42
|
+
|
43
|
+
def dynamodb
|
44
|
+
end
|
45
|
+
|
46
|
+
def elasticsearch
|
47
|
+
end
|
48
|
+
|
49
|
+
def ftp
|
50
|
+
end
|
51
|
+
|
52
|
+
def bigquery
|
53
|
+
end
|
54
|
+
|
55
|
+
def mongodb
|
56
|
+
end
|
57
|
+
|
58
|
+
def mysql
|
59
|
+
end
|
60
|
+
|
61
|
+
def pull
|
62
|
+
@push[:format] = @op.format
|
63
|
+
return @push
|
64
|
+
end
|
65
|
+
|
66
|
+
def redis
|
67
|
+
end
|
68
|
+
|
69
|
+
def s3
|
70
|
+
@push[:auth][:access_key] = @op.auth_access_key
|
71
|
+
@push[:auth][:secret_key] = @op.auth_secret_key
|
72
|
+
@push[:bucket] = @op.bucket
|
73
|
+
@push[:directory] = @op.directory
|
74
|
+
@push[:acl] = @op.acl
|
75
|
+
@push[:file_prefix] = @op.file_prefix
|
76
|
+
@push[:max_size] = @op.max_size
|
77
|
+
return @push
|
78
|
+
end
|
79
|
+
|
80
|
+
def sftp
|
81
|
+
end
|
82
|
+
|
83
|
+
def splunkstormrest
|
84
|
+
end
|
85
|
+
|
86
|
+
def splunkenterprise
|
87
|
+
end
|
88
|
+
|
89
|
+
def zoomdata
|
90
|
+
end
|
91
|
+
|
92
|
+
|
93
|
+
|
94
|
+
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
data/lib/dscli/core.rb
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
class Historics < Thor
|
2
|
+
|
3
|
+
desc 'list [page]', 'Lists all historic queries'
|
4
|
+
def list(page = 1)
|
5
|
+
api = Dscli::API.new
|
6
|
+
results = api.historics_list(page)
|
7
|
+
|
8
|
+
puts "\nTotal Historic Queries: #{results.count}\n\n"
|
9
|
+
puts 'ID | Name | Created | Definition Hash | Status '
|
10
|
+
puts '---------------------------------------------------------------------------------------------------------------------------'
|
11
|
+
|
12
|
+
results[:data].each { |s| puts "#{s[:id]} | #{ '%-20.20s' % s[:name] } | #{Time.at(s[:created_at])} | #{ s[:definition_id] } | #{s[:status]}" }
|
13
|
+
puts "\n"
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
desc 'get (id)', 'Gets details of a historic query'
|
18
|
+
def get(id)
|
19
|
+
|
20
|
+
begin
|
21
|
+
api = Dscli::API.new
|
22
|
+
response = api.historics_get(id)
|
23
|
+
puts response[:data].to_yaml
|
24
|
+
rescue ApiResourceNotFoundError => e
|
25
|
+
puts "Specified historic query '#{id}' was not found. It may have already been deleted."
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
desc 'stop (id)', 'Stops a historic query'
|
31
|
+
def stop(id)
|
32
|
+
api = Dscli::API.new
|
33
|
+
|
34
|
+
begin
|
35
|
+
response = api.historics_stop(id)
|
36
|
+
|
37
|
+
if response[:http][:status] == 204
|
38
|
+
puts "Historic query '#{id}' stopped successfully"
|
39
|
+
else
|
40
|
+
# TODO: How do we handle a different code?
|
41
|
+
response
|
42
|
+
end
|
43
|
+
rescue ApiResourceNotFoundError => e
|
44
|
+
puts "Specified historic query '#{id}' not found. It may have been deleted."
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
|
49
|
+
desc 'delete (id)', 'Deletes a historic query'
|
50
|
+
def delete(id)
|
51
|
+
api = Dscli::API.new
|
52
|
+
|
53
|
+
begin
|
54
|
+
|
55
|
+
response = api.historics_delete(id)
|
56
|
+
|
57
|
+
if response[:http][:status] == 204
|
58
|
+
puts "Historic query '#{id}' deleted successfully"
|
59
|
+
else
|
60
|
+
response
|
61
|
+
end
|
62
|
+
rescue ApiResourceNotFoundError => e
|
63
|
+
puts "Specified historic query '#{id}' not found. It may have already been deleted."
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
@@ -0,0 +1,159 @@
|
|
1
|
+
require 'interact'
|
2
|
+
|
3
|
+
module Dscli
|
4
|
+
class OutputParams
|
5
|
+
include Interactive
|
6
|
+
|
7
|
+
FORMATS = ["json_new_line", "json_meta", "json_array"]
|
8
|
+
S3_ACL = ["private", "public-read", "public-read-write", "authenticated-read", "bucket-owner-read", "bucket-owner-full-control"]
|
9
|
+
HTTP_AUTH_TYPE = ["none", "basic"]
|
10
|
+
DELIVERY_FREQUENCY = [0, 10, 30, 60, 300]
|
11
|
+
MAX_SIZE = ["100KB", "250KB", "500KB", "1MB", "2MB", "5MB", "10MB", "20MB"]
|
12
|
+
HTTP_METHOD = ["POST", "PUT"]
|
13
|
+
BOOL = ["true", "false"]
|
14
|
+
|
15
|
+
def acl
|
16
|
+
ask "ACL", choices: S3_ACL.each, indexed: true, default: S3_ACL.first
|
17
|
+
end
|
18
|
+
|
19
|
+
def api_hostname
|
20
|
+
ask "Splunk Storm REST API host"
|
21
|
+
end
|
22
|
+
|
23
|
+
def auth_access_key
|
24
|
+
ask "AWS access key"
|
25
|
+
end
|
26
|
+
|
27
|
+
def auth_access_token
|
28
|
+
ask "Splunk Storm REST access token"
|
29
|
+
end
|
30
|
+
|
31
|
+
def auth_password
|
32
|
+
ask "Authentication password"
|
33
|
+
end
|
34
|
+
|
35
|
+
def auth_secret_key
|
36
|
+
ask "AWS secret key"
|
37
|
+
end
|
38
|
+
|
39
|
+
def auth_username
|
40
|
+
ask "Authentication username"
|
41
|
+
end
|
42
|
+
|
43
|
+
def auth_type
|
44
|
+
ask "Auth type", choices: HTTP_AUTH_TYPE.each, indexed: true, default: HTTP_AUTH_TYPE.first
|
45
|
+
end
|
46
|
+
|
47
|
+
def bucket
|
48
|
+
ask "Bucket name"
|
49
|
+
end
|
50
|
+
|
51
|
+
def collection_name
|
52
|
+
ask "MongoDB Collection name"
|
53
|
+
end
|
54
|
+
|
55
|
+
def database
|
56
|
+
ask "Numeric ID of your existing Redis database"
|
57
|
+
end
|
58
|
+
|
59
|
+
def db_name
|
60
|
+
ask "Database name"
|
61
|
+
end
|
62
|
+
|
63
|
+
def delivery_frequency
|
64
|
+
ask "Delivery frequency (seconds)", choices: DELIVERY_FREQUENCY.each, indexed: true, default: DELIVERY_FREQUENCY.first
|
65
|
+
end
|
66
|
+
|
67
|
+
def directory
|
68
|
+
ask "Directory"
|
69
|
+
end
|
70
|
+
|
71
|
+
def file_prefix
|
72
|
+
ask "Optional filename prefix", default: "DataSift"
|
73
|
+
end
|
74
|
+
|
75
|
+
def format
|
76
|
+
ask "Output format", choices: FORMATS.each, indexed: true
|
77
|
+
end
|
78
|
+
|
79
|
+
def host
|
80
|
+
ask "The host to connect to"
|
81
|
+
end
|
82
|
+
|
83
|
+
def list
|
84
|
+
ask "Name of a Redis list to stores interactions"
|
85
|
+
end
|
86
|
+
|
87
|
+
def max_size
|
88
|
+
max_size = ask "Max delivery size", choices: MAX_SIZE.each, indexed: true, default: MAX_SIZE.last
|
89
|
+
human_size_to_number(max_size)
|
90
|
+
end
|
91
|
+
|
92
|
+
def method
|
93
|
+
ask "HTTP request type", choices: HTTP_METHOD.each, indexed: true, default: HTTP_METHOD.first
|
94
|
+
end
|
95
|
+
|
96
|
+
def port
|
97
|
+
ask "Port on the remote machine"
|
98
|
+
end
|
99
|
+
|
100
|
+
def project_id
|
101
|
+
ask "ID of your Splunk Storm REST project"
|
102
|
+
end
|
103
|
+
|
104
|
+
def region
|
105
|
+
ask "AWS region the destination table is set up in"
|
106
|
+
end
|
107
|
+
|
108
|
+
def server_level_auth_password
|
109
|
+
ask "Server-level authentication password"
|
110
|
+
end
|
111
|
+
|
112
|
+
def server_level_auth_username
|
113
|
+
ask "Server-level authentication username"
|
114
|
+
end
|
115
|
+
|
116
|
+
def source
|
117
|
+
ask "Label used to mark data delivered to Zoomdata"
|
118
|
+
end
|
119
|
+
|
120
|
+
def table
|
121
|
+
ask "Name of the table to use"
|
122
|
+
end
|
123
|
+
|
124
|
+
def type
|
125
|
+
ask "The type that you want to use for the index"
|
126
|
+
end
|
127
|
+
|
128
|
+
def url
|
129
|
+
ask "The URL to post data to"
|
130
|
+
end
|
131
|
+
|
132
|
+
def use_gzip
|
133
|
+
ask "Use GZip?", choices: BOOL.each, indexed: true, default: BOOL.last
|
134
|
+
end
|
135
|
+
|
136
|
+
def use_ssl
|
137
|
+
ask "Use SSL?", choices: BOOL.each, indexed: true, default: BOOL.last
|
138
|
+
end
|
139
|
+
|
140
|
+
def verify_ssl
|
141
|
+
ask "Verify SSL certificate?", choices: BOOL.each, indexed: true, default: BOOL.last
|
142
|
+
end
|
143
|
+
|
144
|
+
###############
|
145
|
+
|
146
|
+
#TODO properly add back in the namespacing. i.e. op_params.auth.passw
|
147
|
+
|
148
|
+
private
|
149
|
+
|
150
|
+
def human_size_to_number(size)
|
151
|
+
value = size.scan(/[A-Z]+\d*|\d+/)
|
152
|
+
bytes = value[0].to_i * 1024 if value[1] == "KB"
|
153
|
+
bytes = value[0].to_i * 1024 * 1024 if value[1] == "MB"
|
154
|
+
return bytes.to_s
|
155
|
+
end
|
156
|
+
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
data/lib/dscli/push.rb
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
class Push < Thor
|
2
|
+
|
3
|
+
desc 'list [page]', 'Lists all current subscriptions'
|
4
|
+
def list(page = 1)
|
5
|
+
api = Dscli::API.new
|
6
|
+
results = api.push_list(page)
|
7
|
+
|
8
|
+
puts "\nTotal Subscriptions: #{results[:subscriptions].count}\n\n"
|
9
|
+
if results[:subscriptions].count > 0
|
10
|
+
puts 'ID | Name | Output Type | Created | Status '
|
11
|
+
puts '--------------------------------------------------------------------------------------------------------------'
|
12
|
+
|
13
|
+
results[:subscriptions].each { |s| puts "#{s[:id]} | #{ '%-20.20s' % s[:name] } | #{ '%-11.11s' % s[:output_type] } | #{Time.at(s[:created_at])} | #{s[:status]}" }
|
14
|
+
puts "\n"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
desc 'get (id)', 'Gets details of a push subscription'
|
19
|
+
def get(id)
|
20
|
+
begin
|
21
|
+
api = Dscli::API.new
|
22
|
+
response = api.push_get(id)
|
23
|
+
puts response[:data].to_yaml
|
24
|
+
rescue ApiResourceNotFoundError => e
|
25
|
+
puts "Specified push subscription '#{id}' not found. It may have already been deleted."
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
desc 'stop (id)', 'Stops a push subscription'
|
30
|
+
def stop(id)
|
31
|
+
begin
|
32
|
+
api = Dscli::API.new
|
33
|
+
api.push_stop(id)
|
34
|
+
rescue ApiResourceNotFoundError => e
|
35
|
+
puts "Specified push subscription '#{id}' not found. It may have been deleted."
|
36
|
+
rescue BadRequestError => e
|
37
|
+
puts "Specified push subscription '#{id}' could not be stopped. It may have already been stopped."
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
desc 'delete (id)', 'Deletes a push subscription'
|
42
|
+
def delete(id)
|
43
|
+
api = Dscli::API.new
|
44
|
+
|
45
|
+
begin
|
46
|
+
response = api.push_delete(id)
|
47
|
+
|
48
|
+
if response[:http][:status] == 204
|
49
|
+
puts "Push subscription #{id} deleted successfully"
|
50
|
+
else
|
51
|
+
# TODO: How do we handle a different code?
|
52
|
+
response
|
53
|
+
end
|
54
|
+
rescue ApiResourceNotFoundError => e
|
55
|
+
puts "Specified push subscription '#{id}' not found. It may have already been deleted."
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
desc 'log [id]', 'Retrieves logs for all subscriptions (or for the subscription ID supplied)'
|
60
|
+
def log(id = nil)
|
61
|
+
api = Dscli::API.new
|
62
|
+
|
63
|
+
begin
|
64
|
+
results = api.push_log(id)
|
65
|
+
|
66
|
+
puts "\nMessage count: #{results[:count]}\n\n"
|
67
|
+
|
68
|
+
if results[:count] > 0
|
69
|
+
puts ' Time | Subscription ID | Message '
|
70
|
+
puts '-----------------------------------------------------------------------------------------------------------------'
|
71
|
+
|
72
|
+
results[:log_entries].each { |s| puts "#{Time.at(s[:request_time]) } | #{s[:subscription_id]} | #{ '%-50.50s' % s[:message] }" }
|
73
|
+
puts "\n"
|
74
|
+
end
|
75
|
+
rescue ApiResourceNotFoundError => e
|
76
|
+
puts "Specified push subscription '#{id}' not found. It may have been deleted."
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
data/lib/dscli/source.rb
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
class Source < Thor
|
2
|
+
|
3
|
+
desc 'list [page]', 'Lists all managed sources'
|
4
|
+
def list(page = 1)
|
5
|
+
api = Dscli::API.new
|
6
|
+
results = api.source_list(page)
|
7
|
+
|
8
|
+
puts "\nTotal Managed Sources: #{results.count}\n\n"
|
9
|
+
puts 'ID | Name | Type | Status'
|
10
|
+
puts '-----------------------------------------------------------------------------------------------------'
|
11
|
+
|
12
|
+
results[:sources].each { |s| puts "#{s[:id]} | #{ '%-20.20s' % s[:name] } | #{ '%-20.20s' % s[:source_type] } | #{s[:status]}" }
|
13
|
+
puts "\n"
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
desc 'get (id)', 'Gets details of a managed source'
|
18
|
+
def get(id)
|
19
|
+
|
20
|
+
begin
|
21
|
+
api = Dscli::API.new
|
22
|
+
response = api.source_get(id)
|
23
|
+
puts response[:data].to_yaml
|
24
|
+
rescue ApiResourceNotFoundError => e
|
25
|
+
puts "Specified managed source '#{id}' was not found. It may have been deleted."
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
desc 'start (id)', 'Starts the specified source'
|
31
|
+
def start(id)
|
32
|
+
|
33
|
+
api = Dscli::API.new
|
34
|
+
|
35
|
+
begin
|
36
|
+
response = api.source_start(id)
|
37
|
+
|
38
|
+
if response[:http][:status] == 200
|
39
|
+
puts "Managed source '#{id}' started successfully"
|
40
|
+
else
|
41
|
+
# TODO: How do we handle a different code?
|
42
|
+
response
|
43
|
+
end
|
44
|
+
rescue ApiResourceNotFoundError => e
|
45
|
+
puts "Specified source '#{id}' not found. It may have been deleted."
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
desc 'stop (id)', 'Stops the specified source'
|
51
|
+
def stop(id)
|
52
|
+
|
53
|
+
api = Dscli::API.new
|
54
|
+
|
55
|
+
begin
|
56
|
+
response = api.source_stop(id)
|
57
|
+
|
58
|
+
if response[:http][:status] == 200
|
59
|
+
puts "Managed source '#{id}' stopped successfully"
|
60
|
+
else
|
61
|
+
# TODO: How do we handle a different code?
|
62
|
+
response
|
63
|
+
end
|
64
|
+
rescue ApiResourceNotFoundError => e
|
65
|
+
puts "Specified source '#{id}' not found. It may have been deleted."
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
|
70
|
+
desc 'delete (id)', 'Deletes a managed source'
|
71
|
+
def delete(id)
|
72
|
+
api = Dscli::API.new
|
73
|
+
|
74
|
+
begin
|
75
|
+
|
76
|
+
response = api.source_delete(id)
|
77
|
+
|
78
|
+
if response[:http][:status] == 204
|
79
|
+
puts "Source '#{id}' deleted successfully"
|
80
|
+
else
|
81
|
+
response
|
82
|
+
end
|
83
|
+
rescue ApiResourceNotFoundError => e
|
84
|
+
puts "Source '#{id}' not found. It may have been deleted."
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
88
|
+
|
89
|
+
desc 'logs (id)', 'Retrieves the log for a managed source'
|
90
|
+
def logs(id = nil)
|
91
|
+
api = Dscli::API.new
|
92
|
+
|
93
|
+
begin
|
94
|
+
|
95
|
+
results = api.source_log(id)
|
96
|
+
|
97
|
+
puts "\nMessage count: #{results[:count]}\n\n"
|
98
|
+
puts ' Time | Message '
|
99
|
+
puts '-----------------------------------------------------------------------------------------------------------------'
|
100
|
+
|
101
|
+
results[:log_entries].each { |s| puts "#{Time.at(s[:event_time]) } | #{ '%-50.50s' % s[:message] }" }
|
102
|
+
puts "\n"
|
103
|
+
|
104
|
+
rescue ApiResourceNotFoundError => e
|
105
|
+
puts "Specified source '#{id}' not found. It may have been deleted."
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
|
110
|
+
end
|
data/lib/dscli/storage.rb
CHANGED
@@ -19,34 +19,11 @@ module Dscli
|
|
19
19
|
f.write(Yajl::Encoder.encode(auth) + "\n")
|
20
20
|
}
|
21
21
|
end
|
22
|
-
#TODO Set 'else' to write over config if it already exists
|
23
22
|
end
|
24
23
|
|
25
24
|
def get_auth
|
26
25
|
config = YAML.load(File.read(@configfile))
|
27
26
|
end
|
28
27
|
|
29
|
-
# def add_push_endpoint(config)
|
30
|
-
# File.open(@configfile, 'a') { |f|
|
31
|
-
# f.puts Yajl::Encoder.encode(config) + "\n"
|
32
|
-
# }
|
33
|
-
# end
|
34
|
-
|
35
|
-
# def get_push_endpoint(name)
|
36
|
-
# parser = Yajl::Parser
|
37
|
-
# File.open(@configfile, 'r').each_line do |line|
|
38
|
-
# json = parser.parse(line)
|
39
|
-
# if json['name'] && json['name'] == name
|
40
|
-
# return json
|
41
|
-
# end
|
42
|
-
# end
|
43
|
-
# return ''
|
44
|
-
# end
|
45
|
-
|
46
|
-
# def list_push_endpoints
|
47
|
-
#
|
48
|
-
# end
|
49
|
-
#
|
50
|
-
#
|
51
28
|
end
|
52
29
|
end
|
data/lib/dscli/version.rb
CHANGED
data/lib/dscli.rb
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
dir = File.dirname(__FILE__)
|
2
|
+
require dir + '/Dscli/push'
|
3
|
+
require dir + '/Dscli/historics'
|
4
|
+
require dir + '/Dscli/source'
|
5
|
+
require dir + '/Dscli/core'
|
6
|
+
require dir + '/Dscli/cli'
|
2
7
|
require dir + '/Dscli/version'
|
3
|
-
require dir + '/Dscli/commands'
|
4
8
|
require dir + '/Dscli/api'
|
5
9
|
require dir + '/Dscli/storage'
|
6
10
|
require dir + '/Dscli/parameters'
|
metadata
CHANGED
@@ -1,97 +1,111 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dscli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Dugdale
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-08-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: thor
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: yajl-ruby
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: datasift
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: '0'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
82
|
+
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: interact
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '0'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- -
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: multi_json
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
95
109
|
- !ruby/object:Gem::Version
|
96
110
|
version: '0'
|
97
111
|
description: DS CLI is a CLI wrapper for the DataSift Ruby client library, allowing
|
@@ -103,15 +117,21 @@ executables:
|
|
103
117
|
extensions: []
|
104
118
|
extra_rdoc_files: []
|
105
119
|
files:
|
120
|
+
- LICENSE.txt
|
121
|
+
- README.md
|
106
122
|
- bin/dscli
|
123
|
+
- lib/dscli.rb
|
107
124
|
- lib/dscli/api.rb
|
108
|
-
- lib/dscli/
|
125
|
+
- lib/dscli/cli.rb
|
126
|
+
- lib/dscli/connectors.rb
|
127
|
+
- lib/dscli/core.rb
|
128
|
+
- lib/dscli/historics.rb
|
129
|
+
- lib/dscli/outputParams.rb
|
109
130
|
- lib/dscli/parameters.rb
|
131
|
+
- lib/dscli/push.rb
|
132
|
+
- lib/dscli/source.rb
|
110
133
|
- lib/dscli/storage.rb
|
111
134
|
- lib/dscli/version.rb
|
112
|
-
- lib/dscli.rb
|
113
|
-
- LICENSE.txt
|
114
|
-
- README.md
|
115
135
|
homepage: https://github.com/dugjason/dscli
|
116
136
|
licenses:
|
117
137
|
- BSD
|
@@ -122,17 +142,17 @@ require_paths:
|
|
122
142
|
- lib
|
123
143
|
required_ruby_version: !ruby/object:Gem::Requirement
|
124
144
|
requirements:
|
125
|
-
- -
|
145
|
+
- - ">="
|
126
146
|
- !ruby/object:Gem::Version
|
127
147
|
version: '0'
|
128
148
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
129
149
|
requirements:
|
130
|
-
- -
|
150
|
+
- - ">="
|
131
151
|
- !ruby/object:Gem::Version
|
132
152
|
version: '0'
|
133
153
|
requirements: []
|
134
154
|
rubyforge_project:
|
135
|
-
rubygems_version: 2.0
|
155
|
+
rubygems_version: 2.3.0
|
136
156
|
signing_key:
|
137
157
|
specification_version: 4
|
138
158
|
summary: Simple CLI wrapper for the DataSift Ruby client library
|
data/lib/dscli/commands.rb
DELETED
@@ -1,69 +0,0 @@
|
|
1
|
-
require 'thor'
|
2
|
-
require 'yaml'
|
3
|
-
|
4
|
-
module Dscli
|
5
|
-
class Commands < Thor
|
6
|
-
include Thor::Actions
|
7
|
-
|
8
|
-
def self.source_root
|
9
|
-
File.dirname(__FILE__)
|
10
|
-
end
|
11
|
-
|
12
|
-
desc 'config', 'Configure your DataSift API credentials'
|
13
|
-
option :user, type: :boolean, desc: 'Enter username and API key for your DataSift API user'
|
14
|
-
#option :push, type: :boolean, desc: 'Enter details of a new Push Destination'
|
15
|
-
|
16
|
-
def config
|
17
|
-
|
18
|
-
#if options[:user]
|
19
|
-
open(ENV['HOME'] + '/.datasiftcli', 'w') do |f|
|
20
|
-
f.write Dscli::Parameters.new.user_config(options).to_yaml
|
21
|
-
end
|
22
|
-
#end
|
23
|
-
|
24
|
-
#TODO Add other config options such as new push destinations
|
25
|
-
end
|
26
|
-
|
27
|
-
#
|
28
|
-
# Core API endpoints
|
29
|
-
# [ Missing /validate /balance ]
|
30
|
-
#
|
31
|
-
|
32
|
-
desc 'compile (csdl)', 'Compile CSDL'
|
33
|
-
def compile(csdl)
|
34
|
-
api = Dscli::API.new
|
35
|
-
definition = api.compile(csdl)
|
36
|
-
puts definition
|
37
|
-
end
|
38
|
-
|
39
|
-
desc 'usage [period]', 'Find your stream and license fee usage'
|
40
|
-
def usage(period = 'day')
|
41
|
-
api = Dscli::API.new
|
42
|
-
puts Yajl::Encoder.encode(api.usage(period), :pretty => true)
|
43
|
-
end
|
44
|
-
|
45
|
-
desc 'dpu (hash)', 'Find the DPU cost of a given stream hash'
|
46
|
-
def dpu(hash)
|
47
|
-
api = Dscli::API.new
|
48
|
-
puts Yajl::Encoder.encode(api.dpu(hash), :pretty => true)
|
49
|
-
end
|
50
|
-
|
51
|
-
desc 'stream (hash)', 'Stream interactions from a DataSift stream to the command line'
|
52
|
-
def stream(hash)
|
53
|
-
api = Dscli::API.new
|
54
|
-
api.stream(hash)
|
55
|
-
end
|
56
|
-
|
57
|
-
#
|
58
|
-
# Push API endpoints
|
59
|
-
#
|
60
|
-
#
|
61
|
-
|
62
|
-
#desc 'pushcreate', ''
|
63
|
-
#def pushcreate(name)
|
64
|
-
# api = Dscli::API.new
|
65
|
-
# api.pushcreate(name)
|
66
|
-
#end
|
67
|
-
|
68
|
-
end
|
69
|
-
end
|