gooddata 0.6.0.pre6 → 0.6.0.pre7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -27,19 +27,18 @@ module GoodData::Bricks
27
27
  :oauth_token => oauth_token,
28
28
  :refresh_token => refresh_token
29
29
  }
30
- else
31
- fail "Salesforce middleware failed while trying to log in. Either salesforce_username, salesforce_password, salesforce_token or salesforce_oauth_token, salesforce_refresh_token are needed. Additionally you have to specify salesforce_client_id and salesforce_client_secret parameters in both cases"
32
30
  end
33
31
 
32
+ client = if credentials
33
+ credentials.merge!({
34
+ :client_id => client_id,
35
+ :client_secret => client_secret,
36
+ })
37
+ credentials[:host] = host unless host.nil?
34
38
 
35
- credentials.merge!({
36
- :client_id => client_id,
37
- :client_secret => client_secret,
38
- })
39
- credentials[:host] = host unless host.nil?
40
-
41
- Restforce.log = true if params[:salesforce_client_logger]
42
- client = Restforce.new(credentials)
39
+ Restforce.log = true if params[:salesforce_client_logger]
40
+ Restforce.new(credentials)
41
+ end
43
42
  @app.call(params.merge(:salesforce_client => client))
44
43
  end
45
44
 
@@ -1,5 +1,6 @@
1
1
  require 'gooddata/version'
2
2
  require 'gooddata/connection'
3
+ require 'gooddata/helpers'
3
4
 
4
5
  # fastercsv is built in Ruby 1.9
5
6
  if RUBY_VERSION < "1.9"
@@ -94,6 +95,17 @@ module GoodData
94
95
 
95
96
  end
96
97
 
98
+ def logging_on
99
+ if logger.is_a? NilLogger
100
+ GoodData::logger = Logger.new(STDOUT)
101
+ end
102
+ end
103
+
104
+ def logging_off
105
+ GoodData::logger = NilLogger.new
106
+ end
107
+
108
+
97
109
  # Hepler for starting with SST easier
98
110
  # === Parameters
99
111
  #
@@ -123,7 +135,7 @@ module GoodData
123
135
  old_project = GoodData.project
124
136
  begin
125
137
  GoodData.use(project)
126
- bl.call(project)
138
+ bl.call(GoodData.project)
127
139
  rescue Exception => e
128
140
  fail e
129
141
  ensure
@@ -251,6 +263,14 @@ module GoodData
251
263
  }))
252
264
  end
253
265
 
266
+ def download_form_user_webdav(file, where, options={})
267
+ u = URI(connection.options[:webdav_server] || GoodData.project.links["uploads"])
268
+ url = URI.join(u.to_s.chomp(u.path.to_s), "/uploads/")
269
+ connection.download(file, where, options.merge({
270
+ :staging_url => url
271
+ }))
272
+ end
273
+
254
274
  def poll(result, key, options={})
255
275
  sleep_interval = options[:sleep_interval] || 10
256
276
  link = result[key]["links"]["poll"]
@@ -1,80 +1,88 @@
1
1
  module GoodData::Command
2
- class Auth < Base
3
- def connect
4
- unless defined? @connected
5
- GoodData.connect user, password, url, :auth_token => auth_token
6
- @connected = true
2
+ class Auth
3
+
4
+ class << self
5
+ def connect
6
+ unless defined? @connected
7
+ GoodData.connect({
8
+ :login => user,
9
+ :password => password,
10
+ :server => url,
11
+ :auth_token => auth_token
12
+ })
13
+ @connected = true
14
+ end
15
+ @connected
7
16
  end
8
- @connected
9
- end
10
-
11
- def user
12
- ensure_credentials
13
- @credentials[:username]
14
- end
15
-
16
- def password
17
- ensure_credentials
18
- @credentials[:password]
19
- end
20
17
 
21
- def url
22
- ensure_credentials
23
- @credentials[:url]
24
- end
18
+ def user
19
+ ensure_credentials
20
+ @credentials[:username]
21
+ end
25
22
 
26
- def auth_token
27
- ensure_credentials
28
- @credentials[:auth_token]
29
- end
23
+ def password
24
+ ensure_credentials
25
+ @credentials[:password]
26
+ end
30
27
 
31
- def credentials_file
32
- "#{home_directory}/.gooddata"
33
- end
28
+ def url
29
+ ensure_credentials
30
+ @credentials[:url]
31
+ end
34
32
 
35
- def ensure_credentials
36
- return if defined? @credentials
37
- unless @credentials = read_credentials
38
- @credentials = ask_for_credentials
33
+ def auth_token
34
+ ensure_credentials
35
+ @credentials[:auth_token]
39
36
  end
40
- @credentials
41
- end
42
37
 
43
- def read_credentials
44
- if File.exists?(credentials_file) then
45
- config = File.read(credentials_file)
46
- JSON.parser.new(config, :symbolize_names => true).parse
38
+ def credentials_file
39
+ "#{GoodData::Helpers.home_directory}/.gooddata"
47
40
  end
48
- end
49
41
 
50
- def ask_for_credentials
51
- puts "Enter your GoodData credentials."
52
- user = ask("Email")
53
- password = ask("Password", :secret => true)
54
- auth_token = ask("Authorization Token")
55
- { :username => user, :password => password, :auth_token => auth_token }
56
- end
42
+ def ensure_credentials
43
+ return if defined? @credentials
44
+ unless @credentials = read_credentials
45
+ @credentials = ask_for_credentials
46
+ end
47
+ @credentials
48
+ end
57
49
 
58
- def store
59
- credentials = ask_for_credentials
50
+ def read_credentials
51
+ if File.exists?(credentials_file) then
52
+ config = File.read(credentials_file)
53
+ JSON.parser.new(config, :symbolize_names => true).parse
54
+ end
55
+ end
60
56
 
61
- ovewrite = if File.exist?(credentials_file)
62
- ask "Overwrite existing stored credentials", :answers => %w(y n)
63
- else
64
- 'y'
57
+ def ask_for_credentials
58
+ puts "Enter your GoodData credentials."
59
+ user = HighLine::ask("Email")
60
+ password = HighLine::ask("Password") { |q| q.echo = "x" }
61
+ auth_token = HighLine::ask("Authorization Token")
62
+ { :username => user, :password => password, :auth_token => auth_token }
65
63
  end
66
64
 
67
- if ovewrite == 'y'
68
- File.open(credentials_file, 'w', 0600) do |f|
69
- f.puts JSON.pretty_generate(credentials)
65
+ def store
66
+ credentials = ask_for_credentials
67
+
68
+ ovewrite = if File.exist?(credentials_file)
69
+ HighLine::ask("Overwrite existing stored credentials (y/n)")
70
+ # { |q| q.validate = /[y,n]/ }
71
+ else
72
+ 'y'
73
+ end
74
+ if ovewrite == 'y'
75
+ File.open(credentials_file, 'w', 0600) do |f|
76
+ f.puts JSON.pretty_generate(credentials)
77
+ end
78
+ else
79
+ puts 'Aborting...'
70
80
  end
71
- else
72
- puts 'Aborting...'
73
81
  end
74
- end
75
82
 
76
- def unstore
77
- FileUtils.rm_f(credentials_file)
83
+ def unstore
84
+ FileUtils.rm_f(credentials_file)
85
+ end
78
86
  end
79
87
  end
80
88
  end
@@ -2,74 +2,42 @@ module GoodData::Command
2
2
  class Process
3
3
 
4
4
  def self.list(options={})
5
- # with project usage
6
- processes = GoodData::Process[:all]
5
+ GoodData.with_project(options[:project_id]) do
6
+ processes = GoodData::Process[:all]
7
+ end
7
8
  end
8
9
 
9
10
  def self.get(options={})
10
11
  id = options[:process_id]
11
12
  fail "Unspecified process id" if id.nil?
12
- GoodData::Process[id]
13
+
14
+ GoodData.with_project(options[:project_id]) do
15
+ GoodData::Process[id]
16
+ end
13
17
  end
14
18
 
15
- def self.with_deploy(dir, options={}, &block)
19
+ def self.deploy(dir, options={})
16
20
  verbose = options[:verbose] || false
17
- if block
18
- begin
19
- res = deploy_graph(dir, options)
20
- block.call(res)
21
- ensure
22
- # self_link = res["process"]["links"]["self"]
23
- # GoodData.delete(self_link)
24
- end
25
- else
26
- deploy_graph(dir, options)
21
+ GoodData.with_project(options[:project_id]) do
22
+ deploy_graph(dir, options.merge({:files_to_exclude => [options[:params]]}))
27
23
  end
28
24
  end
29
25
 
30
- def self.deploy_graph(dir, options={})
31
- dir = Pathname(dir) || fail("Directory is not specified")
32
- fail "\"#{dir}\" is not a directory" unless dir.directory?
33
- project_id = options[:project_id] || fail("Project Id has to be specified")
34
-
35
-
36
- type = options[:type] || fail("Type of deployment is not specified")
37
- deploy_name = options[:name]
26
+ def self.with_deploy(dir, options={}, &block)
38
27
  verbose = options[:verbose] || false
39
- project_pid = options[:project_pid]
40
- puts HighLine::color("Deploying #{dir}", HighLine::BOLD) if verbose
41
- res = nil
42
-
43
- Tempfile.open("deploy-graph-archive") do |temp|
44
-
45
- Zip::OutputStream.open(temp.path) do |zio|
46
- Dir.glob(dir + "**/*") do |item|
47
- puts "including #{item}" if verbose
48
- unless File.directory?(item)
49
- zio.put_next_entry(item)
50
- zio.print IO.read(item)
51
- end
28
+ GoodData.with_project(options[:project_id]) do
29
+ if block
30
+ begin
31
+ res = deploy_graph(dir, options.merge({:files_to_exclude => [options[:params]]}))
32
+ block.call(res)
33
+ ensure
34
+ # self_link = res["process"]["links"]["self"]
35
+ # GoodData.delete(self_link)
52
36
  end
53
- end
54
-
55
- GoodData.upload_to_user_webdav(temp.path)
56
- process_id = options[:process]
57
-
58
- data = {
59
- :process => {
60
- :name => deploy_name,
61
- :path => "/uploads/#{File.basename(temp.path)}",
62
- :type => type
63
- }
64
- }
65
- res = if process_id.nil?
66
- GoodData.post("/gdc/projects/#{project_pid}/dataload/processes", data)
67
37
  else
68
- GoodData.put("/gdc/projects/#{project_pid}/dataload/processes/#{process_id}", data)
38
+ deploy_graph(dir, options.merge({:files_to_exclude => [options[:params]]}))
69
39
  end
70
40
  end
71
- puts HighLine::color("Deploy DONE #{dir}", HighLine::BOLD) if verbose
72
- res
73
41
  end
74
42
 
75
43
  def self.execute_process(link, dir, options={})
@@ -78,8 +46,8 @@ module GoodData::Command
78
46
  if type == :ruby
79
47
  result = GoodData.post(link, {
80
48
  :execution => {
81
- :graph => (dir + "main.rb").to_s,
82
- :params => options[:params]
49
+ :graph => ("./main.rb").to_s,
50
+ :params => options[:expanded_params]
83
51
  }
84
52
  })
85
53
  begin
@@ -122,16 +90,61 @@ module GoodData::Command
122
90
 
123
91
  with_deploy(dir, options.merge(:name => name)) do |deploy_response|
124
92
  puts HighLine::color("Executing", HighLine::BOLD) if verbose
125
- # if email.nil?
126
- # result = execute_process(deploy_response["process"]["links"]["executions"], dir, options)
127
- # else
128
- # create_email_channel(options) do |channel_response|
129
- # subscribe_on_finish(:success, channel_response, deploy_response, options)
130
- result = execute_process(deploy_response["process"]["links"]["executions"], dir, options)
131
- # end
132
- # end
93
+ result = execute_process(deploy_response["process"]["links"]["executions"], dir, options)
133
94
  end
134
95
  end
135
96
 
97
+ private
98
+ def self.deploy_graph(dir, options={})
99
+ dir = Pathname(dir) || fail("Directory is not specified")
100
+ fail "\"#{dir}\" is not a directory" unless dir.directory?
101
+ files_to_exclude = options[:files_to_exclude].map {|p| Pathname(p)}
102
+
103
+ # project_id = options[:project_id] || fail("Project Id has to be specified")
104
+
105
+
106
+ type = options[:type] || "GRAPH"
107
+ deploy_name = options[:name]
108
+ verbose = options[:verbose] || false
109
+
110
+ puts HighLine::color("Deploying #{dir}", HighLine::BOLD) if verbose
111
+ res = nil
112
+
113
+ Tempfile.open("deploy-graph-archive") do |temp|
114
+ Zip::OutputStream.open(temp.path) do |zio|
115
+ FileUtils::cd(dir) do
116
+
117
+ files_to_pack = Dir.glob("./**/*").reject {|f| files_to_exclude.include?(Pathname(dir) + f)}
118
+ files_to_pack.each do |item|
119
+ puts "including #{item}" if verbose
120
+ unless File.directory?(item)
121
+ zio.put_next_entry(item)
122
+ zio.print IO.read(item)
123
+ end
124
+ end
125
+ end
126
+ end
127
+
128
+ GoodData.upload_to_user_webdav(temp.path)
129
+ process_id = options[:process]
130
+
131
+ data = {
132
+ :process => {
133
+ :name => deploy_name,
134
+ :path => "/uploads/#{File.basename(temp.path)}",
135
+ :type => type
136
+ }
137
+ }
138
+ res = if process_id.nil?
139
+ GoodData.post("/gdc/projects/#{GoodData.project.pid}/dataload/processes", data)
140
+ else
141
+ GoodData.put("/gdc/projects/#{GoodData.project.pid}/dataload/processes/#{process_id}", data)
142
+ end
143
+ end
144
+ puts HighLine::color("Deploy DONE #{dir}", HighLine::BOLD) if verbose
145
+ res
146
+ end
147
+
148
+
136
149
  end
137
150
  end
@@ -4,18 +4,18 @@ module GoodData::Command
4
4
  class Runners
5
5
 
6
6
  def self.run_ruby_locally(brick_dir, options={})
7
- pid = options[:project]
7
+ pid = options[:project_id]
8
8
  fail "You have to specify a project ID" if pid.nil?
9
9
  fail "You have to specify directory of the brick run" if brick_dir.nil?
10
10
  fail "You specified file as a birck run directory. You have to specify directory." if File.exist?(brick_dir) && !File.directory?(brick_dir)
11
11
 
12
- params = options[:params] || {}
12
+ params = options[:expanded_params] || {}
13
13
 
14
14
  GoodData.connection.connect!
15
15
  sst = GoodData.connection.cookies[:cookies]["GDCAuthSST"]
16
16
  pwd = Pathname.new(Dir.pwd)
17
17
  logger_stream = STDOUT
18
-
18
+
19
19
  server_uri = URI(options[:server]) unless options[:server].nil?
20
20
  scheme = server_uri.nil? ? "" : server_uri.scheme
21
21
  hostname = server_uri.nil? ? "" : server_uri.host
@@ -24,12 +24,15 @@ script_body = <<-script_body
24
24
  require 'fileutils'
25
25
  FileUtils::cd(\"#{pwd+brick_dir}\") do\
26
26
  require 'bundler/setup'
27
- eval(File.read(\"main.rb\")).call({
27
+
28
+ $SCRIPT_PARAMS = {
28
29
  :GDC_SST => \"#{sst}\",
29
30
  :GDC_PROJECT_ID => \"#{pid}\",
30
31
  :GDC_PROTOCOL => \"#{scheme}\",
31
- :GDC_SERVER => \"#{hostname}\"
32
- }.merge(#{params}))
32
+ :GDC_SERVER => \"#{hostname}\",
33
+ :GDC_LOGGER_FILE => STDOUT
34
+ }.merge(#{params})
35
+ eval(File.read(\"./main.rb\"))
33
36
  end
34
37
  script_body
35
38
 
@@ -74,7 +74,7 @@ module GoodData
74
74
  @username = username
75
75
  @password = password
76
76
  @url = options[:server] || DEFAULT_URL
77
- @auth_token = options.delete(:auth_token)
77
+ @auth_token = options.delete(:token)
78
78
  @options = options
79
79
 
80
80
  @server = create_server_connection(@url, @options)
@@ -118,9 +118,10 @@ module GoodData
118
118
  #
119
119
  # Connection.new(username, password).post '/gdc/projects', { ... }
120
120
  def post(path, data, options = {})
121
- payload = data.is_a?(Hash) ? data.to_json : data
122
- GoodData.logger.debug "POST #{@server}#{path}, payload: #{payload}"
121
+
122
+ GoodData.logger.debug("POST #{@server}#{path}, payload: #{scrub_params(data, [:password, :login, :authorizationToken])}")
123
123
  ensure_connection
124
+ payload = data.is_a?(Hash) ? data.to_json : data
124
125
  b = Proc.new { @server[path].post payload, cookies }
125
126
  process_response(options, &b)
126
127
  end
@@ -196,6 +197,7 @@ module GoodData
196
197
  # /uploads/ resources are special in that they use a different
197
198
  # host and a basic authentication.
198
199
  def upload(file, options={})
200
+
199
201
  ensure_connection
200
202
 
201
203
  dir = options[:directory] || ''
@@ -236,8 +238,7 @@ module GoodData
236
238
  filename = options[:filename] || options[:stream] ? "randome-filename.txt" : File.basename(file)
237
239
 
238
240
  # Upload the file
239
- puts "uploading the file #{URI.join(url, filename).to_s}"
240
-
241
+ # puts "uploading the file #{URI.join(url, filename).to_s}"
241
242
  req = RestClient::Request.new({
242
243
  :method => :put,
243
244
  :url => URI.join(url, filename).to_s,
@@ -246,24 +247,35 @@ module GoodData
246
247
  :user_agent => GoodData.gem_version_string,
247
248
  },
248
249
  :payload => payload,
249
- :raw_response => true
250
- }.merge(cookies))
250
+ :raw_response => true,
251
+ :user => @username,
252
+ :password => @password
253
+ })
254
+ # .merge(cookies))
251
255
  resp = req.execute
252
256
  true
253
257
  end
254
258
 
255
- def download(what, where)
256
- stage_url = @options[:webdav_server] || @url.sub(/\./, '-di.')
257
- url = stage_url + STAGE_PATH + what
258
- File.open(where, 'w') do |f|
259
- resp = RestClient::Request.execute({
260
- :method => 'GET',
261
- :url => url,
262
- :user => @username,
263
- :password => @password,
264
- :timeout => 0
265
- }) do |chunk, x, y|
266
- f.write chunk
259
+ def download(what, where, options={})
260
+ staging_uri = options[:staging_url].to_s
261
+ url = staging_uri + what
262
+ req = RestClient::Request.new({
263
+ :method => 'GET',
264
+ :url => url,
265
+ :user => @username,
266
+ :password => @password
267
+ })
268
+
269
+ if where.is_a?(String)
270
+ File.open(where, 'w') do |f|
271
+ req.execute do |chunk, x, y|
272
+ f.write chunk
273
+ end
274
+ end
275
+ else
276
+ # Assume it is a IO stream
277
+ req.execute do |chunk, x, y|
278
+ where.write chunk
267
279
  end
268
280
  end
269
281
  end
@@ -285,7 +297,7 @@ module GoodData
285
297
  end
286
298
 
287
299
  def connect
288
- # GoodData.logger.info "Connecting to GoodData..."
300
+ GoodData.logger.info "Connecting to GoodData..."
289
301
  @status = :connecting
290
302
  authenticate
291
303
  end
@@ -350,5 +362,18 @@ module GoodData
350
362
  authenticate
351
363
  end
352
364
  end
365
+
366
+ def scrub_params(params, keys)
367
+ keys = keys.reduce([]) {|memo, k| memo.concat([k.to_s, k.to_sym])}
368
+
369
+ new_params = Marshal.load(Marshal.dump(params))
370
+ GoodData::Helpers.hash_dfs(new_params) do |k, key|
371
+ keys.each do |key_to_scrub|
372
+ k[key_to_scrub] = ("*" * k[key_to_scrub].length) if k && k.has_key?(key_to_scrub)
373
+ end
374
+ end
375
+ new_params
376
+ end
377
+
353
378
  end
354
379
  end