gd_bam 0.0.15 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. data/README.md +313 -5
  2. data/bin/bam +126 -48
  3. data/lib/bam/version.rb +1 -1
  4. data/lib/bam.rb +51 -0
  5. data/lib/base/errors.rb +15 -0
  6. data/lib/base/flow.rb +37 -0
  7. data/lib/base/graph.rb +23 -0
  8. data/lib/base/metadata.rb +107 -0
  9. data/lib/base/project.rb +95 -0
  10. data/lib/base/repo.rb +35 -0
  11. data/lib/base/sink.rb +44 -0
  12. data/lib/base/step.rb +47 -0
  13. data/lib/base/tap.rb +167 -0
  14. data/lib/base/taps.rb +19 -0
  15. data/lib/cloud_connect/dsl/cc.rb +42 -0
  16. data/lib/cloud_connect/dsl/es_helpers.rb +49 -0
  17. data/lib/cloud_connect/dsl/helpers.rb +199 -0
  18. data/lib/{nodes → cloud_connect/dsl}/nodes.rb +106 -16
  19. data/lib/cloud_connect/dsl/sf_helpers.rb +39 -0
  20. data/lib/cloud_connect/dsl/structure_helpers.rb +94 -0
  21. data/lib/commands/commands.rb +110 -0
  22. data/lib/commands/deployment.rb +217 -0
  23. data/lib/commands/docs_commands.rb +41 -0
  24. data/lib/commands/gd_commands.rb +95 -0
  25. data/lib/commands/scaffold_commands.rb +103 -0
  26. data/lib/commands/sf_commands.rb +37 -0
  27. data/lib/commands/validators.rb +19 -0
  28. data/lib/compatibility.rb +19 -0
  29. data/lib/compiler/compiler.rb +76 -0
  30. data/lib/compiler/etl_visitor.rb +165 -0
  31. data/lib/dsl/dsl.rb +125 -0
  32. data/lib/generators/downloaders.rb +449 -0
  33. data/lib/generators/etl.rb +261 -0
  34. data/lib/generators/validators.rb +445 -0
  35. data/lib/graphs/docentize.grf +1 -1
  36. data/lib/graphs/dummy.grf +1 -1
  37. data/lib/graphs/goodsales_v2/docentize.grf +47 -0
  38. data/lib/graphs/goodsales_v2/dummy.grf +46 -0
  39. data/lib/graphs/goodsales_v2/load_history.grf +579 -0
  40. data/lib/graphs/goodsales_v2/process_account.grf +47 -0
  41. data/lib/graphs/goodsales_v2/process_activity.grf +222 -0
  42. data/lib/graphs/goodsales_v2/process_activity_dim.grf +88 -0
  43. data/lib/graphs/goodsales_v2/process_activity_owner.grf +48 -0
  44. data/lib/graphs/goodsales_v2/process_forecast.grf +20 -0
  45. data/lib/graphs/goodsales_v2/process_opp_records.grf +84 -0
  46. data/lib/graphs/goodsales_v2/process_opportunity.grf +46 -0
  47. data/lib/graphs/goodsales_v2/process_opportunity_line_item.grf +171 -0
  48. data/lib/graphs/goodsales_v2/process_opportunity_snapshot.grf +94 -0
  49. data/lib/graphs/goodsales_v2/process_owner.grf +48 -0
  50. data/lib/graphs/goodsales_v2/process_stage.grf +51 -0
  51. data/lib/graphs/goodsales_v2/process_stage_history.grf +184 -0
  52. data/lib/graphs/goodsales_v2/process_velocity_duration.grf +140 -0
  53. data/lib/graphs/process_account.grf +1 -1
  54. data/lib/graphs/process_activity.grf +1 -1
  55. data/lib/graphs/process_activity_dim.grf +1 -1
  56. data/lib/graphs/process_activity_owner.grf +1 -1
  57. data/lib/graphs/process_forecast.grf +1 -1
  58. data/lib/graphs/process_opp_records.grf +1 -1
  59. data/lib/graphs/process_opportunity.grf +1 -1
  60. data/lib/graphs/process_opportunity_line_item.grf +1 -1
  61. data/lib/graphs/process_opportunity_snapshot.grf +1 -1
  62. data/lib/graphs/process_owner.grf +1 -1
  63. data/lib/graphs/process_stage.grf +1 -1
  64. data/lib/graphs/process_stage_history.grf +1 -1
  65. data/lib/graphs/process_velocity_duration.grf +1 -1
  66. data/lib/nodes/clover_gen.rb +59 -946
  67. data/lib/nodes/dependency.rb +95 -96
  68. data/lib/runtime.rb +7 -648
  69. data/lib/utils/utils.rb +66 -0
  70. data/templates/flow.rb.erb +7 -6
  71. data/templates/join_template.grf.erb +1 -1
  72. data/templates/reformat_template.grf.erb +1 -1
  73. data/templates/sink.json.erb +28 -0
  74. data/templates/tap.json.erb +3 -5
  75. data/templates/workspace.prm.erb +4 -0
  76. metadata +50 -8
  77. data/lib/contract_checkers/contract_checkers.rb +0 -53
  78. data/lib/dsl/project_dsl.rb +0 -259
  79. data/lib/repo/1_config.json +0 -8
  80. data/templates/dataset.json.erb +0 -13
  81. data/templates/source.json.erb +0 -22
@@ -0,0 +1,110 @@
1
+ module GoodData
2
+ module Bam
3
+ module Commands
4
+
5
+ include GoodData::CloudConnect
6
+
7
+ VALIDATORS_HOME = "validators-project"
8
+ DOWNLOADERS_HOME = "downloaders-project"
9
+ ETL_HOME = "etl-project"
10
+
11
+ def self.generate_graph_template(name, target)
12
+ template_name = "#{name}_template.grf.erb"
13
+ render_template(template_name, PARAMS, :to_file => USER_DEFINED_GRAPHS_ROOT + target)
14
+ end
15
+
16
+ def self.project_jack_in(params)
17
+ project = GoodData::Bam::Project.build_project
18
+ end
19
+
20
+ def self.generate_validators(home, params)
21
+ home = Pathname(home)
22
+ clobber_contract_project(home, params)
23
+ setup_contract_project(home, params)
24
+
25
+ project = GoodData::Bam::Project.build_project(params)
26
+ file_taps = Taps.get_file(project[:taps])
27
+ file_taps = file_taps.find_all {|t| t[:validation_source]}
28
+ file_taps.each do |t|
29
+ puts "Will generate validator for tap \"#{t[:id]}\""
30
+ end
31
+ GoodData::Bam::Validators.generate_checker_file_list(home + VALIDATORS_HOME + "graphs/main.grf", file_taps)
32
+
33
+ file_taps.each do |tap|
34
+ id = tap[:id]
35
+ GoodData::Bam::Validators.generate_contract(home + VALIDATORS_HOME + "graphs/#{id}_checker.grf", tap)
36
+ Helpers::loop_over_file(home + VALIDATORS_HOME + "graphs/#{id}_loop.grf", {
37
+ :token => id,
38
+ :file_to_loop => "data/#{id}_files_to_read.csv",
39
+ :graph_to_run => "graphs/#{id}_checker.grf"
40
+ })
41
+
42
+ GoodData::Bam::Validators.generate_checker_success(home + VALIDATORS_HOME + "graphs/#{id}_success.grf", tap)
43
+ GoodData::Bam::Validators.backup_files_from_glob(home + VALIDATORS_HOME + "graphs/#{id}_failure.grf", {
44
+ :files_to_backup => ["${DATA}/validation_output_*.csv", "${DATA}/*_files_to_read.csv"],
45
+ :remote_url => "`\"https://\" + replace(replace(\"${GD_LOGIN}\",\"@\",\"%40\"),\"\\\\+\",\"%2B\") + \":${GD_PASSWORD}@${GDC_WEBDAV_HOST}/project-uploads/${GDC_PROJECT_ID}/validation_reports/report_${NOW}/\"`"
46
+ })
47
+ end
48
+ end
49
+
50
+ def self.setup_contract_project(home, params={})
51
+ home = Pathname(home)
52
+ paths = Utils::get_paths(home + VALIDATORS_HOME)
53
+ Utils::setup_basic_project(paths, params)
54
+ end
55
+
56
+ def self.clobber_contract_project(home, params={})
57
+ home = Pathname(home)
58
+ paths = Utils::get_paths(home + VALIDATORS_HOME)
59
+ clobber_project(paths)
60
+ end
61
+
62
+ def self.setup_etl_project(home, params={})
63
+ home = Pathname(home)
64
+ paths = Utils::get_paths(home + ETL_HOME)
65
+ Utils::setup_basic_project(paths, params)
66
+ end
67
+
68
+ def self.clobber_etl_project(home, params={})
69
+ home = Pathname(home)
70
+ paths = Utils::get_paths(home + ETL_HOME)
71
+ clobber_project(paths)
72
+ end
73
+
74
+ def self.clobber_downloaders_project(home, params={})
75
+ home = Pathname(home)
76
+ paths = Utils::get_paths(home + DOWNLOADERS_HOME)
77
+ clobber_project(paths)
78
+ end
79
+
80
+ def self.setup_downloaders_project(home, params={})
81
+ home = Pathname(home)
82
+ paths = Utils::get_paths(home + DOWNLOADERS_HOME)
83
+ Utils::setup_basic_project(paths, params)
84
+ end
85
+
86
+ def self.clobber_project(paths)
87
+ FileUtils::rm_rf(paths[:home_path])
88
+ end
89
+
90
+ def self.generate_downloaders(home, params)
91
+ home = Pathname(home)
92
+ clobber_downloaders_project(home)
93
+ project = GoodData::Bam::Project.build_project(params)
94
+ setup_downloaders_project(home, params)
95
+ GoodData::Bam::Generators::Downloaders.generate_downloaders(home + DOWNLOADERS_HOME + "graphs", project, params)
96
+ end
97
+
98
+ def self.generate(home, params)
99
+ project = GoodData::Bam::Project.build_project(params)
100
+
101
+ # fail "The flow you specified was not found" if flows.empty? && !only_flow.nil?
102
+ fail "There are no flows to generate from" if project[:flows].empty?
103
+
104
+ v = Compiler::EtlVisitor.new
105
+ v.visit(project, params)
106
+ end
107
+
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,217 @@
1
+ module GoodData
2
+ module Bam
3
+ module Commands
4
+
5
+ def self.upload_files(files, now)
6
+ upload_paths = files.map do |file|
7
+ file = Pathname(file)
8
+ ext = file.extname
9
+ "#{file.to_s.chomp(ext)}_#{now}#{ext}"
10
+ end
11
+
12
+ Dir.mktmpdir do |dir|
13
+ dir = Pathname(dir)
14
+ files.zip(upload_paths).each do |f|
15
+ FileUtils::cp(f[0], dir + f[1])
16
+ system "curl -u #{PARAMS[:gd_login]}:#{PARAMS[:gd_pass]} -k -T #{dir+f[1]} https://secure-di.gooddata.com/project-uploads/#{PARAMS[:project_pid]}/"
17
+ puts "Uploaded #{f[1]}"
18
+ end
19
+ end
20
+ end
21
+
22
+ def self.clean_files(files, now)
23
+ upload_paths = files.map do |file|
24
+ file = Pathname(file)
25
+ ext = file.extname
26
+ "#{file.to_s.chomp(ext)}_#{now}#{ext}"
27
+ end
28
+
29
+ upload_paths.each do |f|
30
+ system "curl -u #{PARAMS[:gd_login]}:#{PARAMS[:gd_pass]} -k -X DELETE https://secure-di.gooddata.com/project-uploads/#{PARAMS[:project_pid]}/#{f}"
31
+ puts "Uploaded #{f[1]}"
32
+ end
33
+ end
34
+
35
+ def self.execute_process(link, dir)
36
+ dir = Pathname(dir)
37
+ result = GoodData.post(link, {
38
+ :execution => {
39
+ :graph => dir + "graphs/main.grf",
40
+ :params => {}
41
+ }
42
+ })
43
+ begin
44
+ GoodData.poll(result, "executionTask")
45
+ rescue RestClient::RequestFailed => e
46
+
47
+ ensure
48
+ result = GoodData.get(result["executionTask"]["links"]["detail"])
49
+ if result["executionDetail"]["status"] == "ERROR"
50
+ fail "Runing process failed. You can look at a log here #{result["executionDetail"]["logFileName"]}"
51
+ end
52
+ end
53
+ result
54
+ end
55
+
56
+ def self.run(dir, options={})
57
+ email = options[:email]
58
+ verbose = options[:v]
59
+
60
+ dir = Pathname(dir)
61
+
62
+ deploy(dir, options.merge(:name => "Temporary deploy[#{dir}][#{PARAMS[:project_name]}]")) do |deploy_response|
63
+ puts HighLine::color("Executing", HighLine::BOLD) if verbose
64
+ if email.nil?
65
+ result = execute_process(deploy_response["process"]["links"]["executions"], dir)
66
+ else
67
+ create_email_channel(options) do |channel_response|
68
+ subscribe_on_finish(:success, channel_response, deploy_response, options)
69
+ result = execute_process(deploy_response["process"]["links"]["executions"], dir)
70
+ end
71
+ end
72
+ end
73
+ end
74
+
75
+ def self.subscribe_on_finish(event_type, channel, deploy, options={})
76
+ channel_uri = channel["channelConfiguration"]["meta"]["uri"]
77
+ process_id = deploy["process"]["links"]["self"].split('/').last
78
+ event_id = case event_type
79
+ when :success
80
+ "dataload.process.finish.ok"
81
+ when :failure
82
+ "dataload.process.finish.error"
83
+ else
84
+ fail "You specified unknown event \"#{event_type}\""
85
+ end
86
+
87
+ templates = {
88
+ :success => {
89
+ :message => "Just wanted to let you know that ETL for \"#{deploy["process"]["name"]}\" Succeeded",
90
+ :subject => "GoodData ETL SUCCESS: process with name \"#{deploy["process"]["name"]}\""
91
+ },
92
+ :failure => {
93
+ :message => "Just wanted to let you know that ETL for \"#{deploy["process"]["name"]}\" Failed",
94
+ :subject => "GoodData ETL FAILURE: process with name \"#{deploy["process"]["name"]}\""
95
+ }
96
+ }
97
+
98
+ templates_to_be_used = templates[event_type]
99
+
100
+ data = {
101
+ :subscription => {
102
+ :triggers => [
103
+ {
104
+ :projectEventTrigger => {
105
+ :types => [event_id]
106
+ }
107
+ }
108
+ ],
109
+ :condition => {
110
+ :condition => {
111
+ :expression => "params.PROCESS_ID==\"#{process_id}\""
112
+ }
113
+ },
114
+ :subject => {
115
+ :template => {
116
+ :expression => templates_to_be_used[:subject]
117
+ }
118
+ },
119
+ :message => {
120
+ :template => {
121
+ :expression => templates_to_be_used[:message]
122
+ }
123
+ },
124
+ :channels => [channel_uri],
125
+ :meta => {
126
+ :title => "Notification for process #{process_id}"
127
+ }
128
+ }
129
+ }
130
+ profile_id = GoodData.connection.user["profile"].split("/").last
131
+ GoodData.post("/gdc/projects/#{PARAMS[:project_pid]}/users/#{profile_id}/subscriptions", data)
132
+ end
133
+
134
+ def self.deploy(dir, options={}, &block)
135
+ verbose = options[:verbose] || false
136
+ if block
137
+ begin
138
+ res = deploy_graph(dir, options)
139
+ block.call(res)
140
+ ensure
141
+ self_link = res["process"]["links"]["self"]
142
+ GoodData.delete(self_link)
143
+ end
144
+ else
145
+ deploy_graph(dir, options)
146
+ end
147
+ end
148
+
149
+ def self.deploy_graph(dir, options={})
150
+ dir = Pathname(dir)
151
+ deploy_name = options[:name] || "#{PARAMS[:project_name]}"
152
+ verbose = options[:verbose] || false
153
+ puts HighLine::color("Deploying #{dir}", HighLine::BOLD) if verbose
154
+ res = nil
155
+
156
+ Tempfile.open("deploy-graph-archive") do |temp|
157
+ Zip::ZipOutputStream.open(temp.path) do |zio|
158
+ Dir.glob(dir + "**/*") do |item|
159
+ puts "including #{item}" if verbose
160
+ unless File.directory?(item)
161
+ zio.put_next_entry(item)
162
+ zio.print IO.read(item)
163
+ end
164
+ end
165
+ end
166
+
167
+ GoodData.connection.upload(temp.path)
168
+ process_id = options[:process]
169
+
170
+ data = {
171
+ :process => {
172
+ :name => deploy_name,
173
+ :path => "/uploads/#{File.basename(temp.path)}"
174
+ }
175
+ }
176
+ res = if process_id.nil?
177
+ GoodData.post("/gdc/projects/#{PARAMS[:project_pid]}/dataload/processes", data)
178
+ else
179
+ GoodData.put("/gdc/projects/#{PARAMS[:project_pid]}/dataload/processes/#{process_id}", data)
180
+ end
181
+ end
182
+ puts HighLine::color("Deploy DONE #{dir}", HighLine::BOLD) if verbose
183
+ res
184
+ end
185
+
186
+ def self.create_email_channel(options={}, &block)
187
+ email = options[:email]
188
+
189
+ data = {
190
+ :channelConfiguration => {
191
+ :configuration => {
192
+ :emailConfiguration => {
193
+ :to => email
194
+ }
195
+ },
196
+ :meta => {
197
+ :title => "temporary email channel"
198
+ }
199
+ }
200
+ }
201
+ profile_id = GoodData.connection.user["profile"].split("/").last
202
+ res = GoodData.post("/gdc/account/profile/#{profile_id}/channelConfigurations", data)
203
+ self_link = res["channelConfiguration"]["meta"]["uri"]
204
+ if block
205
+ begin
206
+ block.call(res)
207
+ ensure
208
+ GoodData.delete(self_link)
209
+ end
210
+ else
211
+ res
212
+ end
213
+ end
214
+
215
+ end
216
+ end
217
+ end
@@ -0,0 +1,41 @@
1
+ module GoodData
2
+ module Bam
3
+ module Commands
4
+
5
+ def self.generate_docs(path=".")
6
+ project = GoodData::Bam::Project.build_project
7
+
8
+ taps = ""
9
+ project[:taps].each do |tap|
10
+ taps += "####{tap[:object]}"
11
+ taps += "\n"
12
+ tap[:fields].each do |f|
13
+ if f[:acts_as]
14
+ taps += " #{f[:name]} -> #{f[:acts_as].join(", ")}"
15
+ else
16
+ taps += " #{f[:name]}"
17
+ end
18
+ taps += "\n"
19
+ end
20
+
21
+ taps += "\n"
22
+ end
23
+
24
+ sinks = ""
25
+ project[:sinks].each do |sink|
26
+ name = sink[:gd_name] || sink[:id]
27
+ sinks += "####{name}\n"
28
+ sink[:fields].each do |field|
29
+ name = field[:name] || "#{field[:schema]}:#{field[:ref]}"
30
+ type = field[:type]
31
+ sinks += " #{type.upcase} #{field[:meta]} => #{name}\n"
32
+ end
33
+ sinks += "\n"
34
+
35
+ end
36
+
37
+ GoodData::Bam::Utils.render_template("README.md.erb", PARAMS.merge(:taps => taps, :sinks => sinks), :to_file => 'README.md', :root => Pathname(path))
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,95 @@
1
+ module GoodData
2
+ module Bam
3
+ module Commands
4
+
5
+ def self.connect_to_gd(options={})
6
+ GoodData.connect(PARAMS[:gd_login], PARAMS[:gd_pass])
7
+ GoodData.project = PARAMS[:project_pid] if !PARAMS[:project_pid].nil? && !PARAMS[:project_pid].empty?
8
+ end
9
+
10
+ def self.set_up_debug(project, flow, graph)
11
+ fail "Project \"#{project}\" was not found" unless File.exist?(project)
12
+ fail "Project \"#{project}\" is not a directory" unless File.directory?(project)
13
+ # :TODO make the proram really check there is a flow and a graph etc before writing to the file
14
+ project = Pathname(project)
15
+ File.open(project + 'params.prm', 'w') do |f|
16
+ f << "FLOW=#{flow}\n"
17
+ f << "NAME=#{graph}\n"
18
+ end
19
+ end
20
+
21
+ def self.model_sync(options)
22
+ dry_run = options[:dry]
23
+ project = build_project
24
+ datasets = project.sinks
25
+ model_update_dir = Pathname('model_update')
26
+ cl_home = ENV['CL_HOME'] || PARAMS['CL_HOME'] || fail("Home of cl tool cannot be found. Either set up CL_HOME in your env with 'export CL_HOME=path/to/cl or set it up in your params.json. Point to the directory of CL not to the bin dir.'")
27
+ cl_home = Pathname(cl_home) + 'bin/gdi.sh'
28
+
29
+ FileUtils::mkdir_p(model_update_dir)
30
+ File.open(model_update_dir + 'dummy', 'w')
31
+ FileUtils::cd(model_update_dir) do
32
+ datasets.each do |ds|
33
+ dataset_path = Pathname("cl_file_#{ds[:id]}")
34
+ File.open(dataset_path, "w") do |temp|
35
+ builder = Builder::XmlMarkup.new(:target=>temp, :indent=>2)
36
+ builder.schema do |builder|
37
+ builder.name(ds[:gd_name])
38
+ builder.title(ds[:gd_name])
39
+ builder.columns do |b|
40
+ ds[:fields].each do |f|
41
+ builder.column do |builder|
42
+ builder.title(f[:name])
43
+ builder.ldmType(f[:type].upcase)
44
+ builder.reference(f[:for]) if f.has_key?(:for)
45
+ builder.reference(f[:ref]) if f.has_key?(:ref)
46
+ builder.schemaReference(f[:schema]) if f.has_key?(:schema)
47
+ if f[:type] == "date"
48
+ builder.schemaReference("#{f[:dd]}")
49
+ builder.name("#{f[:name]}")
50
+ else
51
+ builder.name(f[:name] || f[:ref])
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
58
+ template_name = dry_run ? "update_dataset_dry.script.erb" : "update_dataset.script.erb"
59
+ render_template(template_name, PARAMS.merge({"config_file" => dataset_path.expand_path}), :to_file => 'update_dataset.script')
60
+ puts "Generate #{ds[:id]}"
61
+
62
+ system("#{cl_home} update_dataset.script --username #{PARAMS[:gd_login]} --password #{PARAMS[:gd_pass]}")
63
+ File.delete(dataset_path)
64
+ end
65
+ end
66
+ FileUtils::rm_rf(model_update_dir)
67
+ end
68
+
69
+ def self.create_project(options={})
70
+ project_name = options[:project_name]
71
+
72
+ pr = {
73
+ :project => {
74
+ :content => {
75
+ :guidedNavigation => 1,
76
+ :driver => "Pg",
77
+ :authorizationToken => options[:token]
78
+ },
79
+ :meta => {
80
+ :title => project_name,
81
+ :summary => "Testing Project"
82
+ }
83
+ }
84
+ }
85
+ result = GoodData.post("/gdc/projects/", pr)
86
+ uri = result["uri"]
87
+ while(GoodData.get(uri)["project"]["content"]["state"] == "LOADING")
88
+ sleep(5)
89
+ end
90
+ p = GoodData::Project[uri]
91
+ p.obj_id
92
+ end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,103 @@
1
+ module GoodData
2
+ module Bam
3
+ module Commands
4
+
5
+ def self.generate_graph_template(name, target)
6
+ template_name = "#{name}_template.grf.erb"
7
+ Utils::render_template(template_name, PARAMS, :to_file => GoodData::CloverGenerator::USER_DEFINED_GRAPHS_ROOT + target)
8
+ end
9
+
10
+ def self.setup_bash_structure(name)
11
+ fail "Directory \"#{name}\" already exists. Please remove it if you want to move forward." if File.exist?(name)
12
+ FileUtils::mkdir_p name
13
+ FileUtils::cd(name) do
14
+ Utils::render_template("params.json.erb", PARAMS, :to_file => 'params.json')
15
+ ['flows', 'sinks', 'taps'].each do |dir|
16
+ FileUtils::mkdir_p dir
17
+ end
18
+
19
+ setup_flow('user')
20
+ Utils::render_template("tap.json.erb", PARAMS, :to_file => 'taps/source_example.json')
21
+ Utils::render_template("sink.json.erb", PARAMS, :to_file => 'sinks/dataset_example.json')
22
+ end
23
+ end
24
+
25
+ def self.setup_flow(name)
26
+ Utils::render_template("flow.rb.erb", {:flow_name => name}, :to_file => "flows/#{name}.rb")
27
+ end
28
+
29
+ def self.setup_tap(name)
30
+ Utils::render_template("tap.json.erb", {:tap_name => name}, :to_file => "taps/#{name}.json")
31
+ end
32
+
33
+ def self.setup_sink(name)
34
+ Utils::render_template("sink.json.erb", {:sink_name => name}, :to_file => "sinks/#{name}.json")
35
+ end
36
+
37
+ def self.clone_project(pid, options={})
38
+ project_name = options[:project_name]
39
+ fail "project name has to be filled in" if project_name.blank?
40
+ with_users = options[:with_users]
41
+
42
+ export = {
43
+ :exportProject => {
44
+ :exportUsers => with_users ? 1 : 0,
45
+ :exportData => 1
46
+ }
47
+ }
48
+
49
+ result = GoodData.post("/gdc/md/#{pid}/maintenance/export", export)
50
+ token = result["exportArtifact"]["token"]
51
+ status_url = result["exportArtifact"]["status"]["uri"]
52
+
53
+ state = GoodData.get(status_url)["taskState"]["status"]
54
+ while state == "RUNNING"
55
+ sleep 5
56
+ result = GoodData.get(status_url)
57
+ state = result["taskState"]["status"]
58
+ end
59
+
60
+ old_project = GoodData::Project[pid]
61
+
62
+ pr = {
63
+ :project => {
64
+ :content => {
65
+ :guidedNavigation => 1,
66
+ :driver => "Pg",
67
+ :authorizationToken => options[:token]
68
+ },
69
+ :meta => {
70
+ :title => project_name,
71
+ :summary => "Testing Project"
72
+ }
73
+ }
74
+ }
75
+ result = GoodData.post("/gdc/projects/", pr)
76
+ uri = result["uri"]
77
+ while(GoodData.get(uri)["project"]["content"]["state"] == "LOADING")
78
+ sleep(5)
79
+ end
80
+
81
+ new_project = GoodData::Project[uri]
82
+
83
+ import = {
84
+ :importProject => {
85
+ :token => token
86
+ }
87
+ }
88
+
89
+ result = GoodData.post("/gdc/md/#{new_project.obj_id}/maintenance/import", import)
90
+ status_url = result["uri"]
91
+ state = GoodData.get(status_url)["taskState"]["status"]
92
+ while state == "RUNNING"
93
+ sleep 5
94
+ result = GoodData.get(status_url)
95
+ state = result["taskState"]["status"]
96
+ end
97
+ GoodData.post "/gdc/projects/#{new_project.obj_id}/eventStore/stores", {:store => {:storeId => "es_0"}}
98
+ new_project.obj_id
99
+ end
100
+
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,37 @@
1
+ module GoodData
2
+ module Bam
3
+ module Commands
4
+
5
+ def self.validate_sf_taps(params)
6
+ project = GoodData::Bam::Project.build_project(params)
7
+ client = get_sf_client(params)
8
+ validate_sf_metadata(client, Taps.get_salesforce(project[:taps]))
9
+ end
10
+
11
+ def self.validate_sf_metadata(sf_client, taps)
12
+ taps.reduce({}) do |memo, tap|
13
+ sf_object = tap[:object]
14
+ u = sf_client.describe(sf_object)
15
+ sf_fields = u[:describeSObjectResponse][:result][:fields].map {|field| field[:name]}
16
+ fields_to_validate = tap[:fields].map {|field| field[:name]}
17
+ memo[sf_object] = (fields_to_validate - sf_fields)
18
+ memo
19
+ end
20
+ end
21
+
22
+ def self.sf_jack_in(params)
23
+ client = get_sf_client(params)
24
+ client.pry
25
+ end
26
+
27
+ def self.sf_validate_connection(params)
28
+ get_sf_client(params)
29
+ end
30
+
31
+ def self.get_sf_client(params)
32
+ Salesforce::Client.new(params[:sf_login], params[:sf_password] + params[:sf_token], :server => params[:sf_server])
33
+ end
34
+
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,19 @@
1
+ module GoodData
2
+ module Bam
3
+ module Commands
4
+
5
+ include GoodData::CloudConnect
6
+
7
+ def self.run_validator(process, files, params)
8
+ now = Time.now.to_i.to_s
9
+ upload_files(files, now)
10
+ begin
11
+ execute_process("/gdc/projects/#{params[:project_pid]}/dataload/processes/#{process}/executions", VALIDATORS_HOME)
12
+ rescue
13
+ clean_files(files, now)
14
+ exit_now!("Data validation failed failed. Please look here for details. \"https://secure-di.gooddata.com/project-uploads/#{params[:project_pid]}/validation_reports/\"")
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,19 @@
1
+ module Enumerable
2
+ def uniq_by
3
+ seen = Hash.new { |h,k| h[k] = true; false }
4
+ reject { |v| seen[yield(v)] }
5
+ end
6
+ end
7
+
8
+ class Hash
9
+ #pass single or array of keys, which will be removed, returning the remaining hash
10
+ def remove!(*keys)
11
+ keys.each{|key| self.delete(key) }
12
+ self
13
+ end
14
+
15
+ #non-destructive version
16
+ def remove(*keys)
17
+ self.dup.remove!(*keys)
18
+ end
19
+ end