gd_bam 0.0.15 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. data/README.md +313 -5
  2. data/bin/bam +126 -48
  3. data/lib/bam/version.rb +1 -1
  4. data/lib/bam.rb +51 -0
  5. data/lib/base/errors.rb +15 -0
  6. data/lib/base/flow.rb +37 -0
  7. data/lib/base/graph.rb +23 -0
  8. data/lib/base/metadata.rb +107 -0
  9. data/lib/base/project.rb +95 -0
  10. data/lib/base/repo.rb +35 -0
  11. data/lib/base/sink.rb +44 -0
  12. data/lib/base/step.rb +47 -0
  13. data/lib/base/tap.rb +167 -0
  14. data/lib/base/taps.rb +19 -0
  15. data/lib/cloud_connect/dsl/cc.rb +42 -0
  16. data/lib/cloud_connect/dsl/es_helpers.rb +49 -0
  17. data/lib/cloud_connect/dsl/helpers.rb +199 -0
  18. data/lib/{nodes → cloud_connect/dsl}/nodes.rb +106 -16
  19. data/lib/cloud_connect/dsl/sf_helpers.rb +39 -0
  20. data/lib/cloud_connect/dsl/structure_helpers.rb +94 -0
  21. data/lib/commands/commands.rb +110 -0
  22. data/lib/commands/deployment.rb +217 -0
  23. data/lib/commands/docs_commands.rb +41 -0
  24. data/lib/commands/gd_commands.rb +95 -0
  25. data/lib/commands/scaffold_commands.rb +103 -0
  26. data/lib/commands/sf_commands.rb +37 -0
  27. data/lib/commands/validators.rb +19 -0
  28. data/lib/compatibility.rb +19 -0
  29. data/lib/compiler/compiler.rb +76 -0
  30. data/lib/compiler/etl_visitor.rb +165 -0
  31. data/lib/dsl/dsl.rb +125 -0
  32. data/lib/generators/downloaders.rb +449 -0
  33. data/lib/generators/etl.rb +261 -0
  34. data/lib/generators/validators.rb +445 -0
  35. data/lib/graphs/docentize.grf +1 -1
  36. data/lib/graphs/dummy.grf +1 -1
  37. data/lib/graphs/goodsales_v2/docentize.grf +47 -0
  38. data/lib/graphs/goodsales_v2/dummy.grf +46 -0
  39. data/lib/graphs/goodsales_v2/load_history.grf +579 -0
  40. data/lib/graphs/goodsales_v2/process_account.grf +47 -0
  41. data/lib/graphs/goodsales_v2/process_activity.grf +222 -0
  42. data/lib/graphs/goodsales_v2/process_activity_dim.grf +88 -0
  43. data/lib/graphs/goodsales_v2/process_activity_owner.grf +48 -0
  44. data/lib/graphs/goodsales_v2/process_forecast.grf +20 -0
  45. data/lib/graphs/goodsales_v2/process_opp_records.grf +84 -0
  46. data/lib/graphs/goodsales_v2/process_opportunity.grf +46 -0
  47. data/lib/graphs/goodsales_v2/process_opportunity_line_item.grf +171 -0
  48. data/lib/graphs/goodsales_v2/process_opportunity_snapshot.grf +94 -0
  49. data/lib/graphs/goodsales_v2/process_owner.grf +48 -0
  50. data/lib/graphs/goodsales_v2/process_stage.grf +51 -0
  51. data/lib/graphs/goodsales_v2/process_stage_history.grf +184 -0
  52. data/lib/graphs/goodsales_v2/process_velocity_duration.grf +140 -0
  53. data/lib/graphs/process_account.grf +1 -1
  54. data/lib/graphs/process_activity.grf +1 -1
  55. data/lib/graphs/process_activity_dim.grf +1 -1
  56. data/lib/graphs/process_activity_owner.grf +1 -1
  57. data/lib/graphs/process_forecast.grf +1 -1
  58. data/lib/graphs/process_opp_records.grf +1 -1
  59. data/lib/graphs/process_opportunity.grf +1 -1
  60. data/lib/graphs/process_opportunity_line_item.grf +1 -1
  61. data/lib/graphs/process_opportunity_snapshot.grf +1 -1
  62. data/lib/graphs/process_owner.grf +1 -1
  63. data/lib/graphs/process_stage.grf +1 -1
  64. data/lib/graphs/process_stage_history.grf +1 -1
  65. data/lib/graphs/process_velocity_duration.grf +1 -1
  66. data/lib/nodes/clover_gen.rb +59 -946
  67. data/lib/nodes/dependency.rb +95 -96
  68. data/lib/runtime.rb +7 -648
  69. data/lib/utils/utils.rb +66 -0
  70. data/templates/flow.rb.erb +7 -6
  71. data/templates/join_template.grf.erb +1 -1
  72. data/templates/reformat_template.grf.erb +1 -1
  73. data/templates/sink.json.erb +28 -0
  74. data/templates/tap.json.erb +3 -5
  75. data/templates/workspace.prm.erb +4 -0
  76. metadata +50 -8
  77. data/lib/contract_checkers/contract_checkers.rb +0 -53
  78. data/lib/dsl/project_dsl.rb +0 -259
  79. data/lib/repo/1_config.json +0 -8
  80. data/templates/dataset.json.erb +0 -13
  81. data/templates/source.json.erb +0 -22
data/lib/runtime.rb CHANGED
@@ -1,7 +1,5 @@
1
1
  require 'nodes/clover_gen'
2
- require 'nodes/nodes'
3
2
  require 'nodes/dependency'
4
- require 'dsl/project_dsl'
5
3
  require 'pathname'
6
4
  require 'json'
7
5
  require 'builder'
@@ -29,6 +27,7 @@ module GoodData
29
27
 
30
28
  CLOVER_PROJECT_ROOT = DEFINITION_ROOT + "clover-project"
31
29
  CLOVER_DOWNLOADERS_ROOT = DEFINITION_ROOT + "downloader-project"
30
+ CONTRACT_PROJECT_ROOT = DEFINITION_ROOT + "contract-project"
32
31
 
33
32
  PROJECT_GRAPHS_ROOT = "graphs"
34
33
  PROJECT_METADATA_ROOT = "metadata"
@@ -36,654 +35,14 @@ module GoodData
36
35
 
37
36
  PARAM_FILE = DEFINITION_ROOT + 'params.json'
38
37
  FLOWS_ROOT = DEFINITION_ROOT + "flows"
39
- PARAMS = File.exist?(PARAM_FILE) ? JSON.parse(File.read(PARAM_FILE), :symbolize_names => true) : {}
38
+ USER_DEFINED_PARAMS = File.exist?(PARAM_FILE) ? JSON.parse(File.read(PARAM_FILE), :symbolize_names => true) : {}
40
39
 
41
- PARAMS = {:additional_params => nil}.merge(PARAMS)
40
+ PARAMS = {:additional_params => {}}.merge(USER_DEFINED_PARAMS).merge(USER_DEFINED_PARAMS[:additional_params] || {})
42
41
 
43
- include GoodData::CloverGenerator::Dependency
42
+ # TODO: REMOVE WHEN WEBDAV cann use SST
43
+ PARAMS[:additional_params][:GD_LOGIN] = PARAMS[:gd_login]
44
+ PARAMS[:additional_params][:GD_PASSWORD] = PARAMS[:gd_pass]
45
+ PARAMS[:additional_params][:GDC_WEBDAV_HOST] = PARAMS[:web_dav_host] || "secure-di.gooddata.com"
44
46
 
45
-
46
- def self.render_template(template, params, options={})
47
- to_file = options[:to_file]
48
- root = options[:root] || TEMPLATE_ROOT
49
- t = Pathname(template)
50
- output = Erubis::Eruby.new(File.read(root + t)).result(params)
51
- if to_file.nil?
52
- output
53
- else
54
- File.open(to_file, 'w') {|f| f.write(output)}
55
- end
56
- end
57
-
58
- def self.setup_clover_project(base, options={})
59
-
60
- name = options[:name] || PARAMS[:project_name]
61
-
62
- [PROJECT_GRAPHS_ROOT, PROJECT_METADATA_ROOT, PROJECT_DATA_ROOT].each do |dir|
63
- FileUtils::mkdir_p base + dir
64
- end
65
- FileUtils::touch(base + 'params.txt')
66
- render_template("project.erb", PARAMS.merge(:project_name => name), :to_file => base + '.project')
67
- render_template("workspace.prm.erb", PARAMS, :to_file => base + 'workspace.prm')
68
- end
69
-
70
- def self.setup_bash_structure(name)
71
- fail "Directory \"#{name}\" already exists. Please remove it if you want to move forward." if File.exist?(name)
72
- FileUtils::mkdir_p name
73
- FileUtils::cd(name) do
74
- render_template("params.json.erb", PARAMS, :to_file => 'params.json')
75
- ['flows', 'sinks', 'taps'].each do |dir|
76
- FileUtils::mkdir_p dir
77
- end
78
-
79
- setup_flow('user')
80
- # render_template("flow.rb.erb", PARAMS, :to_file => 'flows/flow_example.rb')
81
- render_template("source.json.erb", PARAMS, :to_file => 'taps/source_example.json')
82
- render_template("dataset.json.erb", PARAMS, :to_file => 'sinks/dataset_example.json')
83
- end
84
-
85
- end
86
-
87
- def self.setup_flow(name)
88
- render_template("flow.rb.erb", {:flow_name => name}, :to_file => "flows/#{name}.rb")
89
- end
90
-
91
- def self.setup_tap(name)
92
- render_template("tap.json.erb", {:tap_name => name}, :to_file => "taps/#{name}.json")
93
- end
94
-
95
- def self.clobber_clover_project()
96
- FileUtils::rm_rf(CLOVER_PROJECT_ROOT)
97
- end
98
-
99
- def self.clobber_downloader_project()
100
- FileUtils::rm_rf(CLOVER_DOWNLOADERS_ROOT)
101
- end
102
-
103
- def self.build_project()
104
- p = GoodData::CloverGenerator::DSL.project do |p|
105
- project_name("Goodsales for ABCD")
106
-
107
- use_dims ["GoodSales/opportunity", "GoodSales/user"]
108
- use_usecase("GoodSales/quota_attainment")
109
- end
110
-
111
- repos = GoodData::CloverGenerator::Repo.load(REPO_ROOT).map {|config| N.new(config)}
112
- p.run(repos)
113
- p
114
- end
115
-
116
- def self.validate_datasets
117
- project = build_project
118
- datasets = project.get_datasets
119
- GoodData.connect(PARAMS[:gd_login], PARAMS[:gd_pass])
120
- GoodData.project = PARAMS[:project_pid]
121
- gd_datasets = GoodData.get("/gdc/md/#{PARAMS[:project_pid]}/query/datasets")['query']['entries'].map {|m| GoodData.get(m['link'])}
122
- report1 = GoodData::CloverGenerator::validate_gd_datasets_metadata(datasets, gd_datasets)
123
- report2 = GoodData::CloverGenerator::validate_gd_datasets(datasets, gd_datasets)
124
- report1.merge(report2)
125
- end
126
-
127
- def self.procs_list(options={})
128
- project = build_project
129
- datasets = project.get_datasets
130
- GoodData.connect(PARAMS[:gd_login], PARAMS[:gd_pass])
131
- procs = GoodData.get("/gdc/projects/#{PARAMS[:project_pid]}/dataload/processes")
132
- procs["processes"]["items"].map {|p| [p["process"]["name"], p["process"]["links"]["self"]]}
133
- end
134
-
135
- def self.validate_taps
136
- project = build_project
137
- sources = project.get_sources
138
- client = get_sf_client(PARAMS)
139
- sf_sources = sources.find_all {|tap| tap[:source] == "salesforce"}
140
- GoodData::CloverGenerator::validate_sf_metadata(client, sf_sources)
141
- end
142
-
143
- def self.sf_jack_in
144
- client = get_sf_client(PARAMS)
145
- client.pry
146
- end
147
-
148
- def self.generate_docs
149
- project = build_project
150
- sources = project.get_sources
151
- datasets = project.get_datasets
152
-
153
- taps = ""
154
- sources.each do |source|
155
- taps += "####{source[:object]}"
156
- taps += "\n"
157
- source[:fields].each do |f|
158
- if f[:acts_as]
159
- taps += " #{f[:name]} -> #{f[:acts_as].join(", ")}"
160
- else
161
- taps += " #{f[:name]}"
162
- end
163
- taps += "\n"
164
- end
165
-
166
- taps += "\n"
167
- end
168
-
169
- sinks = ""
170
- datasets.each do |sink|
171
- name = sink[:gd_name] || sink[:id]
172
- sinks += "####{name}\n"
173
- sink[:fields].each do |field|
174
- name = field[:name] || "#{field[:schema]}:#{field[:ref]}"
175
- type = field[:type]
176
- sinks += " #{type.upcase} #{field[:meta]} => #{name}\n"
177
- end
178
- sinks += "\n"
179
-
180
- end
181
-
182
-
183
- render_template("README.md.erb", PARAMS.merge(:taps => taps, :sinks => sinks), :to_file => 'README.md', :root => DEFINITION_ROOT)
184
- end
185
-
186
- def self.model_sync(options)
187
- dry_run = options[:dry]
188
- project = build_project
189
- datasets = project.get_datasets
190
- model_update_dir = Pathname('model_update')
191
- cl_home = ENV['CL_HOME'] || PARAMS['CL_HOME'] || fail("Home of cl tool cannot be found. Either set up CL_HOME in your env with 'export CL_HOME=path/to/cl or set it up in your params.json. Point to the directory of CL not to the bin dir.'")
192
- cl_home = Pathname(cl_home) + 'bin/gdi.sh'
193
-
194
- FileUtils::mkdir_p(model_update_dir)
195
- File.open(model_update_dir + 'dummy', 'w')
196
- FileUtils::cd(model_update_dir) do
197
- datasets.each do |ds|
198
- dataset_path = Pathname("cl_file_#{ds[:id]}")
199
- File.open(dataset_path, "w") do |temp|
200
- builder = Builder::XmlMarkup.new(:target=>temp, :indent=>2)
201
- builder.schema do |builder|
202
- builder.name(ds[:gd_name])
203
- builder.title(ds[:gd_name])
204
- builder.columns do |b|
205
- ds[:fields].each do |f|
206
- builder.column do |builder|
207
- builder.title(f[:name])
208
- builder.ldmType(f[:type].upcase)
209
- builder.reference(f[:for]) if f.has_key?(:for)
210
- builder.reference(f[:ref]) if f.has_key?(:ref)
211
- builder.schemaReference(f[:schema]) if f.has_key?(:schema)
212
- if f[:type] == "date"
213
- builder.schemaReference("#{f[:dd]}")
214
- builder.name("#{f[:name]}")
215
- else
216
- builder.name(f[:name] || f[:ref])
217
- end
218
- end
219
- end
220
- end
221
- end
222
- end
223
- template_name = dry_run ? "update_dataset_dry.script.erb" : "update_dataset.script.erb"
224
- render_template(template_name, PARAMS.merge({"config_file" => dataset_path.expand_path}), :to_file => 'update_dataset.script')
225
- puts "Generate #{ds[:id]}"
226
-
227
- system("#{cl_home} update_dataset.script --username #{PARAMS[:gd_login]} --password #{PARAMS[:gd_pass]}")
228
- File.delete(dataset_path)
229
- end
230
- end
231
- FileUtils::rm_rf(model_update_dir)
232
- end
233
-
234
- def self.generate_downloaders(options={})
235
- setup_clover_project(CLOVER_DOWNLOADERS_ROOT, :name => "downloaders-#{PARAMS[:project_name]}")
236
- project = build_project
237
- sources = project.get_sources
238
- sf_sources = sources.find_all {|tap| tap[:source] == "salesforce" && tap[:incremental] == true}
239
- create_incremental_downloader_run_graph(CLOVER_DOWNLOADERS_ROOT + PROJECT_GRAPHS_ROOT + "main.grf", sf_sources)
240
- s3_backup = PARAMS[:S3_SECRET_ACCESS_KEY] && PARAMS[:S3_ACCESS_KEY_ID] && PARAMS[:S3_BUCKETNAME]
241
-
242
- GoodData::CloverGenerator::create_incremental_downloading_graph(CLOVER_DOWNLOADERS_ROOT + PROJECT_GRAPHS_ROOT + "incremental.grf", sf_sources, {
243
- :password => PARAMS[:sf_password],
244
- :token => PARAMS[:sf_token],
245
- :login => PARAMS[:sf_login],
246
- :sf_server => PARAMS[:sf_server],
247
- :s3_backup => s3_backup
248
- })
249
- end
250
-
251
-
252
- def self.execute_process(link, dir)
253
- result = GoodData.post(link, {
254
- :execution => {
255
- :graph => "./#{dir}/graphs/main.grf",
256
- :params => {}
257
- }
258
- })
259
- begin
260
- GoodData.poll(result, "executionTask")
261
- rescue RestClient::RequestFailed => e
262
-
263
- ensure
264
- result = GoodData.get(result["executionTask"]["links"]["detail"])
265
- if result["executionDetail"]["status"] == "ERROR"
266
- fail "Runing process failed. You can look at a log here #{result["executionDetail"]["logFileName"]}"
267
- end
268
- end
269
- result
270
- end
271
-
272
- def self.connect_to_gd(options={})
273
- GoodData.logger = options[:logger]
274
- GoodData.connect(PARAMS[:gd_login], PARAMS[:gd_pass])
275
- GoodData.project = PARAMS[:project_pid] if !PARAMS[:project_pid].nil? && !PARAMS[:project_pid].empty?
276
- end
277
-
278
- def self.subscribe_on_finish(event_type, channel_uri, process_id)
279
- event_id = case event_type
280
- when :success
281
- "dataload.process.finish.ok"
282
- when :failure
283
- "dataload.process.finish.error"
284
- else
285
- fail "You specified unknown event \"#{event_type}\""
286
- end
287
- data = {
288
- :subscription => {
289
- :triggers => [
290
- {
291
- :projectEventTrigger => {
292
- :types => [event_id]
293
- }
294
- }
295
- ],
296
- :condition => {
297
- :condition => {
298
- :expression => "params.PROCESS_ID==\"#{process_id}\""
299
- }
300
- },
301
- :subject => {
302
- :template => {
303
- :expression => "Subject"
304
- }
305
- },
306
- :message => {
307
- :template => {
308
- :expression => "JEDU"
309
- }
310
- },
311
- :channels => [channel_uri],
312
- :meta => {
313
- :title => "Temporary notification for process #{process_id}"
314
- }
315
- }
316
- }
317
- profile_id = GoodData.connection.user["profile"].split("/").last
318
- GoodData.post("/gdc/projects/#{PARAMS[:project_pid]}/users/#{profile_id}/subscriptions", data)
319
- end
320
-
321
- def self.clone_project(options={})
322
- pid = case options[:blueprint]
323
- when "goodsales"
324
- "nt935rwzls50zfqwy6dh62tabu8h0ocy"
325
- when nil
326
- fail "Empty project not supported now"
327
- end
328
-
329
- project_name = PARAMS[:project_name]
330
- fail "project name has to be filled in" if project_name.blank?
331
- with_users = options[:with_users]
332
-
333
- export = {
334
- :exportProject => {
335
- :exportUsers => with_users ? 1 : 0,
336
- :exportData => 1
337
- }
338
- }
339
-
340
- result = GoodData.post("/gdc/md/#{pid}/maintenance/export", export)
341
- token = result["exportArtifact"]["token"]
342
- status_url = result["exportArtifact"]["status"]["uri"]
343
-
344
- state = GoodData.get(status_url)["taskState"]["status"]
345
- while state == "RUNNING"
346
- sleep 5
347
- result = GoodData.get(status_url)
348
- state = result["taskState"]["status"]
349
- end
350
-
351
- old_project = GoodData::Project[pid]
352
-
353
- pr = {
354
- :project => {
355
- :content => {
356
- :guidedNavigation => 1,
357
- :driver => "Pg",
358
- :authorizationToken => options[:token]
359
- },
360
- :meta => {
361
- :title => project_name,
362
- :summary => "Testing Project"
363
- }
364
- }
365
- }
366
- result = GoodData.post("/gdc/projects/", pr)
367
- uri = result["uri"]
368
- while(GoodData.get(uri)["project"]["content"]["state"] == "LOADING")
369
- sleep(5)
370
- end
371
-
372
- new_project = GoodData::Project[uri]
373
-
374
- import = {
375
- :importProject => {
376
- :token => token
377
- }
378
- }
379
-
380
- result = GoodData.post("/gdc/md/#{new_project.obj_id}/maintenance/import", import)
381
- status_url = result["uri"]
382
- state = GoodData.get(status_url)["taskState"]["status"]
383
- while state == "RUNNING"
384
- sleep 5
385
- result = GoodData.get(status_url)
386
- state = result["taskState"]["status"]
387
- end
388
- GoodData.post "/gdc/projects/#{new_project.obj_id}/eventStore/stores", {:store => {:storeId => "es_0"}}
389
- new_project.obj_id
390
- end
391
-
392
- def self.create_email_channel(options={}, &block)
393
- # email = option[:email]
394
- data = {
395
- :channelConfiguration => {
396
- :configuration => {
397
- :emailConfiguration => {
398
- :to => 'svarovsky@gooddata.com'
399
- }
400
- },
401
- :meta => {
402
- :title => "temporary email channel"
403
- }
404
- }
405
- }
406
- profile_id = GoodData.connection.user["profile"].split("/").last
407
- res = GoodData.post("/gdc/account/profile/#{profile_id}/channelConfigurations", data)
408
- self_link = res["channelConfiguration"]["meta"]["uri"]
409
- if block
410
- begin
411
- block.call(res)
412
- ensure
413
- GoodData.delete(self_link)
414
- end
415
- else
416
- res
417
- end
418
- end
419
-
420
- def self.deploy_graph(dir, options={})
421
- deploy_name = options[:name]
422
- verbose = options[:verbose] || false
423
- puts HighLine::color("Deploying #{dir}", HighLine::BOLD) if verbose
424
- res = nil
425
-
426
- Tempfile.open("deploy-graph-archive") do |temp|
427
- Zip::ZipOutputStream.open(temp.path) do |zio|
428
- Dir.glob("./#{dir}/**/*") do |item|
429
- puts "including #{item}" if verbose
430
- unless File.directory?(item)
431
- zio.put_next_entry(item)
432
- zio.print IO.read(item)
433
- end
434
- end
435
- end
436
-
437
- GoodData.connection.upload(temp.path)
438
- process_id = options[:process]
439
-
440
- data = {
441
- :process => {
442
- :name => deploy_name || "#{PARAMS[:project_name]}",
443
- :path => "/uploads/#{File.basename(temp.path)}"
444
- }
445
- }
446
- res = if process_id.nil?
447
- GoodData.post("/gdc/projects/#{PARAMS[:project_pid]}/dataload/processes", data)
448
- else
449
- GoodData.put("/gdc/projects/#{PARAMS[:project_pid]}/dataload/processes/#{process_id}", data)
450
- end
451
- end
452
- puts HighLine::color("Deploy DONE #{dir}", HighLine::BOLD) if verbose
453
- res
454
- end
455
-
456
- def self.deploy(dir, options={}, &block)
457
- verbose = options[:verbose] || false
458
- if block
459
- begin
460
- res = deploy_graph(dir, options)
461
- block.call(res)
462
- ensure
463
- self_link = res["process"]["links"]["self"]
464
- GoodData.delete(self_link)
465
- end
466
- else
467
- deploy_graph(dir, options)
468
- end
469
- end
470
-
471
- def self.generate_graph_template(name, target)
472
- template_name = "#{name}_template.grf.erb"
473
- render_template(template_name, PARAMS, :to_file => USER_DEFINED_GRAPHS_ROOT + target)
474
- end
475
-
476
-
477
- def self.generate(options)
478
-
479
- only_flow = options[:only]
480
- setup_clover_project(CLOVER_PROJECT_ROOT, :name => "etl-#{PARAMS[:project_name]}")
481
- p = build_project
482
- sources = p.get_sources
483
- datasets = p.get_datasets
484
- s3_backup = PARAMS[:S3_SECRET_ACCESS_KEY] && PARAMS[:S3_ACCESS_KEY_ID] && PARAMS[:S3_BUCKETNAME]
485
-
486
- flows = []
487
- FileUtils::cd FLOWS_ROOT do
488
- flows_sources = Dir.glob("*.rb")
489
- flows = flows_sources.map do |f|
490
- instance_eval(File.read(f))
491
- end
492
- end
493
-
494
- flows = flows.find_all {|flow| flow && flow.name == only_flow} unless only_flow.nil? || only_flow.empty?
495
- fail "The flow you specified was not found" if flows.empty? && !only_flow.nil?
496
- fail "There are no flows to generate from" if flows.empty?
497
- super_flow = []
498
- FileUtils::cd CLOVER_PROJECT_ROOT do
499
-
500
- flows.each do |f|
501
-
502
- current_metadata = {}
503
- steps_to_be_wrapped = []
504
- flow_sources = []
505
- if f.nil?
506
- puts "Flow skipped"
507
- next
508
- end
509
- name = f.name
510
- step_no = 0
511
-
512
- begin
513
- f.steps.each_with_index do |s, i|
514
-
515
- if s[:type] == :tap
516
- source_name = s[:source_name] || f.name
517
- source = sources.find do |source|
518
- source[:id] == source_name
519
- end
520
-
521
- fail "Source \"#{source_name}\" was not found" if source.nil?
522
-
523
- flow_sources << source
524
-
525
- dataset_name = source[:dataset] || source[:id]
526
-
527
- current_metadata[source_name] = GoodData::CloverGenerator::create_metadata(source)
528
- graph_name = "graphs/#{dataset_name}_#{source[:source]}_#{source[:type]}.grf"
529
- steps_to_be_wrapped << {
530
- :name => "#{source_name}_download",
531
- :file => graph_name,
532
- :flow => source_name
533
- }
534
-
535
- if source[:incremental] == true
536
- current_metadata[source_name] = GoodData::CloverGenerator::DSL::Metadata.new(current_metadata[source_name]).change do |m|
537
- m.remove("timestamp")
538
- end.to_hash
539
-
540
- GoodData::CloverGenerator::create_es_downloading_graph(graph_name, [source], {
541
- :metadata => current_metadata[source_name],
542
- :s3_backup => s3_backup
543
- })
544
- else
545
- GoodData::CloverGenerator::create_sf_downloading_graph(graph_name, [source], {
546
- :password => PARAMS[:sf_password],
547
- :token => PARAMS[:sf_token],
548
- :login => PARAMS[:sf_login],
549
- :sf_server => PARAMS[:sf_server],
550
- :metadata => current_metadata[source_name],
551
- :s3_backup => s3_backup
552
- })
553
- end
554
-
555
- step_no += 1
556
- elsif s[:type] == :upload
557
- source_name = s[:id] || f.name
558
- dataset = datasets.find {|d| d[:id] == source_name}
559
-
560
- fail "Dataset \"#{source_name}\" was not found" if dataset.nil?
561
- fail "Sink needs to have id defined" if dataset[:id].nil?
562
-
563
- unless current_metadata.has_key?(source_name)
564
- fail("Source #{source_name} was not found in current metadata")
565
- end
566
- fail "Dataset \"#{f.name}\" was not found" if dataset.nil?
567
- metadata = current_metadata[source_name]
568
-
569
- graph_name = "graphs/#{dataset[:id]}_#{dataset[:type]}.grf"
570
- steps_to_be_wrapped << {
571
- :name => "#{name}_upload",
572
- :file => graph_name,
573
- :flow => source_name
574
- }
575
-
576
- GoodData::CloverGenerator::create_uploading_graph(graph_name, {
577
- :datasets => dataset,
578
- :metadata => current_metadata[source_name]
579
- })
580
- step_no += 1
581
- elsif s[:type] == :user_provided
582
- # check that what is getting in a step is check
583
- s[:metadata_block] && s[:metadata_block].each do |val|
584
- name = val[:name]
585
- next if(name.nil?)
586
- sources_names = flow_sources.map {|flow_source| flow_source[:id]}
587
- included_in_flow = sources_names.include?(name)
588
- unless included_in_flow
589
- # binding.pry
590
- # fail "Metadata \"#{name}\" is not in the defined by any source"
591
- end
592
- end
593
- graph_name = s[:graph]
594
- graph_filename = "#{s[:graph]}.grf"
595
-
596
- if File.exist?(USER_DEFINED_GRAPHS_ROOT + graph_filename)
597
- FileUtils::cp(USER_DEFINED_GRAPHS_ROOT + graph_filename, CLOVER_PROJECT_ROOT + PROJECT_GRAPHS_ROOT)
598
- elsif File.exist?(BAM_DEFINED_GRAPHS_ROOT + graph_filename)
599
- FileUtils::cp(BAM_DEFINED_GRAPHS_ROOT + graph_filename, CLOVER_PROJECT_ROOT + PROJECT_GRAPHS_ROOT)
600
- else
601
- fail("The graph \"#{graph_filename}\" was not found in any location (local, global)")
602
- end
603
- graph_filename = "graphs/#{graph_filename}"
604
-
605
- step_no += 1
606
- s[:metadata_block] && s[:metadata_block].each_with_index do |metadata, i|
607
- j = i+1
608
- bl = metadata[:block]
609
- name = metadata[:name] || f.name
610
- as = metadata[:out_as]
611
- FileUtils::mkdir_p("metadata/#{f.name}/#{graph_name}")
612
-
613
- input_meta = current_metadata[name].clone
614
-
615
- input_meta[:name] = "in_#{j}"
616
- GoodData::CloverGenerator::save_metadata("metadata/#{f.name}/#{graph_name}/#{j}_in.xml", input_meta)
617
- m = GoodData::CloverGenerator::DSL::Metadata.new(current_metadata[name])
618
- new_m = (bl && bl.call(m)) || current_metadata[name]
619
-
620
- current_metadata[name] = new_m
621
- unless as.nil?
622
- as_metadata = new_m.clone
623
- as_metadata[:name] = as
624
- current_metadata[as] = as_metadata
625
- end
626
-
627
- output_meta = current_metadata[name].clone
628
- output_meta[:name] = "out_#{j}"
629
- GoodData::CloverGenerator::save_metadata("metadata/#{f.name}/#{graph_name}/#{j}_out.xml", output_meta)
630
-
631
- GoodData::CloverGenerator::create_moving_graph("graphs/#{f.name}_#{graph_name}_move_in_#{j}.grf", {
632
- :source => "${DATA}/#{name}.csv",
633
- :target => "${DATA}/#{j}_in.csv",
634
- :operation => "MOVE"
635
- })
636
-
637
- steps_to_be_wrapped << {
638
- :name => "graphs/#{f.name}_#{graph_name}_move_in_#{j}.grf",
639
- :file => "graphs/#{f.name}_#{graph_name}_move_in_#{j}.grf",
640
- :flow => "kopirujeme"
641
- }
642
- end
643
-
644
- steps_to_be_wrapped << {
645
- :name => graph_name,
646
- :file => graph_filename,
647
- :flow => f.name
648
- }
649
- output_id = if s[:metadata_block] && s[:metadata_block].any? {|metadata| metadata[:out_as]}
650
- s[:metadata_block].find {|metadata| metadata[:out_as]}[:out_as]
651
- end
652
-
653
- output_id = s[:metadata_block].first[:name] if output_id.nil? && s[:metadata_block].length == 1
654
-
655
- GoodData::CloverGenerator::create_moving_graph("graphs/#{f.name}_#{graph_name}_move_out.grf", {
656
- :source => "${DATA}/out.csv",
657
- :target => "${DATA}/#{output_id}.csv",
658
- :operation => "MOVE"
659
- })
660
-
661
- steps_to_be_wrapped << {
662
- :name => "graphs/#{f.name}_#{graph_name}_move_out.grf",
663
- :file => "graphs/#{f.name}_#{graph_name}_move_out.grf",
664
- :flow => "kopirujeme"
665
- }
666
-
667
- end
668
- end
669
- rescue GoodData::CloverGenerator::DSL::RemoveMetadataFieldError => e
670
- puts "Removing field \"#{e.field}\" from metadata \"#{e.metadata.name}\" in Flow \"#{f.name}\" there was a roblem with step X."
671
- end
672
-
673
- GoodData::CloverGenerator::create_run_graph("graphs/#{f.name}_main.grf", {
674
- :subgraphs => steps_to_be_wrapped
675
- })
676
- super_flow << {
677
- :name => name,
678
- :file => "graphs/#{f.name}_main.grf",
679
- :flow => f.name
680
- }
681
- end
682
-
683
- GoodData::CloverGenerator::create_run_graph("graphs/main.grf", {
684
- :subgraphs => super_flow
685
- })
686
- end
687
- end
688
47
  end
689
48
  end