gd_bam 0.0.15 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. data/README.md +313 -5
  2. data/bin/bam +126 -48
  3. data/lib/bam/version.rb +1 -1
  4. data/lib/bam.rb +51 -0
  5. data/lib/base/errors.rb +15 -0
  6. data/lib/base/flow.rb +37 -0
  7. data/lib/base/graph.rb +23 -0
  8. data/lib/base/metadata.rb +107 -0
  9. data/lib/base/project.rb +95 -0
  10. data/lib/base/repo.rb +35 -0
  11. data/lib/base/sink.rb +44 -0
  12. data/lib/base/step.rb +47 -0
  13. data/lib/base/tap.rb +167 -0
  14. data/lib/base/taps.rb +19 -0
  15. data/lib/cloud_connect/dsl/cc.rb +42 -0
  16. data/lib/cloud_connect/dsl/es_helpers.rb +49 -0
  17. data/lib/cloud_connect/dsl/helpers.rb +199 -0
  18. data/lib/{nodes → cloud_connect/dsl}/nodes.rb +106 -16
  19. data/lib/cloud_connect/dsl/sf_helpers.rb +39 -0
  20. data/lib/cloud_connect/dsl/structure_helpers.rb +94 -0
  21. data/lib/commands/commands.rb +110 -0
  22. data/lib/commands/deployment.rb +217 -0
  23. data/lib/commands/docs_commands.rb +41 -0
  24. data/lib/commands/gd_commands.rb +95 -0
  25. data/lib/commands/scaffold_commands.rb +103 -0
  26. data/lib/commands/sf_commands.rb +37 -0
  27. data/lib/commands/validators.rb +19 -0
  28. data/lib/compatibility.rb +19 -0
  29. data/lib/compiler/compiler.rb +76 -0
  30. data/lib/compiler/etl_visitor.rb +165 -0
  31. data/lib/dsl/dsl.rb +125 -0
  32. data/lib/generators/downloaders.rb +449 -0
  33. data/lib/generators/etl.rb +261 -0
  34. data/lib/generators/validators.rb +445 -0
  35. data/lib/graphs/docentize.grf +1 -1
  36. data/lib/graphs/dummy.grf +1 -1
  37. data/lib/graphs/goodsales_v2/docentize.grf +47 -0
  38. data/lib/graphs/goodsales_v2/dummy.grf +46 -0
  39. data/lib/graphs/goodsales_v2/load_history.grf +579 -0
  40. data/lib/graphs/goodsales_v2/process_account.grf +47 -0
  41. data/lib/graphs/goodsales_v2/process_activity.grf +222 -0
  42. data/lib/graphs/goodsales_v2/process_activity_dim.grf +88 -0
  43. data/lib/graphs/goodsales_v2/process_activity_owner.grf +48 -0
  44. data/lib/graphs/goodsales_v2/process_forecast.grf +20 -0
  45. data/lib/graphs/goodsales_v2/process_opp_records.grf +84 -0
  46. data/lib/graphs/goodsales_v2/process_opportunity.grf +46 -0
  47. data/lib/graphs/goodsales_v2/process_opportunity_line_item.grf +171 -0
  48. data/lib/graphs/goodsales_v2/process_opportunity_snapshot.grf +94 -0
  49. data/lib/graphs/goodsales_v2/process_owner.grf +48 -0
  50. data/lib/graphs/goodsales_v2/process_stage.grf +51 -0
  51. data/lib/graphs/goodsales_v2/process_stage_history.grf +184 -0
  52. data/lib/graphs/goodsales_v2/process_velocity_duration.grf +140 -0
  53. data/lib/graphs/process_account.grf +1 -1
  54. data/lib/graphs/process_activity.grf +1 -1
  55. data/lib/graphs/process_activity_dim.grf +1 -1
  56. data/lib/graphs/process_activity_owner.grf +1 -1
  57. data/lib/graphs/process_forecast.grf +1 -1
  58. data/lib/graphs/process_opp_records.grf +1 -1
  59. data/lib/graphs/process_opportunity.grf +1 -1
  60. data/lib/graphs/process_opportunity_line_item.grf +1 -1
  61. data/lib/graphs/process_opportunity_snapshot.grf +1 -1
  62. data/lib/graphs/process_owner.grf +1 -1
  63. data/lib/graphs/process_stage.grf +1 -1
  64. data/lib/graphs/process_stage_history.grf +1 -1
  65. data/lib/graphs/process_velocity_duration.grf +1 -1
  66. data/lib/nodes/clover_gen.rb +59 -946
  67. data/lib/nodes/dependency.rb +95 -96
  68. data/lib/runtime.rb +7 -648
  69. data/lib/utils/utils.rb +66 -0
  70. data/templates/flow.rb.erb +7 -6
  71. data/templates/join_template.grf.erb +1 -1
  72. data/templates/reformat_template.grf.erb +1 -1
  73. data/templates/sink.json.erb +28 -0
  74. data/templates/tap.json.erb +3 -5
  75. data/templates/workspace.prm.erb +4 -0
  76. metadata +50 -8
  77. data/lib/contract_checkers/contract_checkers.rb +0 -53
  78. data/lib/dsl/project_dsl.rb +0 -259
  79. data/lib/repo/1_config.json +0 -8
  80. data/templates/dataset.json.erb +0 -13
  81. data/templates/source.json.erb +0 -22
@@ -0,0 +1,445 @@
1
+ module GoodData
2
+ module Bam
3
+ module Validators
4
+
5
+ include GoodData::Bam
6
+ include GoodData::CloudConnect
7
+
8
+ def self.ctl_validation(name, validation)
9
+ ctl = case validation[:type]
10
+ when "integer"
11
+ "!matches($in.0.#{name}, \"^[0-9]+$\")"
12
+ when "decimal"
13
+ "!isNumber($in.0.#{name})"
14
+ when "url"
15
+ "!isUrl($in.0.#{name})"
16
+ when "date"
17
+ "!isDate($in.0.#{name}, \"#{validation[:format]}\")"
18
+ else
19
+ fail "Uknown validation type \"#{validation[:type]}\""
20
+ end
21
+
22
+ format = validation[:format] || ''
23
+
24
+ "fieldName = \"#{name}\";\nvalidationType = \"#{validation[:type]}\";\nif (" + ctl + ") {currentLineFailed = true;failed = true;$out.1.all = \"Error in file '${FILE}' on line \" + line + \" at field \" + fieldName + \". Expected \" + validationType + \" #{format} got '\" + currentValue + \"'\";};"
25
+ end
26
+
27
+ def self.backup_files_from_glob(file, options={})
28
+ files_to_backup = Array(options[:files_to_backup])
29
+ remote_url = options[:remote_url]
30
+ File.open(file, "w") do |f|
31
+
32
+ builder = Builder::XmlMarkup.new(:target=>f, :indent=>2)
33
+ builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
34
+ builder.Graph({
35
+ :name => "Contract Checker: File list reader"
36
+ }) do
37
+ builder.Global do
38
+ Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
39
+ Helpers::property_file(builder, {:id => "now_params", :fileURL => "now.prm"})
40
+
41
+ Helpers::create_trash_meta(builder)
42
+ Helpers::create_file_list_meta(builder)
43
+
44
+ end
45
+
46
+ builder.Phase(:number => 1) do
47
+ ctl = "function integer generate() {$out.0.all = 'Just creating this file';return OK;}"
48
+ Core::build_node2(builder, Nodes.data_generator2({:name => "generate", :id => "generate", :generate => ctl}))
49
+ Core::build_node2(builder, Nodes.edge2({:toNode => "create_dir:0", :fromNode => "generate:0", :metadata => "trash_metadata"}))
50
+ Core::build_node2(builder, Nodes.writer2({
51
+ :fileURL => remote_url + "touch.txt",
52
+ :name => "create_dir",
53
+ :id => "create_dir"
54
+ }))
55
+ end
56
+ phase = 2
57
+
58
+ files_to_backup.each do |file_to_backup|
59
+ builder.Phase(:number => phase += 1) do
60
+ Core::build_node2(builder, Nodes.backup2({
61
+ :baseURL => remote_url,
62
+ :sourcePath => file_to_backup,
63
+ :name => "Backup",
64
+ :id => "backup",
65
+ :mode => "DO_NOT_COMPRESS"
66
+ }))
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
72
+
73
+ def self.generate_checker_success(file, tap)
74
+ id = tap[:id]
75
+ File.open(file, "w") do |f|
76
+
77
+ builder = Builder::XmlMarkup.new(:target=>f, :indent=>2)
78
+ builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
79
+ builder.Graph({
80
+ :name => "Contract Checker: File list reader"
81
+ }) do
82
+ builder.Global do
83
+ Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
84
+
85
+ Helpers::create_trash_meta(builder)
86
+
87
+ builder.Metadata({:id => "file_list"}) do |builder|
88
+ Helpers::csv_metadata(builder, {
89
+ :name => "file_list",
90
+ :fields => [
91
+ {:name => "filePath", :type => "string"},
92
+ {:name => "fileName", :type => "string"}
93
+ ]
94
+ })
95
+ end
96
+ end
97
+
98
+ builder.Phase(:number => 0) do
99
+ Core::build_node2(builder, Nodes.reader2({
100
+ :id => "#{id}_reader",
101
+ :guiName => "#{id}_reader",
102
+ :name => "#{id}_reader",
103
+ :dataPolicy => "Strict",
104
+ :skipRows => 0,
105
+ :fileURL => "data/#{id}_files_to_read.csv"
106
+ }))
107
+
108
+ reformat_func = <<HEREDOC
109
+ function integer transform() {
110
+ //$out.0.filePath = replace($in.0.filePath, ':file', '');
111
+ $out.0.filePath = chop($in.0.filePath, $in.0.fileName);
112
+ $out.0.fileName = $in.0.fileName;
113
+ return ALL;
114
+ }
115
+ HEREDOC
116
+ Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat_graph", :id => "#{id}_reformat", :transformation => reformat_func }))
117
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_reader:0", :metadata => "file_list" }))
118
+
119
+ Core::build_node2(builder, Nodes.file_copy2({
120
+ :baseURL => "${filePath}",
121
+ :sourcePath => "${fileName}",
122
+ :name => "#{id} Backup",
123
+ :id => "#{id}_backup",
124
+ :operation => "MOVE",
125
+ :targetPath => "validated/"
126
+ }))
127
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_backup:0", :fromNode => "#{id}_reformat:0", :metadata => "file_list" }))
128
+ end
129
+ end
130
+ end
131
+ end
132
+
133
+ def self.generate_checker_file_list(file, taps)
134
+ File.open(file, "w") do |f|
135
+
136
+ builder = Builder::XmlMarkup.new(:target=>f, :indent=>2)
137
+ builder.instruct! :xml, :version => "1.0", :encoding => "UTF-8"
138
+ builder.Graph({
139
+ :name => "Contract Checker: File list reader"
140
+ }) do
141
+ builder.Global do
142
+ Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
143
+
144
+ Helpers::create_trash_meta(builder)
145
+ Helpers::create_run_graph_failure_metadata(builder)
146
+
147
+ builder.Metadata({:id => "file_list"}) do |builder|
148
+ Helpers::csv_metadata(builder, {
149
+ :name => "file_list",
150
+ :fields => [
151
+ {:name => "filePath", :type => "string"},
152
+ {:name => "fileName", :type => "string"}
153
+ ]
154
+ })
155
+ end
156
+ end
157
+
158
+ reformat_func = <<HEREDOC
159
+
160
+ function integer transform() {
161
+ $out.0.filePath = replace($in.0.filePath, "${GDC_WEBDAV_HOST}", replace(replace(\"${GD_LOGIN}\",\"@\",\"%40\"),\"\\\\+\",\"%2B\") + ":${GD_PASSWORD}@${GDC_WEBDAV_HOST}");
162
+ $out.0.fileName = $in.0.fileName;
163
+ return ALL;
164
+ }
165
+ HEREDOC
166
+
167
+ i = 0
168
+
169
+ builder.Phase(:number => i += 1) do
170
+ Core::build_node2(builder, Nodes.data_generator2({:name => "now_generator", :id => "now_generator", :generate => "function integer generate() {$out.0.all = \"NOW=\" + toString(date2long(today()));return OK;}"}))
171
+ Core::build_node2(builder, Nodes.writer2({:name => "NOW Writer", :id => "now_csv_writer", :fileURL => "now.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
172
+ Core::build_node2(builder, Nodes.edge2({:toNode => "now_csv_writer:0", :fromNode => "now_generator:0", :metadata => "trash_metadata"}))
173
+ end
174
+
175
+ taps.each do |tap|
176
+ source = tap[:source]
177
+ id = tap[:id]
178
+
179
+ builder.Phase(:number => i += 1) do
180
+ #Core::build_node2(builder, Nodes.file_list2({:id => "#{id}_file_list", :name => "#{id}_file_list", :dataPolicy => "Strict", :baseURL => "#{tap[:validation_source]}", :output_mapping => Nodes::MAP_ALL}))
181
+ Core::build_node2(builder, Nodes.reader2({:id => "#{id}_file_list", :name => "#{id}_file_list", :fileURL => "${gdc_agent_file}" }))
182
+ Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_reformat", :transformation => reformat_func}))
183
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_file_list:0", :metadata => "file_list"}))
184
+ Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_writer", :fileURL => "data/#{id}_files_to_read.csv"}))
185
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_writer:0", :fromNode => "#{id}_reformat:0", :metadata => "file_list"}))
186
+ end
187
+
188
+ builder.Phase(:number => i += 1) do
189
+ ctl = "function integer generate() {$out.0.all = \"#{id}_SKIP_LINES=0\";return OK;}"
190
+ Core::build_node2(builder, Nodes.data_generator2({:name => "#{id}_generator", :id => "#{id}_generator", :generate => ctl}))
191
+ Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_csv_writer", :fileURL => "#{id}_counter.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
192
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_csv_writer:0", :fromNode => "#{id}_generator:0", :metadata => "trash_metadata"}))
193
+ end
194
+
195
+ subgraph_reformat_func = <<HEREDOC
196
+
197
+ function integer transform() {
198
+ $out.0.all = "graphs/#{id}_loop.grf";
199
+ return ALL;
200
+ }
201
+ HEREDOC
202
+
203
+ fail_reformat = <<HEREDOC
204
+ function integer transform() {
205
+ raiseError("Loop failed");
206
+ }
207
+ HEREDOC
208
+
209
+
210
+ builder.Phase(:number => i += 1) do
211
+ Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat graph", :id => "#{id}_reformat_graph", :transformation => subgraph_reformat_func}))
212
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_graph:0", :fromNode => "#{id}_reformat:1", :metadata => "file_list"}))
213
+ Core::build_node2(builder, Nodes.run_graph2({
214
+ :guiName => id,
215
+ :name => id,
216
+ :id => "#{id}_run_graph"
217
+ }))
218
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_run_graph:0", :fromNode => "#{id}_reformat_graph:0", :metadata => "trash_metadata"}))
219
+ Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat fail", :id => "#{id}_reformat_fail", :transformation => fail_reformat}))
220
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_fail:0", :fromNode => "#{id}_run_graph:1", :metadata => "run_graph_failure_metadata"}))
221
+ Core::build_node2(builder, Nodes.trash2({:name => "#{id}_trash", :id => "#{id}_trash", :debugPrint => true}))
222
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_trash:0", :fromNode => "#{id}_reformat_fail:0", :metadata => "run_graph_failure_metadata"}))
223
+ end
224
+ end
225
+
226
+ # If there is a fail.
227
+ # Copies the report files
228
+ taps.each do |tap|
229
+ id = tap[:id]
230
+ builder.Phase(:number => i += 1) do
231
+ Core::build_node2(builder, Nodes.reader2({
232
+ :id => "#{id}_files_copy_check_reader",
233
+ :guiName => "#{id}_files failure_check_reader",
234
+ :name => "#{id}_files_failure_check_reader",
235
+ :fileURL => "data/has_errors_*.csv",
236
+ :skipRows => 0,
237
+ :numRecords => 1
238
+ }))
239
+
240
+ fail_reformat = <<HEREDOC
241
+ function integer transform() {
242
+ $out.0.all = "graphs/#{id}_failure.grf";
243
+ return ALL;
244
+ }
245
+ HEREDOC
246
+
247
+ Core::build_node2(builder, Nodes.reformat2({:name => "#{id}_files_copy_check_reformat", :id => "#{id}_files_copy_check_reformat", :transformation => fail_reformat}))
248
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_files_copy_check_reformat:0", :fromNode => "#{id}_files_copy_check_reader:0", :metadata => "trash_metadata"}))
249
+
250
+ Core::build_node2(builder, Nodes.run_graph2(
251
+ {
252
+ :guiName => "#{id}_failure_run_graph",
253
+ :name => "#{id}_failure_run_graph",
254
+ :id => "#{id}_failure_run_graph"
255
+ }))
256
+
257
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_failure_run_graph:0", :fromNode => "#{id}_files_copy_check_reformat:0", :metadata => "trash_metadata" }))
258
+ end
259
+ end
260
+
261
+ # If there was a failure
262
+ # Kill the graph
263
+ builder.Phase(:number => i += 1) do
264
+ Core::build_node2(builder, Nodes.reader2({
265
+ :id => "failure_check_reader",
266
+ :guiName => "failure_check_reader",
267
+ :name => "failure_check_reader",
268
+ :fileURL => "data/has_errors_*.csv",
269
+ :skipRows => 0
270
+ }))
271
+
272
+ failure_func = <<HEREDOC
273
+
274
+ function integer transform() {
275
+ raiseError("Validation did not pass");
276
+ }
277
+ HEREDOC
278
+ Core::build_node2(builder, Nodes.reformat2({:name => "failure_dedup", :id => "failure_dedup", :transformation => failure_func }))
279
+ Core::build_node2(builder, Nodes.edge2({:toNode => "failure_dedup:0", :fromNode => "failure_check_reader:0", :metadata => "trash_metadata" }))
280
+
281
+ Core::build_node2(builder, Nodes.trash2({:name => "failure_check_trash", :id => "failure_check_trash", :debugPrint => true}))
282
+ Core::build_node2(builder, Nodes.edge2({:toNode => "failure_check_trash:0", :fromNode => "failure_dedup:0", :metadata => "trash_metadata" }))
283
+ end
284
+
285
+ # If everything is success.
286
+ # Copies the files
287
+ taps.each do |tap|
288
+ builder.Phase(:number => i += 1) do
289
+
290
+ Core::build_node2(builder, Nodes.run_graph2(
291
+ {
292
+ :guiName => "success_run_graph",
293
+ :name => "success_run_graph",
294
+ :id => "success_run_graph",
295
+ :graphName => "graphs/#{tap[:id]}_success.grf"
296
+ }))
297
+ end
298
+ end
299
+ end
300
+ end
301
+ end
302
+
303
+ def self.generate_contract(file, tap)
304
+ name = id = tap[:id]
305
+
306
+ File.open(file, "w") do |f|
307
+
308
+ builder = Builder::XmlMarkup.new(:target=>f, :indent=>2)
309
+ builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
310
+ builder.Graph({
311
+ :name => "Contract Checker"
312
+ }) do
313
+ builder.Global do
314
+ Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
315
+ Helpers::property_file(builder, {:id => "file_params", :fileURL => "#{id}_item.prm"})
316
+
317
+ Helpers::create_trash_meta(builder)
318
+ Helpers::create_read_error_meta(builder)
319
+
320
+ builder.Metadata({:id => "#{name}_clover_metadata"}) do |builder|
321
+ Helpers::csv_metadata(builder, Metadata.get_source_metadata(tap))
322
+ end
323
+ end
324
+
325
+ builder.Phase(:number => 0) do
326
+
327
+
328
+ reformat_func = <<HEREDOC
329
+ long line = 0;
330
+ string fieldName;
331
+ string validationType;
332
+ string currentValue;
333
+ boolean failed;
334
+ boolean currentLineFailed;
335
+
336
+ function integer transform() {
337
+ line = line + 1;
338
+ currentLineFailed = false;
339
+
340
+ #{tap[:fields].find_all {|d| d.has_key?(:validates_as)}.map do |d|
341
+ "currentValue = $in.0." + d[:name] + ";" + ctl_validation(d[:name], d[:validates_as])
342
+ end.join("\n")}
343
+ if (currentLineFailed) {
344
+ return 1;
345
+ } else {
346
+ return 0;
347
+ }
348
+ }
349
+
350
+ HEREDOC
351
+
352
+
353
+ has_error_reformat_func = <<HEREDOC
354
+ function integer transform() {
355
+ $out.0.all = "${FILE}";
356
+ return ALL;
357
+ }
358
+
359
+ HEREDOC
360
+
361
+ format_clover_error = <<HEREDOC
362
+ function integer transform() {
363
+ $out.0.all = "Parsing error occured on line " + $in.0.line_number + " of file " + $in.0.file + " on record " + $in.0.record + " with message " + $in.0.message ;
364
+ return ALL;
365
+ }
366
+
367
+ HEREDOC
368
+
369
+ Core::build_node2(builder, Nodes.reader2({:id => "#{id}_reader", :name => "#{id}_reader", :dataPolicy => "controlled", :fileURL => "${FILE}"}))
370
+ Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_reformat", :transformation => reformat_func}))
371
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_reader:0", :metadata => "#{id}_clover_metadata"}))
372
+
373
+ Core::build_node2(builder, Nodes.reformat2({:name => "#{id} reader error Reformat", :id => "#{id}_reader_error_reformat", :transformation => format_clover_error}))
374
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reader_error_reformat:0", :fromNode => "#{id}_reader:1", :metadata => "reader_error_metadata"}))
375
+
376
+ Core::build_node2(builder, Nodes.gather2({:name => "#{id} Gather", :id => "#{id}_gather"}))
377
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_gather:1", :fromNode => "#{id}_reader_error_reformat:0", :metadata => "trash_metadata"}))
378
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_gather:0", :fromNode => "#{id}_reformat:1", :metadata => "trash_metadata"}))
379
+
380
+ Core::build_node2(builder, Nodes.trash2({:name => "PARAMS CSV Writer", :id => "#{id}_trash", :debugPrint => false }))
381
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_trash:0", :fromNode => "#{id}_reformat:0", :metadata => "trash_metadata"}))
382
+
383
+ Core::build_node2(builder, Nodes.copy2({:name => "#{id} copy", :id => "#{id}_copy"}))
384
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_copy:0", :fromNode => "#{id}_gather:0", :metadata => "trash_metadata"}))
385
+
386
+ Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_writer", :fileURL => "data/validation_output_#{id}.csv", :outputFieldNames => "false", :append => "true", :quotedStrings => "true"}))
387
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_writer:0", :fromNode => "#{id}_copy:0", :metadata => "trash_metadata", :append => true}))
388
+
389
+ Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_has_error_reformat", :transformation => has_error_reformat_func}))
390
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_has_error_reformat:0", :fromNode => "#{id}_copy:1", :metadata => "trash_metadata"}))
391
+
392
+ Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_has_error_writer", :fileURL => "data/has_errors_#{id}.csv", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
393
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_has_error_writer:0", :fromNode => "#{id}_has_error_reformat:0", :metadata => "trash_metadata"}))
394
+ end
395
+ end
396
+ end
397
+ end
398
+
399
+ # def self.generate(file)
400
+ # File.open(file, "w") do |file|
401
+ #
402
+ # builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
403
+ # builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
404
+ # builder.Graph({
405
+ # :name => "Contract Checker"
406
+ # }) do
407
+ # builder.Global do
408
+ # Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
409
+ # end
410
+ #
411
+ # builder.Phase(:number => 0) do
412
+ #
413
+ # end
414
+ #
415
+ # # phase = 0
416
+ # # subgraphs.each do |subgraph|
417
+ # # builder.Phase(:number => phase+1) do
418
+ # # id1 = get_id
419
+ # # id2 = get_id
420
+ # # ctl = "function integer generate() {$out.0.all = \"FLOW=#{subgraph[:flow]}\";return OK;}"
421
+ # # build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
422
+ # # build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
423
+ # # build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
424
+ # # end
425
+ # # builder.Phase(:number => phase+2) do
426
+ #
427
+ # # id1 = get_id
428
+ # # id2 = get_id
429
+ # # ctl = "function integer generate() {$out.0.all = \"NAME=#{subgraph[:name]}\";return OK;}"
430
+ # # build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
431
+ # # build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
432
+ # # build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
433
+ # # end
434
+ #
435
+ # # builder.Phase(:number => phase+3) do
436
+ # # build_node2(builder, GoodData::CloverGenerator::Nodes.run_graph2({:guiName => subgraph[:name], :name => subgraph[:name], :id => subgraph[:flow], :graphName => subgraph[:file]}))
437
+ # # end
438
+ # # phase += 4
439
+ # # end
440
+ # end
441
+ # end
442
+ # end
443
+ end
444
+ end
445
+ end
@@ -4,7 +4,7 @@
4
4
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
5
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
6
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
- <Property fileURL="params.txt" id="GraphParameter0"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
8
  <Property fileURL="workspace.prm" id="GraphParameter0"/>
9
9
  <Dictionary/>
10
10
  </Global>
data/lib/graphs/dummy.grf CHANGED
@@ -4,7 +4,7 @@
4
4
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
5
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
6
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
- <Property fileURL="params.txt" id="GraphParameter0"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
8
  <Property fileURL="workspace.prm" id="GraphParameter1"/>
9
9
  <Dictionary/>
10
10
  </Global>
@@ -0,0 +1,47 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <Graph author="fluke" created="Tue Feb 05 15:38:24 PST 2013" guiVersion="3.3.1" id="1360179808937" licenseType="Commercial" modified="Fri Feb 22 12:18:42 PST 2013" modifiedBy="fluke" name="process_name" revision="1.13" showComponentDetails="true">
3
+ <Global>
4
+ <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
+ <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
+ <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
+ <Property fileURL="workspace.prm" id="GraphParameter0"/>
9
+ <Dictionary/>
10
+ </Global>
11
+ <Phase number="0">
12
+ <Node enabled="enabled" fileURL="data/1_in.csv" guiHeight="77" guiName="CSV Reader" guiWidth="128" guiX="124" guiY="169" id="DATA_READER0" quoteCharacter="&quot;" quotedStrings="true" skipRows="1" type="DATA_READER"/>
13
+ <Node enabled="enabled" fileURL="data/out.csv" guiHeight="89" guiName="CSV Writer" guiWidth="128" guiX="609" guiY="169" id="DATA_WRITER0" outputFieldNames="true" quoteCharacter="&quot;" quotedStrings="true" type="DATA_WRITER"/>
14
+ <Node enabled="enabled" guiHeight="65" guiName="Reformat" guiWidth="128" guiX="365" guiY="175" id="REFORMAT0" type="REFORMAT">
15
+ <attr name="transform"><![CDATA[//#CTL2
16
+
17
+ // Transforms input record into output record.
18
+ function integer transform() {
19
+ $out.0.* = $in.0.*;
20
+ $out.0.Name = "Docent " + $in.0.Name;
21
+
22
+ return OK;
23
+ }
24
+
25
+ // Called during component initialization.
26
+ // function boolean init() {}
27
+
28
+ // Called during each graph run before the transform is executed. May be used to allocate and initialize resources
29
+ // required by the transform. All resources allocated within this method should be released
30
+ // by the postExecute() method.
31
+ // function void preExecute() {}
32
+
33
+ // Called only if transform() throws an exception.
34
+ // function integer transformOnError(string errorMessage, string stackTrace) {}
35
+
36
+ // Called during each graph run after the entire transform was executed. Should be used to free any resources
37
+ // allocated within the preExecute() method.
38
+ // function void postExecute() {}
39
+
40
+ // Called to return a user-defined error message when an error occurs.
41
+ // function string getMessage() {}
42
+ ]]></attr>
43
+ </Node>
44
+ <Edge fromNode="DATA_READER0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge0" inPort="Port 0 (in)" metadata="Metadata0" outPort="Port 0 (output)" toNode="REFORMAT0:0"/>
45
+ <Edge fromNode="REFORMAT0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge1" inPort="Port 0 (in)" metadata="Metadata1" outPort="Port 0 (out)" toNode="DATA_WRITER0:0"/>
46
+ </Phase>
47
+ </Graph>
@@ -0,0 +1,46 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <Graph author="fluke" created="Tue Feb 05 15:38:24 PST 2013" guiVersion="3.3.1" id="1360179808937" licenseType="Commercial" modified="Thu Feb 21 23:29:16 PST 2013" modifiedBy="fluke" name="process_name" revision="1.11" showComponentDetails="true">
3
+ <Global>
4
+ <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
+ <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
+ <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
+ <Property fileURL="workspace.prm" id="GraphParameter1"/>
9
+ <Dictionary/>
10
+ </Global>
11
+ <Phase number="0">
12
+ <Node enabled="enabled" fileURL="data/1_in.csv" guiHeight="77" guiName="CSV Reader" guiWidth="128" guiX="124" guiY="169" id="DATA_READER0" quoteCharacter="&quot;" quotedStrings="true" skipRows="1" type="DATA_READER"/>
13
+ <Node enabled="enabled" fileURL="data/out.csv" guiHeight="77" guiName="CSV Writer" guiWidth="128" guiX="609" guiY="169" id="DATA_WRITER0" outputFieldNames="true" quoteCharacter="&quot;" quotedStrings="true" type="DATA_WRITER"/>
14
+ <Node enabled="enabled" guiHeight="65" guiName="Reformat" guiWidth="128" guiX="365" guiY="175" id="REFORMAT0" type="REFORMAT">
15
+ <attr name="transform"><![CDATA[//#CTL2
16
+
17
+ // Transforms input record into output record.
18
+ function integer transform() {
19
+ $out.0.* = $in.0.*;
20
+
21
+ return OK;
22
+ }
23
+
24
+ // Called during component initialization.
25
+ // function boolean init() {}
26
+
27
+ // Called during each graph run before the transform is executed. May be used to allocate and initialize resources
28
+ // required by the transform. All resources allocated within this method should be released
29
+ // by the postExecute() method.
30
+ // function void preExecute() {}
31
+
32
+ // Called only if transform() throws an exception.
33
+ // function integer transformOnError(string errorMessage, string stackTrace) {}
34
+
35
+ // Called during each graph run after the entire transform was executed. Should be used to free any resources
36
+ // allocated within the preExecute() method.
37
+ // function void postExecute() {}
38
+
39
+ // Called to return a user-defined error message when an error occurs.
40
+ // function string getMessage() {}
41
+ ]]></attr>
42
+ </Node>
43
+ <Edge fromNode="DATA_READER0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge0" inPort="Port 0 (in)" metadata="Metadata0" outPort="Port 0 (output)" toNode="REFORMAT0:0"/>
44
+ <Edge fromNode="REFORMAT0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge1" inPort="Port 0 (in)" metadata="Metadata1" outPort="Port 0 (out)" toNode="DATA_WRITER0:0"/>
45
+ </Phase>
46
+ </Graph>