gd_bam 0.0.15 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +313 -5
- data/bin/bam +126 -48
- data/lib/bam/version.rb +1 -1
- data/lib/bam.rb +51 -0
- data/lib/base/errors.rb +15 -0
- data/lib/base/flow.rb +37 -0
- data/lib/base/graph.rb +23 -0
- data/lib/base/metadata.rb +107 -0
- data/lib/base/project.rb +95 -0
- data/lib/base/repo.rb +35 -0
- data/lib/base/sink.rb +44 -0
- data/lib/base/step.rb +47 -0
- data/lib/base/tap.rb +167 -0
- data/lib/base/taps.rb +19 -0
- data/lib/cloud_connect/dsl/cc.rb +42 -0
- data/lib/cloud_connect/dsl/es_helpers.rb +49 -0
- data/lib/cloud_connect/dsl/helpers.rb +199 -0
- data/lib/{nodes → cloud_connect/dsl}/nodes.rb +106 -16
- data/lib/cloud_connect/dsl/sf_helpers.rb +39 -0
- data/lib/cloud_connect/dsl/structure_helpers.rb +94 -0
- data/lib/commands/commands.rb +110 -0
- data/lib/commands/deployment.rb +217 -0
- data/lib/commands/docs_commands.rb +41 -0
- data/lib/commands/gd_commands.rb +95 -0
- data/lib/commands/scaffold_commands.rb +103 -0
- data/lib/commands/sf_commands.rb +37 -0
- data/lib/commands/validators.rb +19 -0
- data/lib/compatibility.rb +19 -0
- data/lib/compiler/compiler.rb +76 -0
- data/lib/compiler/etl_visitor.rb +165 -0
- data/lib/dsl/dsl.rb +125 -0
- data/lib/generators/downloaders.rb +449 -0
- data/lib/generators/etl.rb +261 -0
- data/lib/generators/validators.rb +445 -0
- data/lib/graphs/docentize.grf +1 -1
- data/lib/graphs/dummy.grf +1 -1
- data/lib/graphs/goodsales_v2/docentize.grf +47 -0
- data/lib/graphs/goodsales_v2/dummy.grf +46 -0
- data/lib/graphs/goodsales_v2/load_history.grf +579 -0
- data/lib/graphs/goodsales_v2/process_account.grf +47 -0
- data/lib/graphs/goodsales_v2/process_activity.grf +222 -0
- data/lib/graphs/goodsales_v2/process_activity_dim.grf +88 -0
- data/lib/graphs/goodsales_v2/process_activity_owner.grf +48 -0
- data/lib/graphs/goodsales_v2/process_forecast.grf +20 -0
- data/lib/graphs/goodsales_v2/process_opp_records.grf +84 -0
- data/lib/graphs/goodsales_v2/process_opportunity.grf +46 -0
- data/lib/graphs/goodsales_v2/process_opportunity_line_item.grf +171 -0
- data/lib/graphs/goodsales_v2/process_opportunity_snapshot.grf +94 -0
- data/lib/graphs/goodsales_v2/process_owner.grf +48 -0
- data/lib/graphs/goodsales_v2/process_stage.grf +51 -0
- data/lib/graphs/goodsales_v2/process_stage_history.grf +184 -0
- data/lib/graphs/goodsales_v2/process_velocity_duration.grf +140 -0
- data/lib/graphs/process_account.grf +1 -1
- data/lib/graphs/process_activity.grf +1 -1
- data/lib/graphs/process_activity_dim.grf +1 -1
- data/lib/graphs/process_activity_owner.grf +1 -1
- data/lib/graphs/process_forecast.grf +1 -1
- data/lib/graphs/process_opp_records.grf +1 -1
- data/lib/graphs/process_opportunity.grf +1 -1
- data/lib/graphs/process_opportunity_line_item.grf +1 -1
- data/lib/graphs/process_opportunity_snapshot.grf +1 -1
- data/lib/graphs/process_owner.grf +1 -1
- data/lib/graphs/process_stage.grf +1 -1
- data/lib/graphs/process_stage_history.grf +1 -1
- data/lib/graphs/process_velocity_duration.grf +1 -1
- data/lib/nodes/clover_gen.rb +59 -946
- data/lib/nodes/dependency.rb +95 -96
- data/lib/runtime.rb +7 -648
- data/lib/utils/utils.rb +66 -0
- data/templates/flow.rb.erb +7 -6
- data/templates/join_template.grf.erb +1 -1
- data/templates/reformat_template.grf.erb +1 -1
- data/templates/sink.json.erb +28 -0
- data/templates/tap.json.erb +3 -5
- data/templates/workspace.prm.erb +4 -0
- metadata +50 -8
- data/lib/contract_checkers/contract_checkers.rb +0 -53
- data/lib/dsl/project_dsl.rb +0 -259
- data/lib/repo/1_config.json +0 -8
- data/templates/dataset.json.erb +0 -13
- data/templates/source.json.erb +0 -22
@@ -0,0 +1,445 @@
|
|
1
|
+
module GoodData
|
2
|
+
module Bam
|
3
|
+
module Validators
|
4
|
+
|
5
|
+
include GoodData::Bam
|
6
|
+
include GoodData::CloudConnect
|
7
|
+
|
8
|
+
def self.ctl_validation(name, validation)
|
9
|
+
ctl = case validation[:type]
|
10
|
+
when "integer"
|
11
|
+
"!matches($in.0.#{name}, \"^[0-9]+$\")"
|
12
|
+
when "decimal"
|
13
|
+
"!isNumber($in.0.#{name})"
|
14
|
+
when "url"
|
15
|
+
"!isUrl($in.0.#{name})"
|
16
|
+
when "date"
|
17
|
+
"!isDate($in.0.#{name}, \"#{validation[:format]}\")"
|
18
|
+
else
|
19
|
+
fail "Uknown validation type \"#{validation[:type]}\""
|
20
|
+
end
|
21
|
+
|
22
|
+
format = validation[:format] || ''
|
23
|
+
|
24
|
+
"fieldName = \"#{name}\";\nvalidationType = \"#{validation[:type]}\";\nif (" + ctl + ") {currentLineFailed = true;failed = true;$out.1.all = \"Error in file '${FILE}' on line \" + line + \" at field \" + fieldName + \". Expected \" + validationType + \" #{format} got '\" + currentValue + \"'\";};"
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.backup_files_from_glob(file, options={})
|
28
|
+
files_to_backup = Array(options[:files_to_backup])
|
29
|
+
remote_url = options[:remote_url]
|
30
|
+
File.open(file, "w") do |f|
|
31
|
+
|
32
|
+
builder = Builder::XmlMarkup.new(:target=>f, :indent=>2)
|
33
|
+
builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
|
34
|
+
builder.Graph({
|
35
|
+
:name => "Contract Checker: File list reader"
|
36
|
+
}) do
|
37
|
+
builder.Global do
|
38
|
+
Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
|
39
|
+
Helpers::property_file(builder, {:id => "now_params", :fileURL => "now.prm"})
|
40
|
+
|
41
|
+
Helpers::create_trash_meta(builder)
|
42
|
+
Helpers::create_file_list_meta(builder)
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
builder.Phase(:number => 1) do
|
47
|
+
ctl = "function integer generate() {$out.0.all = 'Just creating this file';return OK;}"
|
48
|
+
Core::build_node2(builder, Nodes.data_generator2({:name => "generate", :id => "generate", :generate => ctl}))
|
49
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "create_dir:0", :fromNode => "generate:0", :metadata => "trash_metadata"}))
|
50
|
+
Core::build_node2(builder, Nodes.writer2({
|
51
|
+
:fileURL => remote_url + "touch.txt",
|
52
|
+
:name => "create_dir",
|
53
|
+
:id => "create_dir"
|
54
|
+
}))
|
55
|
+
end
|
56
|
+
phase = 2
|
57
|
+
|
58
|
+
files_to_backup.each do |file_to_backup|
|
59
|
+
builder.Phase(:number => phase += 1) do
|
60
|
+
Core::build_node2(builder, Nodes.backup2({
|
61
|
+
:baseURL => remote_url,
|
62
|
+
:sourcePath => file_to_backup,
|
63
|
+
:name => "Backup",
|
64
|
+
:id => "backup",
|
65
|
+
:mode => "DO_NOT_COMPRESS"
|
66
|
+
}))
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.generate_checker_success(file, tap)
|
74
|
+
id = tap[:id]
|
75
|
+
File.open(file, "w") do |f|
|
76
|
+
|
77
|
+
builder = Builder::XmlMarkup.new(:target=>f, :indent=>2)
|
78
|
+
builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
|
79
|
+
builder.Graph({
|
80
|
+
:name => "Contract Checker: File list reader"
|
81
|
+
}) do
|
82
|
+
builder.Global do
|
83
|
+
Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
|
84
|
+
|
85
|
+
Helpers::create_trash_meta(builder)
|
86
|
+
|
87
|
+
builder.Metadata({:id => "file_list"}) do |builder|
|
88
|
+
Helpers::csv_metadata(builder, {
|
89
|
+
:name => "file_list",
|
90
|
+
:fields => [
|
91
|
+
{:name => "filePath", :type => "string"},
|
92
|
+
{:name => "fileName", :type => "string"}
|
93
|
+
]
|
94
|
+
})
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
builder.Phase(:number => 0) do
|
99
|
+
Core::build_node2(builder, Nodes.reader2({
|
100
|
+
:id => "#{id}_reader",
|
101
|
+
:guiName => "#{id}_reader",
|
102
|
+
:name => "#{id}_reader",
|
103
|
+
:dataPolicy => "Strict",
|
104
|
+
:skipRows => 0,
|
105
|
+
:fileURL => "data/#{id}_files_to_read.csv"
|
106
|
+
}))
|
107
|
+
|
108
|
+
reformat_func = <<HEREDOC
|
109
|
+
function integer transform() {
|
110
|
+
//$out.0.filePath = replace($in.0.filePath, ':file', '');
|
111
|
+
$out.0.filePath = chop($in.0.filePath, $in.0.fileName);
|
112
|
+
$out.0.fileName = $in.0.fileName;
|
113
|
+
return ALL;
|
114
|
+
}
|
115
|
+
HEREDOC
|
116
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat_graph", :id => "#{id}_reformat", :transformation => reformat_func }))
|
117
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_reader:0", :metadata => "file_list" }))
|
118
|
+
|
119
|
+
Core::build_node2(builder, Nodes.file_copy2({
|
120
|
+
:baseURL => "${filePath}",
|
121
|
+
:sourcePath => "${fileName}",
|
122
|
+
:name => "#{id} Backup",
|
123
|
+
:id => "#{id}_backup",
|
124
|
+
:operation => "MOVE",
|
125
|
+
:targetPath => "validated/"
|
126
|
+
}))
|
127
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_backup:0", :fromNode => "#{id}_reformat:0", :metadata => "file_list" }))
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
def self.generate_checker_file_list(file, taps)
|
134
|
+
File.open(file, "w") do |f|
|
135
|
+
|
136
|
+
builder = Builder::XmlMarkup.new(:target=>f, :indent=>2)
|
137
|
+
builder.instruct! :xml, :version => "1.0", :encoding => "UTF-8"
|
138
|
+
builder.Graph({
|
139
|
+
:name => "Contract Checker: File list reader"
|
140
|
+
}) do
|
141
|
+
builder.Global do
|
142
|
+
Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
|
143
|
+
|
144
|
+
Helpers::create_trash_meta(builder)
|
145
|
+
Helpers::create_run_graph_failure_metadata(builder)
|
146
|
+
|
147
|
+
builder.Metadata({:id => "file_list"}) do |builder|
|
148
|
+
Helpers::csv_metadata(builder, {
|
149
|
+
:name => "file_list",
|
150
|
+
:fields => [
|
151
|
+
{:name => "filePath", :type => "string"},
|
152
|
+
{:name => "fileName", :type => "string"}
|
153
|
+
]
|
154
|
+
})
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
reformat_func = <<HEREDOC
|
159
|
+
|
160
|
+
function integer transform() {
|
161
|
+
$out.0.filePath = replace($in.0.filePath, "${GDC_WEBDAV_HOST}", replace(replace(\"${GD_LOGIN}\",\"@\",\"%40\"),\"\\\\+\",\"%2B\") + ":${GD_PASSWORD}@${GDC_WEBDAV_HOST}");
|
162
|
+
$out.0.fileName = $in.0.fileName;
|
163
|
+
return ALL;
|
164
|
+
}
|
165
|
+
HEREDOC
|
166
|
+
|
167
|
+
i = 0
|
168
|
+
|
169
|
+
builder.Phase(:number => i += 1) do
|
170
|
+
Core::build_node2(builder, Nodes.data_generator2({:name => "now_generator", :id => "now_generator", :generate => "function integer generate() {$out.0.all = \"NOW=\" + toString(date2long(today()));return OK;}"}))
|
171
|
+
Core::build_node2(builder, Nodes.writer2({:name => "NOW Writer", :id => "now_csv_writer", :fileURL => "now.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
|
172
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "now_csv_writer:0", :fromNode => "now_generator:0", :metadata => "trash_metadata"}))
|
173
|
+
end
|
174
|
+
|
175
|
+
taps.each do |tap|
|
176
|
+
source = tap[:source]
|
177
|
+
id = tap[:id]
|
178
|
+
|
179
|
+
builder.Phase(:number => i += 1) do
|
180
|
+
#Core::build_node2(builder, Nodes.file_list2({:id => "#{id}_file_list", :name => "#{id}_file_list", :dataPolicy => "Strict", :baseURL => "#{tap[:validation_source]}", :output_mapping => Nodes::MAP_ALL}))
|
181
|
+
Core::build_node2(builder, Nodes.reader2({:id => "#{id}_file_list", :name => "#{id}_file_list", :fileURL => "${gdc_agent_file}" }))
|
182
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_reformat", :transformation => reformat_func}))
|
183
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_file_list:0", :metadata => "file_list"}))
|
184
|
+
Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_writer", :fileURL => "data/#{id}_files_to_read.csv"}))
|
185
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_writer:0", :fromNode => "#{id}_reformat:0", :metadata => "file_list"}))
|
186
|
+
end
|
187
|
+
|
188
|
+
builder.Phase(:number => i += 1) do
|
189
|
+
ctl = "function integer generate() {$out.0.all = \"#{id}_SKIP_LINES=0\";return OK;}"
|
190
|
+
Core::build_node2(builder, Nodes.data_generator2({:name => "#{id}_generator", :id => "#{id}_generator", :generate => ctl}))
|
191
|
+
Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_csv_writer", :fileURL => "#{id}_counter.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
|
192
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_csv_writer:0", :fromNode => "#{id}_generator:0", :metadata => "trash_metadata"}))
|
193
|
+
end
|
194
|
+
|
195
|
+
subgraph_reformat_func = <<HEREDOC
|
196
|
+
|
197
|
+
function integer transform() {
|
198
|
+
$out.0.all = "graphs/#{id}_loop.grf";
|
199
|
+
return ALL;
|
200
|
+
}
|
201
|
+
HEREDOC
|
202
|
+
|
203
|
+
fail_reformat = <<HEREDOC
|
204
|
+
function integer transform() {
|
205
|
+
raiseError("Loop failed");
|
206
|
+
}
|
207
|
+
HEREDOC
|
208
|
+
|
209
|
+
|
210
|
+
builder.Phase(:number => i += 1) do
|
211
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat graph", :id => "#{id}_reformat_graph", :transformation => subgraph_reformat_func}))
|
212
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_graph:0", :fromNode => "#{id}_reformat:1", :metadata => "file_list"}))
|
213
|
+
Core::build_node2(builder, Nodes.run_graph2({
|
214
|
+
:guiName => id,
|
215
|
+
:name => id,
|
216
|
+
:id => "#{id}_run_graph"
|
217
|
+
}))
|
218
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_run_graph:0", :fromNode => "#{id}_reformat_graph:0", :metadata => "trash_metadata"}))
|
219
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat fail", :id => "#{id}_reformat_fail", :transformation => fail_reformat}))
|
220
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_fail:0", :fromNode => "#{id}_run_graph:1", :metadata => "run_graph_failure_metadata"}))
|
221
|
+
Core::build_node2(builder, Nodes.trash2({:name => "#{id}_trash", :id => "#{id}_trash", :debugPrint => true}))
|
222
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_trash:0", :fromNode => "#{id}_reformat_fail:0", :metadata => "run_graph_failure_metadata"}))
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
# If there is a fail.
|
227
|
+
# Copies the report files
|
228
|
+
taps.each do |tap|
|
229
|
+
id = tap[:id]
|
230
|
+
builder.Phase(:number => i += 1) do
|
231
|
+
Core::build_node2(builder, Nodes.reader2({
|
232
|
+
:id => "#{id}_files_copy_check_reader",
|
233
|
+
:guiName => "#{id}_files failure_check_reader",
|
234
|
+
:name => "#{id}_files_failure_check_reader",
|
235
|
+
:fileURL => "data/has_errors_*.csv",
|
236
|
+
:skipRows => 0,
|
237
|
+
:numRecords => 1
|
238
|
+
}))
|
239
|
+
|
240
|
+
fail_reformat = <<HEREDOC
|
241
|
+
function integer transform() {
|
242
|
+
$out.0.all = "graphs/#{id}_failure.grf";
|
243
|
+
return ALL;
|
244
|
+
}
|
245
|
+
HEREDOC
|
246
|
+
|
247
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "#{id}_files_copy_check_reformat", :id => "#{id}_files_copy_check_reformat", :transformation => fail_reformat}))
|
248
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_files_copy_check_reformat:0", :fromNode => "#{id}_files_copy_check_reader:0", :metadata => "trash_metadata"}))
|
249
|
+
|
250
|
+
Core::build_node2(builder, Nodes.run_graph2(
|
251
|
+
{
|
252
|
+
:guiName => "#{id}_failure_run_graph",
|
253
|
+
:name => "#{id}_failure_run_graph",
|
254
|
+
:id => "#{id}_failure_run_graph"
|
255
|
+
}))
|
256
|
+
|
257
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_failure_run_graph:0", :fromNode => "#{id}_files_copy_check_reformat:0", :metadata => "trash_metadata" }))
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
# If there was a failure
|
262
|
+
# Kill the graph
|
263
|
+
builder.Phase(:number => i += 1) do
|
264
|
+
Core::build_node2(builder, Nodes.reader2({
|
265
|
+
:id => "failure_check_reader",
|
266
|
+
:guiName => "failure_check_reader",
|
267
|
+
:name => "failure_check_reader",
|
268
|
+
:fileURL => "data/has_errors_*.csv",
|
269
|
+
:skipRows => 0
|
270
|
+
}))
|
271
|
+
|
272
|
+
failure_func = <<HEREDOC
|
273
|
+
|
274
|
+
function integer transform() {
|
275
|
+
raiseError("Validation did not pass");
|
276
|
+
}
|
277
|
+
HEREDOC
|
278
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "failure_dedup", :id => "failure_dedup", :transformation => failure_func }))
|
279
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "failure_dedup:0", :fromNode => "failure_check_reader:0", :metadata => "trash_metadata" }))
|
280
|
+
|
281
|
+
Core::build_node2(builder, Nodes.trash2({:name => "failure_check_trash", :id => "failure_check_trash", :debugPrint => true}))
|
282
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "failure_check_trash:0", :fromNode => "failure_dedup:0", :metadata => "trash_metadata" }))
|
283
|
+
end
|
284
|
+
|
285
|
+
# If everything is success.
|
286
|
+
# Copies the files
|
287
|
+
taps.each do |tap|
|
288
|
+
builder.Phase(:number => i += 1) do
|
289
|
+
|
290
|
+
Core::build_node2(builder, Nodes.run_graph2(
|
291
|
+
{
|
292
|
+
:guiName => "success_run_graph",
|
293
|
+
:name => "success_run_graph",
|
294
|
+
:id => "success_run_graph",
|
295
|
+
:graphName => "graphs/#{tap[:id]}_success.grf"
|
296
|
+
}))
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
300
|
+
end
|
301
|
+
end
|
302
|
+
|
303
|
+
def self.generate_contract(file, tap)
|
304
|
+
name = id = tap[:id]
|
305
|
+
|
306
|
+
File.open(file, "w") do |f|
|
307
|
+
|
308
|
+
builder = Builder::XmlMarkup.new(:target=>f, :indent=>2)
|
309
|
+
builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
|
310
|
+
builder.Graph({
|
311
|
+
:name => "Contract Checker"
|
312
|
+
}) do
|
313
|
+
builder.Global do
|
314
|
+
Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
|
315
|
+
Helpers::property_file(builder, {:id => "file_params", :fileURL => "#{id}_item.prm"})
|
316
|
+
|
317
|
+
Helpers::create_trash_meta(builder)
|
318
|
+
Helpers::create_read_error_meta(builder)
|
319
|
+
|
320
|
+
builder.Metadata({:id => "#{name}_clover_metadata"}) do |builder|
|
321
|
+
Helpers::csv_metadata(builder, Metadata.get_source_metadata(tap))
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
builder.Phase(:number => 0) do
|
326
|
+
|
327
|
+
|
328
|
+
reformat_func = <<HEREDOC
|
329
|
+
long line = 0;
|
330
|
+
string fieldName;
|
331
|
+
string validationType;
|
332
|
+
string currentValue;
|
333
|
+
boolean failed;
|
334
|
+
boolean currentLineFailed;
|
335
|
+
|
336
|
+
function integer transform() {
|
337
|
+
line = line + 1;
|
338
|
+
currentLineFailed = false;
|
339
|
+
|
340
|
+
#{tap[:fields].find_all {|d| d.has_key?(:validates_as)}.map do |d|
|
341
|
+
"currentValue = $in.0." + d[:name] + ";" + ctl_validation(d[:name], d[:validates_as])
|
342
|
+
end.join("\n")}
|
343
|
+
if (currentLineFailed) {
|
344
|
+
return 1;
|
345
|
+
} else {
|
346
|
+
return 0;
|
347
|
+
}
|
348
|
+
}
|
349
|
+
|
350
|
+
HEREDOC
|
351
|
+
|
352
|
+
|
353
|
+
has_error_reformat_func = <<HEREDOC
|
354
|
+
function integer transform() {
|
355
|
+
$out.0.all = "${FILE}";
|
356
|
+
return ALL;
|
357
|
+
}
|
358
|
+
|
359
|
+
HEREDOC
|
360
|
+
|
361
|
+
format_clover_error = <<HEREDOC
|
362
|
+
function integer transform() {
|
363
|
+
$out.0.all = "Parsing error occured on line " + $in.0.line_number + " of file " + $in.0.file + " on record " + $in.0.record + " with message " + $in.0.message ;
|
364
|
+
return ALL;
|
365
|
+
}
|
366
|
+
|
367
|
+
HEREDOC
|
368
|
+
|
369
|
+
Core::build_node2(builder, Nodes.reader2({:id => "#{id}_reader", :name => "#{id}_reader", :dataPolicy => "controlled", :fileURL => "${FILE}"}))
|
370
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_reformat", :transformation => reformat_func}))
|
371
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_reader:0", :metadata => "#{id}_clover_metadata"}))
|
372
|
+
|
373
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "#{id} reader error Reformat", :id => "#{id}_reader_error_reformat", :transformation => format_clover_error}))
|
374
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reader_error_reformat:0", :fromNode => "#{id}_reader:1", :metadata => "reader_error_metadata"}))
|
375
|
+
|
376
|
+
Core::build_node2(builder, Nodes.gather2({:name => "#{id} Gather", :id => "#{id}_gather"}))
|
377
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_gather:1", :fromNode => "#{id}_reader_error_reformat:0", :metadata => "trash_metadata"}))
|
378
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_gather:0", :fromNode => "#{id}_reformat:1", :metadata => "trash_metadata"}))
|
379
|
+
|
380
|
+
Core::build_node2(builder, Nodes.trash2({:name => "PARAMS CSV Writer", :id => "#{id}_trash", :debugPrint => false }))
|
381
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_trash:0", :fromNode => "#{id}_reformat:0", :metadata => "trash_metadata"}))
|
382
|
+
|
383
|
+
Core::build_node2(builder, Nodes.copy2({:name => "#{id} copy", :id => "#{id}_copy"}))
|
384
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_copy:0", :fromNode => "#{id}_gather:0", :metadata => "trash_metadata"}))
|
385
|
+
|
386
|
+
Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_writer", :fileURL => "data/validation_output_#{id}.csv", :outputFieldNames => "false", :append => "true", :quotedStrings => "true"}))
|
387
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_writer:0", :fromNode => "#{id}_copy:0", :metadata => "trash_metadata", :append => true}))
|
388
|
+
|
389
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_has_error_reformat", :transformation => has_error_reformat_func}))
|
390
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_has_error_reformat:0", :fromNode => "#{id}_copy:1", :metadata => "trash_metadata"}))
|
391
|
+
|
392
|
+
Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_has_error_writer", :fileURL => "data/has_errors_#{id}.csv", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
|
393
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_has_error_writer:0", :fromNode => "#{id}_has_error_reformat:0", :metadata => "trash_metadata"}))
|
394
|
+
end
|
395
|
+
end
|
396
|
+
end
|
397
|
+
end
|
398
|
+
|
399
|
+
# def self.generate(file)
|
400
|
+
# File.open(file, "w") do |file|
|
401
|
+
#
|
402
|
+
# builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
|
403
|
+
# builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
|
404
|
+
# builder.Graph({
|
405
|
+
# :name => "Contract Checker"
|
406
|
+
# }) do
|
407
|
+
# builder.Global do
|
408
|
+
# Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
|
409
|
+
# end
|
410
|
+
#
|
411
|
+
# builder.Phase(:number => 0) do
|
412
|
+
#
|
413
|
+
# end
|
414
|
+
#
|
415
|
+
# # phase = 0
|
416
|
+
# # subgraphs.each do |subgraph|
|
417
|
+
# # builder.Phase(:number => phase+1) do
|
418
|
+
# # id1 = get_id
|
419
|
+
# # id2 = get_id
|
420
|
+
# # ctl = "function integer generate() {$out.0.all = \"FLOW=#{subgraph[:flow]}\";return OK;}"
|
421
|
+
# # build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
|
422
|
+
# # build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
|
423
|
+
# # build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
|
424
|
+
# # end
|
425
|
+
# # builder.Phase(:number => phase+2) do
|
426
|
+
#
|
427
|
+
# # id1 = get_id
|
428
|
+
# # id2 = get_id
|
429
|
+
# # ctl = "function integer generate() {$out.0.all = \"NAME=#{subgraph[:name]}\";return OK;}"
|
430
|
+
# # build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
|
431
|
+
# # build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
|
432
|
+
# # build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
|
433
|
+
# # end
|
434
|
+
#
|
435
|
+
# # builder.Phase(:number => phase+3) do
|
436
|
+
# # build_node2(builder, GoodData::CloverGenerator::Nodes.run_graph2({:guiName => subgraph[:name], :name => subgraph[:name], :id => subgraph[:flow], :graphName => subgraph[:file]}))
|
437
|
+
# # end
|
438
|
+
# # phase += 4
|
439
|
+
# # end
|
440
|
+
# end
|
441
|
+
# end
|
442
|
+
# end
|
443
|
+
end
|
444
|
+
end
|
445
|
+
end
|
data/lib/graphs/docentize.grf
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
5
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
6
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
-
<Property fileURL="params.
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
8
|
<Property fileURL="workspace.prm" id="GraphParameter0"/>
|
9
9
|
<Dictionary/>
|
10
10
|
</Global>
|
data/lib/graphs/dummy.grf
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
5
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
6
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
-
<Property fileURL="params.
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
8
|
<Property fileURL="workspace.prm" id="GraphParameter1"/>
|
9
9
|
<Dictionary/>
|
10
10
|
</Global>
|
@@ -0,0 +1,47 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<Graph author="fluke" created="Tue Feb 05 15:38:24 PST 2013" guiVersion="3.3.1" id="1360179808937" licenseType="Commercial" modified="Fri Feb 22 12:18:42 PST 2013" modifiedBy="fluke" name="process_name" revision="1.13" showComponentDetails="true">
|
3
|
+
<Global>
|
4
|
+
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
|
+
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
|
+
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
|
+
<Property fileURL="workspace.prm" id="GraphParameter0"/>
|
9
|
+
<Dictionary/>
|
10
|
+
</Global>
|
11
|
+
<Phase number="0">
|
12
|
+
<Node enabled="enabled" fileURL="data/1_in.csv" guiHeight="77" guiName="CSV Reader" guiWidth="128" guiX="124" guiY="169" id="DATA_READER0" quoteCharacter=""" quotedStrings="true" skipRows="1" type="DATA_READER"/>
|
13
|
+
<Node enabled="enabled" fileURL="data/out.csv" guiHeight="89" guiName="CSV Writer" guiWidth="128" guiX="609" guiY="169" id="DATA_WRITER0" outputFieldNames="true" quoteCharacter=""" quotedStrings="true" type="DATA_WRITER"/>
|
14
|
+
<Node enabled="enabled" guiHeight="65" guiName="Reformat" guiWidth="128" guiX="365" guiY="175" id="REFORMAT0" type="REFORMAT">
|
15
|
+
<attr name="transform"><![CDATA[//#CTL2
|
16
|
+
|
17
|
+
// Transforms input record into output record.
|
18
|
+
function integer transform() {
|
19
|
+
$out.0.* = $in.0.*;
|
20
|
+
$out.0.Name = "Docent " + $in.0.Name;
|
21
|
+
|
22
|
+
return OK;
|
23
|
+
}
|
24
|
+
|
25
|
+
// Called during component initialization.
|
26
|
+
// function boolean init() {}
|
27
|
+
|
28
|
+
// Called during each graph run before the transform is executed. May be used to allocate and initialize resources
|
29
|
+
// required by the transform. All resources allocated within this method should be released
|
30
|
+
// by the postExecute() method.
|
31
|
+
// function void preExecute() {}
|
32
|
+
|
33
|
+
// Called only if transform() throws an exception.
|
34
|
+
// function integer transformOnError(string errorMessage, string stackTrace) {}
|
35
|
+
|
36
|
+
// Called during each graph run after the entire transform was executed. Should be used to free any resources
|
37
|
+
// allocated within the preExecute() method.
|
38
|
+
// function void postExecute() {}
|
39
|
+
|
40
|
+
// Called to return a user-defined error message when an error occurs.
|
41
|
+
// function string getMessage() {}
|
42
|
+
]]></attr>
|
43
|
+
</Node>
|
44
|
+
<Edge fromNode="DATA_READER0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge0" inPort="Port 0 (in)" metadata="Metadata0" outPort="Port 0 (output)" toNode="REFORMAT0:0"/>
|
45
|
+
<Edge fromNode="REFORMAT0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge1" inPort="Port 0 (in)" metadata="Metadata1" outPort="Port 0 (out)" toNode="DATA_WRITER0:0"/>
|
46
|
+
</Phase>
|
47
|
+
</Graph>
|
@@ -0,0 +1,46 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<Graph author="fluke" created="Tue Feb 05 15:38:24 PST 2013" guiVersion="3.3.1" id="1360179808937" licenseType="Commercial" modified="Thu Feb 21 23:29:16 PST 2013" modifiedBy="fluke" name="process_name" revision="1.11" showComponentDetails="true">
|
3
|
+
<Global>
|
4
|
+
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
|
+
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
|
+
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
|
+
<Property fileURL="workspace.prm" id="GraphParameter1"/>
|
9
|
+
<Dictionary/>
|
10
|
+
</Global>
|
11
|
+
<Phase number="0">
|
12
|
+
<Node enabled="enabled" fileURL="data/1_in.csv" guiHeight="77" guiName="CSV Reader" guiWidth="128" guiX="124" guiY="169" id="DATA_READER0" quoteCharacter=""" quotedStrings="true" skipRows="1" type="DATA_READER"/>
|
13
|
+
<Node enabled="enabled" fileURL="data/out.csv" guiHeight="77" guiName="CSV Writer" guiWidth="128" guiX="609" guiY="169" id="DATA_WRITER0" outputFieldNames="true" quoteCharacter=""" quotedStrings="true" type="DATA_WRITER"/>
|
14
|
+
<Node enabled="enabled" guiHeight="65" guiName="Reformat" guiWidth="128" guiX="365" guiY="175" id="REFORMAT0" type="REFORMAT">
|
15
|
+
<attr name="transform"><![CDATA[//#CTL2
|
16
|
+
|
17
|
+
// Transforms input record into output record.
|
18
|
+
function integer transform() {
|
19
|
+
$out.0.* = $in.0.*;
|
20
|
+
|
21
|
+
return OK;
|
22
|
+
}
|
23
|
+
|
24
|
+
// Called during component initialization.
|
25
|
+
// function boolean init() {}
|
26
|
+
|
27
|
+
// Called during each graph run before the transform is executed. May be used to allocate and initialize resources
|
28
|
+
// required by the transform. All resources allocated within this method should be released
|
29
|
+
// by the postExecute() method.
|
30
|
+
// function void preExecute() {}
|
31
|
+
|
32
|
+
// Called only if transform() throws an exception.
|
33
|
+
// function integer transformOnError(string errorMessage, string stackTrace) {}
|
34
|
+
|
35
|
+
// Called during each graph run after the entire transform was executed. Should be used to free any resources
|
36
|
+
// allocated within the preExecute() method.
|
37
|
+
// function void postExecute() {}
|
38
|
+
|
39
|
+
// Called to return a user-defined error message when an error occurs.
|
40
|
+
// function string getMessage() {}
|
41
|
+
]]></attr>
|
42
|
+
</Node>
|
43
|
+
<Edge fromNode="DATA_READER0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge0" inPort="Port 0 (in)" metadata="Metadata0" outPort="Port 0 (output)" toNode="REFORMAT0:0"/>
|
44
|
+
<Edge fromNode="REFORMAT0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge1" inPort="Port 0 (in)" metadata="Metadata1" outPort="Port 0 (out)" toNode="DATA_WRITER0:0"/>
|
45
|
+
</Phase>
|
46
|
+
</Graph>
|