gd_bam 0.0.15 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +313 -5
- data/bin/bam +126 -48
- data/lib/bam/version.rb +1 -1
- data/lib/bam.rb +51 -0
- data/lib/base/errors.rb +15 -0
- data/lib/base/flow.rb +37 -0
- data/lib/base/graph.rb +23 -0
- data/lib/base/metadata.rb +107 -0
- data/lib/base/project.rb +95 -0
- data/lib/base/repo.rb +35 -0
- data/lib/base/sink.rb +44 -0
- data/lib/base/step.rb +47 -0
- data/lib/base/tap.rb +167 -0
- data/lib/base/taps.rb +19 -0
- data/lib/cloud_connect/dsl/cc.rb +42 -0
- data/lib/cloud_connect/dsl/es_helpers.rb +49 -0
- data/lib/cloud_connect/dsl/helpers.rb +199 -0
- data/lib/{nodes → cloud_connect/dsl}/nodes.rb +106 -16
- data/lib/cloud_connect/dsl/sf_helpers.rb +39 -0
- data/lib/cloud_connect/dsl/structure_helpers.rb +94 -0
- data/lib/commands/commands.rb +110 -0
- data/lib/commands/deployment.rb +217 -0
- data/lib/commands/docs_commands.rb +41 -0
- data/lib/commands/gd_commands.rb +95 -0
- data/lib/commands/scaffold_commands.rb +103 -0
- data/lib/commands/sf_commands.rb +37 -0
- data/lib/commands/validators.rb +19 -0
- data/lib/compatibility.rb +19 -0
- data/lib/compiler/compiler.rb +76 -0
- data/lib/compiler/etl_visitor.rb +165 -0
- data/lib/dsl/dsl.rb +125 -0
- data/lib/generators/downloaders.rb +449 -0
- data/lib/generators/etl.rb +261 -0
- data/lib/generators/validators.rb +445 -0
- data/lib/graphs/docentize.grf +1 -1
- data/lib/graphs/dummy.grf +1 -1
- data/lib/graphs/goodsales_v2/docentize.grf +47 -0
- data/lib/graphs/goodsales_v2/dummy.grf +46 -0
- data/lib/graphs/goodsales_v2/load_history.grf +579 -0
- data/lib/graphs/goodsales_v2/process_account.grf +47 -0
- data/lib/graphs/goodsales_v2/process_activity.grf +222 -0
- data/lib/graphs/goodsales_v2/process_activity_dim.grf +88 -0
- data/lib/graphs/goodsales_v2/process_activity_owner.grf +48 -0
- data/lib/graphs/goodsales_v2/process_forecast.grf +20 -0
- data/lib/graphs/goodsales_v2/process_opp_records.grf +84 -0
- data/lib/graphs/goodsales_v2/process_opportunity.grf +46 -0
- data/lib/graphs/goodsales_v2/process_opportunity_line_item.grf +171 -0
- data/lib/graphs/goodsales_v2/process_opportunity_snapshot.grf +94 -0
- data/lib/graphs/goodsales_v2/process_owner.grf +48 -0
- data/lib/graphs/goodsales_v2/process_stage.grf +51 -0
- data/lib/graphs/goodsales_v2/process_stage_history.grf +184 -0
- data/lib/graphs/goodsales_v2/process_velocity_duration.grf +140 -0
- data/lib/graphs/process_account.grf +1 -1
- data/lib/graphs/process_activity.grf +1 -1
- data/lib/graphs/process_activity_dim.grf +1 -1
- data/lib/graphs/process_activity_owner.grf +1 -1
- data/lib/graphs/process_forecast.grf +1 -1
- data/lib/graphs/process_opp_records.grf +1 -1
- data/lib/graphs/process_opportunity.grf +1 -1
- data/lib/graphs/process_opportunity_line_item.grf +1 -1
- data/lib/graphs/process_opportunity_snapshot.grf +1 -1
- data/lib/graphs/process_owner.grf +1 -1
- data/lib/graphs/process_stage.grf +1 -1
- data/lib/graphs/process_stage_history.grf +1 -1
- data/lib/graphs/process_velocity_duration.grf +1 -1
- data/lib/nodes/clover_gen.rb +59 -946
- data/lib/nodes/dependency.rb +95 -96
- data/lib/runtime.rb +7 -648
- data/lib/utils/utils.rb +66 -0
- data/templates/flow.rb.erb +7 -6
- data/templates/join_template.grf.erb +1 -1
- data/templates/reformat_template.grf.erb +1 -1
- data/templates/sink.json.erb +28 -0
- data/templates/tap.json.erb +3 -5
- data/templates/workspace.prm.erb +4 -0
- metadata +50 -8
- data/lib/contract_checkers/contract_checkers.rb +0 -53
- data/lib/dsl/project_dsl.rb +0 -259
- data/lib/repo/1_config.json +0 -8
- data/templates/dataset.json.erb +0 -13
- data/templates/source.json.erb +0 -22
@@ -0,0 +1,140 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<Graph author="gdc-defectivedisplay" created="Sat Mar 16 12:07:29 PDT 2013" guiVersion="3.3.2" id="1363463772522" licenseCode="CLP1DGOODD71636137BY" licenseType="Commercial" modified="Mon Mar 18 14:06:35 PDT 2013" modifiedBy="gdc-defectivedisplay" name="velocity" revision="1.44" showComponentDetails="true">
|
3
|
+
<Global>
|
4
|
+
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
|
+
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata2"/>
|
6
|
+
<Metadata id="Metadata1" previewAttachmentCharset="ISO-8859-1">
|
7
|
+
<Record fieldDelimiter="," name="recordName2" previewAttachmentCharset="ISO-8859-1" recordDelimiter="\n" type="delimited">
|
8
|
+
<Field containerType="list" name="dates" nullable="false" type="string"/>
|
9
|
+
<Field name="Id" type="string"/>
|
10
|
+
<Field name="size" type="number"/>
|
11
|
+
<Field containerType="list" name="stages" nullable="false" type="string"/>
|
12
|
+
<Field containerType="list" name="sortOrders" nullable="false" type="string"/>
|
13
|
+
</Record>
|
14
|
+
</Metadata>
|
15
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
16
|
+
<Property fileURL="workspace.prm" id="GraphParameter2"/>
|
17
|
+
<Dictionary/>
|
18
|
+
</Global>
|
19
|
+
<Phase number="0">
|
20
|
+
<Node enabled="enabled" fileURL="${PROJECT}/data/1_in.csv" guiHeight="89" guiName="CSV Reader" guiWidth="128" guiX="13" guiY="119" id="DATA_READER0" quotedStrings="true" skipRows="1" type="DATA_READER"/>
|
21
|
+
<Node enabled="enabled" fileURL="${PROJECT}/data/out.csv" guiHeight="89" guiName="CSV Writer" guiWidth="128" guiX="649" guiY="119" id="DATA_WRITER0" outputFieldNames="true" quotedStrings="true" type="DATA_WRITER"/>
|
22
|
+
<Node enabled="enabled" guiHeight="77" guiName="ExtSort" guiWidth="128" guiX="211" guiY="119" id="EXT_SORT0" sortKey="Id(a);Snapshot(a)" type="EXT_SORT"/>
|
23
|
+
<Node enabled="enabled" groupAccumulatorMetadataId="Metadata1" groupKeyFields="Id" guiHeight="65" guiName="Rollup" guiWidth="128" guiX="390" guiY="119" id="ROLLUP0" inputSorted="true" type="ROLLUP">
|
24
|
+
<attr name="transform"><![CDATA[//#CTL2
|
25
|
+
|
26
|
+
//string date_template = "joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ";
|
27
|
+
string date_template = "joda:yyyy-MM-dd";
|
28
|
+
|
29
|
+
// Called for the first data record in a new group. Starts the parsing of the new group.
|
30
|
+
function void initGroup(recordName2 groupAccumulator) {
|
31
|
+
groupAccumulator.Id = $0.Id;
|
32
|
+
groupAccumulator.size = 0;
|
33
|
+
}
|
34
|
+
|
35
|
+
// Called for each data record in the group (including the first one and the last one).
|
36
|
+
// Implicitly returns false => updateTransform() is not called. When returns true, calls updateTransform().
|
37
|
+
function boolean updateGroup(recordName2 groupAccumulator) {
|
38
|
+
append(groupAccumulator.dates, $0.Snapshot);
|
39
|
+
append(groupAccumulator.stages, $0.StageName);
|
40
|
+
groupAccumulator.size++;
|
41
|
+
return false;
|
42
|
+
}
|
43
|
+
|
44
|
+
// Called for the last data records in all groups sequentially, but only after all incoming data records have been parsed.
|
45
|
+
// Implicitly returns true => transform() is called for the whole group.
|
46
|
+
function boolean finishGroup(recordName2 groupAccumulator) {
|
47
|
+
return true;
|
48
|
+
}
|
49
|
+
|
50
|
+
// Called to transform data records that have been parsed so far into user-specified number of output data record(s).
|
51
|
+
// Counter (incremented by 1 starting from 0) stores the number of previous calls to this method for the current group update.
|
52
|
+
// Group accumulator can optionally be used.
|
53
|
+
// Function implicitly returns SKIP to skip sending any data records to output.
|
54
|
+
// Returning ALL causes each data record to be sent to all output port(s).
|
55
|
+
// Can also return a number of the output port to which individual data record should be sent.
|
56
|
+
function integer updateTransform(integer counter, VoidMetadata groupAccumulator) {
|
57
|
+
raiseError("Function not implemented!");
|
58
|
+
}
|
59
|
+
|
60
|
+
// Called to transform the whole group of incoming data record(s) into user-specified number of output data record(s).
|
61
|
+
// Counter (incremented by 1 starting from 0) stores the number of previous calls to this method for the current group update.
|
62
|
+
// Group accumulator can optionally be used.
|
63
|
+
// Function implicitly returns SKIP to skip sending any data records to output.
|
64
|
+
// Returning ALL causes each data record to be sent to all output port(s).
|
65
|
+
// Can also return a number of the output port to which individual data record should be sent.
|
66
|
+
function integer transform(integer counter, recordName2 groupAccumulator) {
|
67
|
+
|
68
|
+
if (counter >= length(groupAccumulator.dates)) {
|
69
|
+
return SKIP;
|
70
|
+
}
|
71
|
+
|
72
|
+
string stage = groupAccumulator.stages[counter];
|
73
|
+
date date0 = str2date(groupAccumulator.dates[0], date_template);
|
74
|
+
date date1 = str2date(groupAccumulator.dates[counter], date_template);
|
75
|
+
string velocity;
|
76
|
+
string duration;
|
77
|
+
date date2;
|
78
|
+
|
79
|
+
if(counter == (length(groupAccumulator.dates) - 1)) {
|
80
|
+
velocity = num2str(dateDiff(date1, date0, day));
|
81
|
+
duration = "0";
|
82
|
+
} else {
|
83
|
+
date2 = str2date(groupAccumulator.dates[counter + 1], date_template);
|
84
|
+
duration = num2str(dateDiff(date2, date1, day));
|
85
|
+
velocity = num2str(dateDiff(date1, date0, day));
|
86
|
+
}
|
87
|
+
|
88
|
+
|
89
|
+
//printErr([groupAccumulator.Id, stage, duration, velocity]);
|
90
|
+
$out.0.* = $in.0.*;
|
91
|
+
$out.0.Id = groupAccumulator.Id;
|
92
|
+
$out.0.StageName = stage;
|
93
|
+
$out.0.Duration = duration;
|
94
|
+
$out.0.Velocity = velocity;
|
95
|
+
|
96
|
+
return ALL;
|
97
|
+
|
98
|
+
}
|
99
|
+
|
100
|
+
// Called during component initialization.
|
101
|
+
// function void init() {}
|
102
|
+
|
103
|
+
// Called during each graph run before the transform is executed. May be used to allocate and initialize resources
|
104
|
+
// required by the transform. All resources allocated within this method should be released
|
105
|
+
// by the postExecute() method.
|
106
|
+
// function void preExecute() {}
|
107
|
+
|
108
|
+
// Called only if initGroup(DataRecord) throws an exception.
|
109
|
+
//function void initGroupOnError(string errorMessage, string stackTrace, VoidMetadata groupAccumulator) {
|
110
|
+
//}
|
111
|
+
|
112
|
+
// Called only if updateGroup(DataRecord) throws an exception.
|
113
|
+
//function boolean updateGroupOnError(string errorMessage, string stackTrace, VoidMetadata groupAccumulator) {
|
114
|
+
//}
|
115
|
+
|
116
|
+
// Called only if finishGroup(DataRecord) throws an exception.
|
117
|
+
//function boolean finishGroupOnError(string errorMessage, string stackTrace, VoidMetadata groupAccumulator) {
|
118
|
+
//}
|
119
|
+
|
120
|
+
// Called only if updateTransform(integer, DataRecord) throws an exception.
|
121
|
+
//function integer updateTransformOnError(string errorMessage, string stackTrace, integer counter, VoidMetadata groupAccumulator) {
|
122
|
+
//}
|
123
|
+
|
124
|
+
// Called only if transform(integer, DataRecord) throws an exception.
|
125
|
+
//function integer transformOnError(string errorMessage, string stackTrace, integer counter, VoidMetadata groupAccumulator) {
|
126
|
+
//}
|
127
|
+
|
128
|
+
// Called during each graph run after the entire transform was executed. Should be used to free any resources
|
129
|
+
// allocated within the preExecute() method.
|
130
|
+
// function void postExecute() {}
|
131
|
+
|
132
|
+
// Called to return a user-defined error message when an error occurs.
|
133
|
+
// function string getMessage() {}
|
134
|
+
]]></attr>
|
135
|
+
</Node>
|
136
|
+
<Edge fromNode="DATA_READER0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge0" inPort="Port 0 (in)" metadata="Metadata0" outPort="Port 0 (output)" toNode="EXT_SORT0:0"/>
|
137
|
+
<Edge fromNode="EXT_SORT0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge2" inPort="Port 0 (in)" metadata="Metadata0" outPort="Port 0 (out)" toNode="ROLLUP0:0"/>
|
138
|
+
<Edge fromNode="ROLLUP0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge1" inPort="Port 0 (in)" metadata="Metadata2" outPort="Port 0 (out)" toNode="DATA_WRITER0:0"/>
|
139
|
+
</Phase>
|
140
|
+
</Graph>
|
@@ -4,7 +4,7 @@
|
|
4
4
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
5
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
6
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
-
<Property fileURL="params.
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
8
|
<Property fileURL="workspace.prm" id="GraphParameter1"/>
|
9
9
|
<Dictionary/>
|
10
10
|
</Global>
|
@@ -8,7 +8,7 @@
|
|
8
8
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/3_in.xml" id="Metadata5"/>
|
9
9
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/3_out.xml" id="Metadata6"/>
|
10
10
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
11
|
-
<Property fileURL="params.
|
11
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
12
12
|
<Property fileURL="workspace.prm" id="GraphParameter2"/>
|
13
13
|
<Dictionary/>
|
14
14
|
</Global>
|
@@ -6,7 +6,7 @@
|
|
6
6
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_in.xml" id="Metadata3"/>
|
7
7
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_out.xml" id="Metadata4"/>
|
8
8
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
9
|
-
<Property fileURL="params.
|
9
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
10
10
|
<Property fileURL="workspace.prm" id="GraphParameter1"/>
|
11
11
|
<Dictionary/>
|
12
12
|
</Global>
|
@@ -4,7 +4,7 @@
|
|
4
4
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
5
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
6
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
-
<Property fileURL="params.
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
8
|
<Property fileURL="workspace.prm" id="GraphParameter1"/>
|
9
9
|
<Dictionary/>
|
10
10
|
</Global>
|
@@ -4,7 +4,7 @@
|
|
4
4
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
5
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
6
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
-
<Property fileURL="params.
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
8
|
<Property fileURL="workspace.prm" id="GraphParameter2"/>
|
9
9
|
<Dictionary/>
|
10
10
|
</Global>
|
@@ -8,7 +8,7 @@
|
|
8
8
|
<Field name="field1" type="string"/>
|
9
9
|
</Record>
|
10
10
|
</Metadata>
|
11
|
-
<Property fileURL="params.
|
11
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
12
12
|
<Property fileURL="workspace.prm" id="GraphParameter2"/>
|
13
13
|
<Dictionary/>
|
14
14
|
</Global>
|
@@ -4,7 +4,7 @@
|
|
4
4
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
5
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
6
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
-
<Property fileURL="params.
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
8
|
<Property fileURL="workspace.prm" id="GraphParameter1"/>
|
9
9
|
<Dictionary/>
|
10
10
|
</Global>
|
@@ -9,7 +9,7 @@
|
|
9
9
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/3_out.xml" id="Metadata5"/>
|
10
10
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/4_in.xml" id="Metadata6"/>
|
11
11
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/4_out.xml" id="Metadata7"/>
|
12
|
-
<Property fileURL="params.
|
12
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
13
13
|
<Property fileURL="workspace.prm" id="GraphParameter1"/>
|
14
14
|
<Dictionary/>
|
15
15
|
</Global>
|
@@ -6,7 +6,7 @@
|
|
6
6
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_in.xml" id="Metadata2"/>
|
7
7
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_out.xml" id="Metadata4"/>
|
8
8
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
9
|
-
<Property fileURL="params.
|
9
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
10
10
|
<Property fileURL="workspace.prm" id="GraphParameter2"/>
|
11
11
|
<LookupTable charset="ISO-8859-1" id="LookupTable0" initialSize="512" key="MasterLabel" metadata="Metadata4" name="StageLookup" type="simpleLookup"/>
|
12
12
|
<Dictionary/>
|
@@ -4,7 +4,7 @@
|
|
4
4
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
5
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
6
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
-
<Property fileURL="params.
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
8
|
<Property fileURL="workspace.prm" id="GraphParameter1"/>
|
9
9
|
<Dictionary/>
|
10
10
|
</Global>
|
@@ -4,7 +4,7 @@
|
|
4
4
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
|
5
5
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
|
6
6
|
<MetadataGroup id="ComponentGroup0" name="metadata"/>
|
7
|
-
<Property fileURL="params.
|
7
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
8
8
|
<Property fileURL="workspace.prm" id="GraphParameter2"/>
|
9
9
|
<Dictionary/>
|
10
10
|
</Global>
|
@@ -9,7 +9,7 @@
|
|
9
9
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/3_out.xml" id="Metadata5"/>
|
10
10
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/4_in.xml" id="Metadata6"/>
|
11
11
|
<Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/4_out.xml" id="Metadata7"/>
|
12
|
-
<Property fileURL="params.
|
12
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
13
13
|
<Property fileURL="workspace.prm" id="GraphParameter2"/>
|
14
14
|
<Dictionary/>
|
15
15
|
</Global>
|
@@ -12,7 +12,7 @@
|
|
12
12
|
<Field containerType="list" name="sortOrders" nullable="false" type="string"/>
|
13
13
|
</Record>
|
14
14
|
</Metadata>
|
15
|
-
<Property fileURL="params.
|
15
|
+
<Property fileURL="params.prm" id="GraphParameter0"/>
|
16
16
|
<Property fileURL="workspace.prm" id="GraphParameter2"/>
|
17
17
|
<Dictionary/>
|
18
18
|
</Global>
|