gd_bam 0.0.15 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. data/README.md +313 -5
  2. data/bin/bam +126 -48
  3. data/lib/bam/version.rb +1 -1
  4. data/lib/bam.rb +51 -0
  5. data/lib/base/errors.rb +15 -0
  6. data/lib/base/flow.rb +37 -0
  7. data/lib/base/graph.rb +23 -0
  8. data/lib/base/metadata.rb +107 -0
  9. data/lib/base/project.rb +95 -0
  10. data/lib/base/repo.rb +35 -0
  11. data/lib/base/sink.rb +44 -0
  12. data/lib/base/step.rb +47 -0
  13. data/lib/base/tap.rb +167 -0
  14. data/lib/base/taps.rb +19 -0
  15. data/lib/cloud_connect/dsl/cc.rb +42 -0
  16. data/lib/cloud_connect/dsl/es_helpers.rb +49 -0
  17. data/lib/cloud_connect/dsl/helpers.rb +199 -0
  18. data/lib/{nodes → cloud_connect/dsl}/nodes.rb +106 -16
  19. data/lib/cloud_connect/dsl/sf_helpers.rb +39 -0
  20. data/lib/cloud_connect/dsl/structure_helpers.rb +94 -0
  21. data/lib/commands/commands.rb +110 -0
  22. data/lib/commands/deployment.rb +217 -0
  23. data/lib/commands/docs_commands.rb +41 -0
  24. data/lib/commands/gd_commands.rb +95 -0
  25. data/lib/commands/scaffold_commands.rb +103 -0
  26. data/lib/commands/sf_commands.rb +37 -0
  27. data/lib/commands/validators.rb +19 -0
  28. data/lib/compatibility.rb +19 -0
  29. data/lib/compiler/compiler.rb +76 -0
  30. data/lib/compiler/etl_visitor.rb +165 -0
  31. data/lib/dsl/dsl.rb +125 -0
  32. data/lib/generators/downloaders.rb +449 -0
  33. data/lib/generators/etl.rb +261 -0
  34. data/lib/generators/validators.rb +445 -0
  35. data/lib/graphs/docentize.grf +1 -1
  36. data/lib/graphs/dummy.grf +1 -1
  37. data/lib/graphs/goodsales_v2/docentize.grf +47 -0
  38. data/lib/graphs/goodsales_v2/dummy.grf +46 -0
  39. data/lib/graphs/goodsales_v2/load_history.grf +579 -0
  40. data/lib/graphs/goodsales_v2/process_account.grf +47 -0
  41. data/lib/graphs/goodsales_v2/process_activity.grf +222 -0
  42. data/lib/graphs/goodsales_v2/process_activity_dim.grf +88 -0
  43. data/lib/graphs/goodsales_v2/process_activity_owner.grf +48 -0
  44. data/lib/graphs/goodsales_v2/process_forecast.grf +20 -0
  45. data/lib/graphs/goodsales_v2/process_opp_records.grf +84 -0
  46. data/lib/graphs/goodsales_v2/process_opportunity.grf +46 -0
  47. data/lib/graphs/goodsales_v2/process_opportunity_line_item.grf +171 -0
  48. data/lib/graphs/goodsales_v2/process_opportunity_snapshot.grf +94 -0
  49. data/lib/graphs/goodsales_v2/process_owner.grf +48 -0
  50. data/lib/graphs/goodsales_v2/process_stage.grf +51 -0
  51. data/lib/graphs/goodsales_v2/process_stage_history.grf +184 -0
  52. data/lib/graphs/goodsales_v2/process_velocity_duration.grf +140 -0
  53. data/lib/graphs/process_account.grf +1 -1
  54. data/lib/graphs/process_activity.grf +1 -1
  55. data/lib/graphs/process_activity_dim.grf +1 -1
  56. data/lib/graphs/process_activity_owner.grf +1 -1
  57. data/lib/graphs/process_forecast.grf +1 -1
  58. data/lib/graphs/process_opp_records.grf +1 -1
  59. data/lib/graphs/process_opportunity.grf +1 -1
  60. data/lib/graphs/process_opportunity_line_item.grf +1 -1
  61. data/lib/graphs/process_opportunity_snapshot.grf +1 -1
  62. data/lib/graphs/process_owner.grf +1 -1
  63. data/lib/graphs/process_stage.grf +1 -1
  64. data/lib/graphs/process_stage_history.grf +1 -1
  65. data/lib/graphs/process_velocity_duration.grf +1 -1
  66. data/lib/nodes/clover_gen.rb +59 -946
  67. data/lib/nodes/dependency.rb +95 -96
  68. data/lib/runtime.rb +7 -648
  69. data/lib/utils/utils.rb +66 -0
  70. data/templates/flow.rb.erb +7 -6
  71. data/templates/join_template.grf.erb +1 -1
  72. data/templates/reformat_template.grf.erb +1 -1
  73. data/templates/sink.json.erb +28 -0
  74. data/templates/tap.json.erb +3 -5
  75. data/templates/workspace.prm.erb +4 -0
  76. metadata +50 -8
  77. data/lib/contract_checkers/contract_checkers.rb +0 -53
  78. data/lib/dsl/project_dsl.rb +0 -259
  79. data/lib/repo/1_config.json +0 -8
  80. data/templates/dataset.json.erb +0 -13
  81. data/templates/source.json.erb +0 -22
@@ -0,0 +1,140 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <Graph author="gdc-defectivedisplay" created="Sat Mar 16 12:07:29 PDT 2013" guiVersion="3.3.2" id="1363463772522" licenseCode="CLP1DGOODD71636137BY" licenseType="Commercial" modified="Mon Mar 18 14:06:35 PDT 2013" modifiedBy="gdc-defectivedisplay" name="velocity" revision="1.44" showComponentDetails="true">
3
+ <Global>
4
+ <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
+ <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata2"/>
6
+ <Metadata id="Metadata1" previewAttachmentCharset="ISO-8859-1">
7
+ <Record fieldDelimiter="," name="recordName2" previewAttachmentCharset="ISO-8859-1" recordDelimiter="\n" type="delimited">
8
+ <Field containerType="list" name="dates" nullable="false" type="string"/>
9
+ <Field name="Id" type="string"/>
10
+ <Field name="size" type="number"/>
11
+ <Field containerType="list" name="stages" nullable="false" type="string"/>
12
+ <Field containerType="list" name="sortOrders" nullable="false" type="string"/>
13
+ </Record>
14
+ </Metadata>
15
+ <Property fileURL="params.prm" id="GraphParameter0"/>
16
+ <Property fileURL="workspace.prm" id="GraphParameter2"/>
17
+ <Dictionary/>
18
+ </Global>
19
+ <Phase number="0">
20
+ <Node enabled="enabled" fileURL="${PROJECT}/data/1_in.csv" guiHeight="89" guiName="CSV Reader" guiWidth="128" guiX="13" guiY="119" id="DATA_READER0" quotedStrings="true" skipRows="1" type="DATA_READER"/>
21
+ <Node enabled="enabled" fileURL="${PROJECT}/data/out.csv" guiHeight="89" guiName="CSV Writer" guiWidth="128" guiX="649" guiY="119" id="DATA_WRITER0" outputFieldNames="true" quotedStrings="true" type="DATA_WRITER"/>
22
+ <Node enabled="enabled" guiHeight="77" guiName="ExtSort" guiWidth="128" guiX="211" guiY="119" id="EXT_SORT0" sortKey="Id(a);Snapshot(a)" type="EXT_SORT"/>
23
+ <Node enabled="enabled" groupAccumulatorMetadataId="Metadata1" groupKeyFields="Id" guiHeight="65" guiName="Rollup" guiWidth="128" guiX="390" guiY="119" id="ROLLUP0" inputSorted="true" type="ROLLUP">
24
+ <attr name="transform"><![CDATA[//#CTL2
25
+
26
+ //string date_template = "joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ";
27
+ string date_template = "joda:yyyy-MM-dd";
28
+
29
+ // Called for the first data record in a new group. Starts the parsing of the new group.
30
+ function void initGroup(recordName2 groupAccumulator) {
31
+ groupAccumulator.Id = $0.Id;
32
+ groupAccumulator.size = 0;
33
+ }
34
+
35
+ // Called for each data record in the group (including the first one and the last one).
36
+ // Implicitly returns false => updateTransform() is not called. When returns true, calls updateTransform().
37
+ function boolean updateGroup(recordName2 groupAccumulator) {
38
+ append(groupAccumulator.dates, $0.Snapshot);
39
+ append(groupAccumulator.stages, $0.StageName);
40
+ groupAccumulator.size++;
41
+ return false;
42
+ }
43
+
44
+ // Called for the last data records in all groups sequentially, but only after all incoming data records have been parsed.
45
+ // Implicitly returns true => transform() is called for the whole group.
46
+ function boolean finishGroup(recordName2 groupAccumulator) {
47
+ return true;
48
+ }
49
+
50
+ // Called to transform data records that have been parsed so far into user-specified number of output data record(s).
51
+ // Counter (incremented by 1 starting from 0) stores the number of previous calls to this method for the current group update.
52
+ // Group accumulator can optionally be used.
53
+ // Function implicitly returns SKIP to skip sending any data records to output.
54
+ // Returning ALL causes each data record to be sent to all output port(s).
55
+ // Can also return a number of the output port to which individual data record should be sent.
56
+ function integer updateTransform(integer counter, VoidMetadata groupAccumulator) {
57
+ raiseError("Function not implemented!");
58
+ }
59
+
60
+ // Called to transform the whole group of incoming data record(s) into user-specified number of output data record(s).
61
+ // Counter (incremented by 1 starting from 0) stores the number of previous calls to this method for the current group update.
62
+ // Group accumulator can optionally be used.
63
+ // Function implicitly returns SKIP to skip sending any data records to output.
64
+ // Returning ALL causes each data record to be sent to all output port(s).
65
+ // Can also return a number of the output port to which individual data record should be sent.
66
+ function integer transform(integer counter, recordName2 groupAccumulator) {
67
+
68
+ if (counter >= length(groupAccumulator.dates)) {
69
+ return SKIP;
70
+ }
71
+
72
+ string stage = groupAccumulator.stages[counter];
73
+ date date0 = str2date(groupAccumulator.dates[0], date_template);
74
+ date date1 = str2date(groupAccumulator.dates[counter], date_template);
75
+ string velocity;
76
+ string duration;
77
+ date date2;
78
+
79
+ if(counter == (length(groupAccumulator.dates) - 1)) {
80
+ velocity = num2str(dateDiff(date1, date0, day));
81
+ duration = "0";
82
+ } else {
83
+ date2 = str2date(groupAccumulator.dates[counter + 1], date_template);
84
+ duration = num2str(dateDiff(date2, date1, day));
85
+ velocity = num2str(dateDiff(date1, date0, day));
86
+ }
87
+
88
+
89
+ //printErr([groupAccumulator.Id, stage, duration, velocity]);
90
+ $out.0.* = $in.0.*;
91
+ $out.0.Id = groupAccumulator.Id;
92
+ $out.0.StageName = stage;
93
+ $out.0.Duration = duration;
94
+ $out.0.Velocity = velocity;
95
+
96
+ return ALL;
97
+
98
+ }
99
+
100
+ // Called during component initialization.
101
+ // function void init() {}
102
+
103
+ // Called during each graph run before the transform is executed. May be used to allocate and initialize resources
104
+ // required by the transform. All resources allocated within this method should be released
105
+ // by the postExecute() method.
106
+ // function void preExecute() {}
107
+
108
+ // Called only if initGroup(DataRecord) throws an exception.
109
+ //function void initGroupOnError(string errorMessage, string stackTrace, VoidMetadata groupAccumulator) {
110
+ //}
111
+
112
+ // Called only if updateGroup(DataRecord) throws an exception.
113
+ //function boolean updateGroupOnError(string errorMessage, string stackTrace, VoidMetadata groupAccumulator) {
114
+ //}
115
+
116
+ // Called only if finishGroup(DataRecord) throws an exception.
117
+ //function boolean finishGroupOnError(string errorMessage, string stackTrace, VoidMetadata groupAccumulator) {
118
+ //}
119
+
120
+ // Called only if updateTransform(integer, DataRecord) throws an exception.
121
+ //function integer updateTransformOnError(string errorMessage, string stackTrace, integer counter, VoidMetadata groupAccumulator) {
122
+ //}
123
+
124
+ // Called only if transform(integer, DataRecord) throws an exception.
125
+ //function integer transformOnError(string errorMessage, string stackTrace, integer counter, VoidMetadata groupAccumulator) {
126
+ //}
127
+
128
+ // Called during each graph run after the entire transform was executed. Should be used to free any resources
129
+ // allocated within the preExecute() method.
130
+ // function void postExecute() {}
131
+
132
+ // Called to return a user-defined error message when an error occurs.
133
+ // function string getMessage() {}
134
+ ]]></attr>
135
+ </Node>
136
+ <Edge fromNode="DATA_READER0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge0" inPort="Port 0 (in)" metadata="Metadata0" outPort="Port 0 (output)" toNode="EXT_SORT0:0"/>
137
+ <Edge fromNode="EXT_SORT0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge2" inPort="Port 0 (in)" metadata="Metadata0" outPort="Port 0 (out)" toNode="ROLLUP0:0"/>
138
+ <Edge fromNode="ROLLUP0:0" guiBendpoints="" guiRouter="Manhattan" id="Edge1" inPort="Port 0 (in)" metadata="Metadata2" outPort="Port 0 (out)" toNode="DATA_WRITER0:0"/>
139
+ </Phase>
140
+ </Graph>
@@ -4,7 +4,7 @@
4
4
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
5
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
6
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
- <Property fileURL="params.txt" id="GraphParameter0"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
8
  <Property fileURL="workspace.prm" id="GraphParameter1"/>
9
9
  <Dictionary/>
10
10
  </Global>
@@ -8,7 +8,7 @@
8
8
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/3_in.xml" id="Metadata5"/>
9
9
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/3_out.xml" id="Metadata6"/>
10
10
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
11
- <Property fileURL="params.txt" id="GraphParameter0"/>
11
+ <Property fileURL="params.prm" id="GraphParameter0"/>
12
12
  <Property fileURL="workspace.prm" id="GraphParameter2"/>
13
13
  <Dictionary/>
14
14
  </Global>
@@ -6,7 +6,7 @@
6
6
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_in.xml" id="Metadata3"/>
7
7
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_out.xml" id="Metadata4"/>
8
8
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
9
- <Property fileURL="params.txt" id="GraphParameter0"/>
9
+ <Property fileURL="params.prm" id="GraphParameter0"/>
10
10
  <Property fileURL="workspace.prm" id="GraphParameter1"/>
11
11
  <Dictionary/>
12
12
  </Global>
@@ -4,7 +4,7 @@
4
4
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
5
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
6
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
- <Property fileURL="params.txt" id="GraphParameter0"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
8
  <Property fileURL="workspace.prm" id="GraphParameter1"/>
9
9
  <Dictionary/>
10
10
  </Global>
@@ -4,7 +4,7 @@
4
4
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
5
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
6
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
- <Property fileURL="params.txt" id="GraphParameter0"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
8
  <Property fileURL="workspace.prm" id="GraphParameter2"/>
9
9
  <Dictionary/>
10
10
  </Global>
@@ -8,7 +8,7 @@
8
8
  <Field name="field1" type="string"/>
9
9
  </Record>
10
10
  </Metadata>
11
- <Property fileURL="params.txt" id="GraphParameter0"/>
11
+ <Property fileURL="params.prm" id="GraphParameter0"/>
12
12
  <Property fileURL="workspace.prm" id="GraphParameter2"/>
13
13
  <Dictionary/>
14
14
  </Global>
@@ -4,7 +4,7 @@
4
4
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
5
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
6
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
- <Property fileURL="params.txt" id="GraphParameter0"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
8
  <Property fileURL="workspace.prm" id="GraphParameter1"/>
9
9
  <Dictionary/>
10
10
  </Global>
@@ -9,7 +9,7 @@
9
9
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/3_out.xml" id="Metadata5"/>
10
10
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/4_in.xml" id="Metadata6"/>
11
11
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/4_out.xml" id="Metadata7"/>
12
- <Property fileURL="params.txt" id="GraphParameter0"/>
12
+ <Property fileURL="params.prm" id="GraphParameter0"/>
13
13
  <Property fileURL="workspace.prm" id="GraphParameter1"/>
14
14
  <Dictionary/>
15
15
  </Global>
@@ -6,7 +6,7 @@
6
6
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_in.xml" id="Metadata2"/>
7
7
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_out.xml" id="Metadata4"/>
8
8
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
9
- <Property fileURL="params.txt" id="GraphParameter0"/>
9
+ <Property fileURL="params.prm" id="GraphParameter0"/>
10
10
  <Property fileURL="workspace.prm" id="GraphParameter2"/>
11
11
  <LookupTable charset="ISO-8859-1" id="LookupTable0" initialSize="512" key="MasterLabel" metadata="Metadata4" name="StageLookup" type="simpleLookup"/>
12
12
  <Dictionary/>
@@ -4,7 +4,7 @@
4
4
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
5
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
6
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
- <Property fileURL="params.txt" id="GraphParameter0"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
8
  <Property fileURL="workspace.prm" id="GraphParameter1"/>
9
9
  <Dictionary/>
10
10
  </Global>
@@ -4,7 +4,7 @@
4
4
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
5
5
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
6
6
  <MetadataGroup id="ComponentGroup0" name="metadata"/>
7
- <Property fileURL="params.txt" id="GraphParameter0"/>
7
+ <Property fileURL="params.prm" id="GraphParameter0"/>
8
8
  <Property fileURL="workspace.prm" id="GraphParameter2"/>
9
9
  <Dictionary/>
10
10
  </Global>
@@ -9,7 +9,7 @@
9
9
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/3_out.xml" id="Metadata5"/>
10
10
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/4_in.xml" id="Metadata6"/>
11
11
  <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/4_out.xml" id="Metadata7"/>
12
- <Property fileURL="params.txt" id="GraphParameter0"/>
12
+ <Property fileURL="params.prm" id="GraphParameter0"/>
13
13
  <Property fileURL="workspace.prm" id="GraphParameter2"/>
14
14
  <Dictionary/>
15
15
  </Global>
@@ -12,7 +12,7 @@
12
12
  <Field containerType="list" name="sortOrders" nullable="false" type="string"/>
13
13
  </Record>
14
14
  </Metadata>
15
- <Property fileURL="params.txt" id="GraphParameter0"/>
15
+ <Property fileURL="params.prm" id="GraphParameter0"/>
16
16
  <Property fileURL="workspace.prm" id="GraphParameter2"/>
17
17
  <Dictionary/>
18
18
  </Global>