remi 0.0.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.bundle/config +2 -0
- data/.gitignore +3 -2
- data/.rspec +2 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +123 -0
- data/LICENSE.txt +21 -0
- data/README.md +94 -3
- data/bin/remi +8 -0
- data/doc/install-rbenv-os_x.md +47 -0
- data/lib/remi.rb +56 -9
- data/lib/remi/cli.rb +56 -0
- data/lib/remi/core/daru.rb +28 -0
- data/lib/remi/core/refinements.rb +21 -0
- data/lib/remi/core/string.rb +8 -0
- data/lib/remi/cucumber.rb +7 -0
- data/lib/remi/cucumber/business_rules.rb +504 -0
- data/lib/remi/cucumber/data_source.rb +63 -0
- data/lib/remi/data_source.rb +13 -0
- data/lib/remi/data_source/csv_file.rb +79 -0
- data/lib/remi/data_source/data_frame.rb +10 -0
- data/lib/remi/data_source/postgres.rb +58 -0
- data/lib/remi/data_source/salesforce.rb +78 -0
- data/lib/remi/data_subject.rb +25 -0
- data/lib/remi/data_target.rb +15 -0
- data/lib/remi/data_target/csv_file.rb +49 -0
- data/lib/remi/data_target/data_frame.rb +14 -0
- data/lib/remi/data_target/salesforce.rb +49 -0
- data/lib/remi/extractor/sftp_file.rb +84 -0
- data/lib/remi/field_symbolizers.rb +17 -0
- data/lib/remi/job.rb +200 -0
- data/lib/remi/lookup/regex_sieve.rb +55 -0
- data/lib/remi/project/features/examples.feature +24 -0
- data/lib/remi/project/features/formulas.feature +64 -0
- data/lib/remi/project/features/sample_job.feature +304 -0
- data/lib/remi/project/features/step_definitions/remi_step.rb +310 -0
- data/lib/remi/project/features/support/env.rb +10 -0
- data/lib/remi/project/features/support/env_app.rb +3 -0
- data/lib/remi/project/features/transforms/date_diff.feature +50 -0
- data/lib/remi/project/features/transforms/parse_date.feature +34 -0
- data/lib/remi/project/features/transforms/prefix.feature +15 -0
- data/lib/remi/project/jobs/all_jobs_shared.rb +25 -0
- data/lib/remi/project/jobs/copy_source_job.rb +12 -0
- data/lib/remi/project/jobs/sample_job.rb +164 -0
- data/lib/remi/project/jobs/transforms/date_diff_job.rb +17 -0
- data/lib/remi/project/jobs/transforms/parse_date_job.rb +18 -0
- data/lib/remi/project/jobs/transforms/prefix_job.rb +16 -0
- data/lib/remi/project/jobs/transforms/transform_jobs.rb +3 -0
- data/lib/remi/settings.rb +39 -0
- data/lib/remi/sf_bulk_helper.rb +265 -0
- data/lib/remi/source_to_target_map.rb +93 -0
- data/lib/remi/transform.rb +137 -0
- data/lib/remi/version.rb +3 -0
- data/remi.gemspec +25 -7
- data/workbooks/sample_workbook.ipynb +56 -0
- data/workbooks/workbook_helper.rb +1 -0
- metadata +234 -17
- data/lib/noodling.rb +0 -163
- data/test/test_NAME.rb +0 -19
data/lib/noodling.rb
DELETED
@@ -1,163 +0,0 @@
|
|
1
|
-
#!/usr/bin/ruby
|
2
|
-
|
3
|
-
=begin
|
4
|
-
* Think about a class structure for rows and fields
|
5
|
-
* Read a csv, and create a set of fields (with user-extensible metadata)
|
6
|
-
* Populate those fields from the csv file and define some sort of
|
7
|
-
serialization method that could write out blocks of data to a dataset
|
8
|
-
(that would hold the metadata in it)
|
9
|
-
* Dataset class needs block methods to loop over rows
|
10
|
-
* With group-by style first.dot last.dot processing and retain functionality
|
11
|
-
(don't do lag, but do have a retain function - perhaps retain is the default
|
12
|
-
for block variables)
|
13
|
-
* Hash lookups
|
14
|
-
* Build and process sub-datasets ? that might be cool!
|
15
|
-
|
16
|
-
|
17
|
-
=end
|
18
|
-
|
19
|
-
|
20
|
-
def datastep(dataset)
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
# I think I need a variable class too
|
25
|
-
# then I could define an assignment operator for that
|
26
|
-
|
27
|
-
class Dataset
|
28
|
-
|
29
|
-
def initialize
|
30
|
-
@variables = {}
|
31
|
-
@row = {} # I do want row to be an array, but use a has for the moment
|
32
|
-
# or maybe not, maybe I just want the data output to file to be array, internally it can be a hash
|
33
|
-
end
|
34
|
-
|
35
|
-
def add_variable(varname,varmeta)
|
36
|
-
|
37
|
-
tmpvarlist = @variables.merge({ varname => varmeta })
|
38
|
-
|
39
|
-
if not tmpvarlist[varname].has_key?(:type)
|
40
|
-
raise ":type not defined for variable #{varname}"
|
41
|
-
end
|
42
|
-
|
43
|
-
@variables = tmpvarlist
|
44
|
-
@row[varname] = nil
|
45
|
-
|
46
|
-
end
|
47
|
-
|
48
|
-
def []=(varname,value)
|
49
|
-
if @row.has_key?(varname)
|
50
|
-
@row[varname] = value
|
51
|
-
else
|
52
|
-
raise "Variable '#{varname}' not defined"
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def [](varname)
|
57
|
-
@row[varname]
|
58
|
-
end
|
59
|
-
|
60
|
-
def output
|
61
|
-
|
62
|
-
printf "|"
|
63
|
-
@row.each do |key,value|
|
64
|
-
printf "#{key}=#{value}|"
|
65
|
-
end
|
66
|
-
printf "\n"
|
67
|
-
|
68
|
-
# puts "#{@row.inspect}"
|
69
|
-
end
|
70
|
-
|
71
|
-
end
|
72
|
-
|
73
|
-
mydata = Dataset.new()
|
74
|
-
|
75
|
-
mydata.add_variable(:n, { :type => "num" })
|
76
|
-
mydata.add_variable(:retailer_id, { :type => "string", :length => 40 })
|
77
|
-
|
78
|
-
for num in 1..3
|
79
|
-
mydata[:n] = num
|
80
|
-
mydata[:retailer_id] = "R#{mydata[:n]}"
|
81
|
-
mydata.output
|
82
|
-
end
|
83
|
-
|
84
|
-
# want to do someting like this
|
85
|
-
# the datastep is what opens and closes the files
|
86
|
-
# It also advances through the datastep
|
87
|
-
# It also does an automatic "output" unless disabled
|
88
|
-
=begin
|
89
|
-
|
90
|
-
datastep mydata do |d1|
|
91
|
-
|
92
|
-
d1.add_variable({:msg => {:type => :string, :length => 20}})
|
93
|
-
|
94
|
-
for num in 1..3
|
95
|
-
d1.var(:msg) = "Hello #{num}"
|
96
|
-
d1.output
|
97
|
-
end
|
98
|
-
|
99
|
-
end
|
100
|
-
|
101
|
-
|
102
|
-
=end
|
103
|
-
|
104
|
-
=begin
|
105
|
-
|
106
|
-
class Dataset
|
107
|
-
|
108
|
-
def initialize(columns)
|
109
|
-
@columns = columns
|
110
|
-
@row = []
|
111
|
-
@position = 0
|
112
|
-
@file = ""
|
113
|
-
end
|
114
|
-
|
115
|
-
def add_column
|
116
|
-
end
|
117
|
-
|
118
|
-
def output
|
119
|
-
end
|
120
|
-
|
121
|
-
columns = {}
|
122
|
-
|
123
|
-
def show_columns
|
124
|
-
|
125
|
-
|
126
|
-
if block_given?
|
127
|
-
@columns.each do |key,val|
|
128
|
-
yield val
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
@columns.values if not block_given?
|
133
|
-
|
134
|
-
end
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
end
|
139
|
-
|
140
|
-
|
141
|
-
mydata = Dataset.new({
|
142
|
-
1 => "distributor_id",
|
143
|
-
2 => "retailer_id",
|
144
|
-
3 => "physical_cases"
|
145
|
-
});
|
146
|
-
|
147
|
-
puts mydata.inspect
|
148
|
-
|
149
|
-
|
150
|
-
mydata.show_columns do |f|
|
151
|
-
puts f
|
152
|
-
end
|
153
|
-
|
154
|
-
x = mydata.show_columns
|
155
|
-
puts "x = #{x.inspect}"
|
156
|
-
|
157
|
-
=begin
|
158
|
-
data mydata;
|
159
|
-
set input_set;
|
160
|
-
|
161
|
-
x = x**2;
|
162
|
-
run;
|
163
|
-
=end
|
data/test/test_NAME.rb
DELETED