remi 0.0.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.bundle/config +2 -0
- data/.gitignore +3 -2
- data/.rspec +2 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +123 -0
- data/LICENSE.txt +21 -0
- data/README.md +94 -3
- data/bin/remi +8 -0
- data/doc/install-rbenv-os_x.md +47 -0
- data/lib/remi.rb +56 -9
- data/lib/remi/cli.rb +56 -0
- data/lib/remi/core/daru.rb +28 -0
- data/lib/remi/core/refinements.rb +21 -0
- data/lib/remi/core/string.rb +8 -0
- data/lib/remi/cucumber.rb +7 -0
- data/lib/remi/cucumber/business_rules.rb +504 -0
- data/lib/remi/cucumber/data_source.rb +63 -0
- data/lib/remi/data_source.rb +13 -0
- data/lib/remi/data_source/csv_file.rb +79 -0
- data/lib/remi/data_source/data_frame.rb +10 -0
- data/lib/remi/data_source/postgres.rb +58 -0
- data/lib/remi/data_source/salesforce.rb +78 -0
- data/lib/remi/data_subject.rb +25 -0
- data/lib/remi/data_target.rb +15 -0
- data/lib/remi/data_target/csv_file.rb +49 -0
- data/lib/remi/data_target/data_frame.rb +14 -0
- data/lib/remi/data_target/salesforce.rb +49 -0
- data/lib/remi/extractor/sftp_file.rb +84 -0
- data/lib/remi/field_symbolizers.rb +17 -0
- data/lib/remi/job.rb +200 -0
- data/lib/remi/lookup/regex_sieve.rb +55 -0
- data/lib/remi/project/features/examples.feature +24 -0
- data/lib/remi/project/features/formulas.feature +64 -0
- data/lib/remi/project/features/sample_job.feature +304 -0
- data/lib/remi/project/features/step_definitions/remi_step.rb +310 -0
- data/lib/remi/project/features/support/env.rb +10 -0
- data/lib/remi/project/features/support/env_app.rb +3 -0
- data/lib/remi/project/features/transforms/date_diff.feature +50 -0
- data/lib/remi/project/features/transforms/parse_date.feature +34 -0
- data/lib/remi/project/features/transforms/prefix.feature +15 -0
- data/lib/remi/project/jobs/all_jobs_shared.rb +25 -0
- data/lib/remi/project/jobs/copy_source_job.rb +12 -0
- data/lib/remi/project/jobs/sample_job.rb +164 -0
- data/lib/remi/project/jobs/transforms/date_diff_job.rb +17 -0
- data/lib/remi/project/jobs/transforms/parse_date_job.rb +18 -0
- data/lib/remi/project/jobs/transforms/prefix_job.rb +16 -0
- data/lib/remi/project/jobs/transforms/transform_jobs.rb +3 -0
- data/lib/remi/settings.rb +39 -0
- data/lib/remi/sf_bulk_helper.rb +265 -0
- data/lib/remi/source_to_target_map.rb +93 -0
- data/lib/remi/transform.rb +137 -0
- data/lib/remi/version.rb +3 -0
- data/remi.gemspec +25 -7
- data/workbooks/sample_workbook.ipynb +56 -0
- data/workbooks/workbook_helper.rb +1 -0
- metadata +234 -17
- data/lib/noodling.rb +0 -163
- data/test/test_NAME.rb +0 -19
data/lib/noodling.rb
DELETED
@@ -1,163 +0,0 @@
|
|
1
|
-
#!/usr/bin/ruby
|
2
|
-
|
3
|
-
=begin
|
4
|
-
* Think about a class structure for rows and fields
|
5
|
-
* Read a csv, and create a set of fields (with user-extensible metadata)
|
6
|
-
* Populate those fields from the csv file and define some sort of
|
7
|
-
serialization method that could write out blocks of data to a dataset
|
8
|
-
(that would hold the metadata in it)
|
9
|
-
* Dataset class needs block methods to loop over rows
|
10
|
-
* With group-by style first.dot last.dot processing and retain functionality
|
11
|
-
(don't do lag, but do have a retain function - perhaps retain is the default
|
12
|
-
for block variables)
|
13
|
-
* Hash lookups
|
14
|
-
* Build and process sub-datasets ? that might be cool!
|
15
|
-
|
16
|
-
|
17
|
-
=end
|
18
|
-
|
19
|
-
|
20
|
-
def datastep(dataset)
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
# I think I need a variable class too
|
25
|
-
# then I could define an assignment operator for that
|
26
|
-
|
27
|
-
class Dataset
|
28
|
-
|
29
|
-
def initialize
|
30
|
-
@variables = {}
|
31
|
-
@row = {} # I do want row to be an array, but use a has for the moment
|
32
|
-
# or maybe not, maybe I just want the data output to file to be array, internally it can be a hash
|
33
|
-
end
|
34
|
-
|
35
|
-
def add_variable(varname,varmeta)
|
36
|
-
|
37
|
-
tmpvarlist = @variables.merge({ varname => varmeta })
|
38
|
-
|
39
|
-
if not tmpvarlist[varname].has_key?(:type)
|
40
|
-
raise ":type not defined for variable #{varname}"
|
41
|
-
end
|
42
|
-
|
43
|
-
@variables = tmpvarlist
|
44
|
-
@row[varname] = nil
|
45
|
-
|
46
|
-
end
|
47
|
-
|
48
|
-
def []=(varname,value)
|
49
|
-
if @row.has_key?(varname)
|
50
|
-
@row[varname] = value
|
51
|
-
else
|
52
|
-
raise "Variable '#{varname}' not defined"
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def [](varname)
|
57
|
-
@row[varname]
|
58
|
-
end
|
59
|
-
|
60
|
-
def output
|
61
|
-
|
62
|
-
printf "|"
|
63
|
-
@row.each do |key,value|
|
64
|
-
printf "#{key}=#{value}|"
|
65
|
-
end
|
66
|
-
printf "\n"
|
67
|
-
|
68
|
-
# puts "#{@row.inspect}"
|
69
|
-
end
|
70
|
-
|
71
|
-
end
|
72
|
-
|
73
|
-
mydata = Dataset.new()
|
74
|
-
|
75
|
-
mydata.add_variable(:n, { :type => "num" })
|
76
|
-
mydata.add_variable(:retailer_id, { :type => "string", :length => 40 })
|
77
|
-
|
78
|
-
for num in 1..3
|
79
|
-
mydata[:n] = num
|
80
|
-
mydata[:retailer_id] = "R#{mydata[:n]}"
|
81
|
-
mydata.output
|
82
|
-
end
|
83
|
-
|
84
|
-
# want to do someting like this
|
85
|
-
# the datastep is what opens and closes the files
|
86
|
-
# It also advances through the datastep
|
87
|
-
# It also does an automatic "output" unless disabled
|
88
|
-
=begin
|
89
|
-
|
90
|
-
datastep mydata do |d1|
|
91
|
-
|
92
|
-
d1.add_variable({:msg => {:type => :string, :length => 20}})
|
93
|
-
|
94
|
-
for num in 1..3
|
95
|
-
d1.var(:msg) = "Hello #{num}"
|
96
|
-
d1.output
|
97
|
-
end
|
98
|
-
|
99
|
-
end
|
100
|
-
|
101
|
-
|
102
|
-
=end
|
103
|
-
|
104
|
-
=begin
|
105
|
-
|
106
|
-
class Dataset
|
107
|
-
|
108
|
-
def initialize(columns)
|
109
|
-
@columns = columns
|
110
|
-
@row = []
|
111
|
-
@position = 0
|
112
|
-
@file = ""
|
113
|
-
end
|
114
|
-
|
115
|
-
def add_column
|
116
|
-
end
|
117
|
-
|
118
|
-
def output
|
119
|
-
end
|
120
|
-
|
121
|
-
columns = {}
|
122
|
-
|
123
|
-
def show_columns
|
124
|
-
|
125
|
-
|
126
|
-
if block_given?
|
127
|
-
@columns.each do |key,val|
|
128
|
-
yield val
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
@columns.values if not block_given?
|
133
|
-
|
134
|
-
end
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
end
|
139
|
-
|
140
|
-
|
141
|
-
mydata = Dataset.new({
|
142
|
-
1 => "distributor_id",
|
143
|
-
2 => "retailer_id",
|
144
|
-
3 => "physical_cases"
|
145
|
-
});
|
146
|
-
|
147
|
-
puts mydata.inspect
|
148
|
-
|
149
|
-
|
150
|
-
mydata.show_columns do |f|
|
151
|
-
puts f
|
152
|
-
end
|
153
|
-
|
154
|
-
x = mydata.show_columns
|
155
|
-
puts "x = #{x.inspect}"
|
156
|
-
|
157
|
-
=begin
|
158
|
-
data mydata;
|
159
|
-
set input_set;
|
160
|
-
|
161
|
-
x = x**2;
|
162
|
-
run;
|
163
|
-
=end
|
data/test/test_NAME.rb
DELETED