remi 0.0.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.bundle/config +2 -0
  3. data/.gitignore +3 -2
  4. data/.rspec +2 -0
  5. data/.ruby-version +1 -0
  6. data/Gemfile +4 -0
  7. data/Gemfile.lock +123 -0
  8. data/LICENSE.txt +21 -0
  9. data/README.md +94 -3
  10. data/bin/remi +8 -0
  11. data/doc/install-rbenv-os_x.md +47 -0
  12. data/lib/remi.rb +56 -9
  13. data/lib/remi/cli.rb +56 -0
  14. data/lib/remi/core/daru.rb +28 -0
  15. data/lib/remi/core/refinements.rb +21 -0
  16. data/lib/remi/core/string.rb +8 -0
  17. data/lib/remi/cucumber.rb +7 -0
  18. data/lib/remi/cucumber/business_rules.rb +504 -0
  19. data/lib/remi/cucumber/data_source.rb +63 -0
  20. data/lib/remi/data_source.rb +13 -0
  21. data/lib/remi/data_source/csv_file.rb +79 -0
  22. data/lib/remi/data_source/data_frame.rb +10 -0
  23. data/lib/remi/data_source/postgres.rb +58 -0
  24. data/lib/remi/data_source/salesforce.rb +78 -0
  25. data/lib/remi/data_subject.rb +25 -0
  26. data/lib/remi/data_target.rb +15 -0
  27. data/lib/remi/data_target/csv_file.rb +49 -0
  28. data/lib/remi/data_target/data_frame.rb +14 -0
  29. data/lib/remi/data_target/salesforce.rb +49 -0
  30. data/lib/remi/extractor/sftp_file.rb +84 -0
  31. data/lib/remi/field_symbolizers.rb +17 -0
  32. data/lib/remi/job.rb +200 -0
  33. data/lib/remi/lookup/regex_sieve.rb +55 -0
  34. data/lib/remi/project/features/examples.feature +24 -0
  35. data/lib/remi/project/features/formulas.feature +64 -0
  36. data/lib/remi/project/features/sample_job.feature +304 -0
  37. data/lib/remi/project/features/step_definitions/remi_step.rb +310 -0
  38. data/lib/remi/project/features/support/env.rb +10 -0
  39. data/lib/remi/project/features/support/env_app.rb +3 -0
  40. data/lib/remi/project/features/transforms/date_diff.feature +50 -0
  41. data/lib/remi/project/features/transforms/parse_date.feature +34 -0
  42. data/lib/remi/project/features/transforms/prefix.feature +15 -0
  43. data/lib/remi/project/jobs/all_jobs_shared.rb +25 -0
  44. data/lib/remi/project/jobs/copy_source_job.rb +12 -0
  45. data/lib/remi/project/jobs/sample_job.rb +164 -0
  46. data/lib/remi/project/jobs/transforms/date_diff_job.rb +17 -0
  47. data/lib/remi/project/jobs/transforms/parse_date_job.rb +18 -0
  48. data/lib/remi/project/jobs/transforms/prefix_job.rb +16 -0
  49. data/lib/remi/project/jobs/transforms/transform_jobs.rb +3 -0
  50. data/lib/remi/settings.rb +39 -0
  51. data/lib/remi/sf_bulk_helper.rb +265 -0
  52. data/lib/remi/source_to_target_map.rb +93 -0
  53. data/lib/remi/transform.rb +137 -0
  54. data/lib/remi/version.rb +3 -0
  55. data/remi.gemspec +25 -7
  56. data/workbooks/sample_workbook.ipynb +56 -0
  57. data/workbooks/workbook_helper.rb +1 -0
  58. metadata +234 -17
  59. data/lib/noodling.rb +0 -163
  60. data/test/test_NAME.rb +0 -19
data/lib/noodling.rb DELETED
@@ -1,163 +0,0 @@
1
- #!/usr/bin/ruby
2
-
3
- =begin
4
- * Think about a class structure for rows and fields
5
- * Read a csv, and create a set of fields (with user-extensible metadata)
6
- * Populate those fields from the csv file and define some sort of
7
- serialization method that could write out blocks of data to a dataset
8
- (that would hold the metadata in it)
9
- * Dataset class needs block methods to loop over rows
10
- * With group-by style first.dot last.dot processing and retain functionality
11
- (don't do lag, but do have a retain function - perhaps retain is the default
12
- for block variables)
13
- * Hash lookups
14
- * Build and process sub-datasets ? that might be cool!
15
-
16
-
17
- =end
18
-
19
-
20
- def datastep(dataset)
21
-
22
- end
23
-
24
- # I think I need a variable class too
25
- # then I could define an assignment operator for that
26
-
27
- class Dataset
28
-
29
- def initialize
30
- @variables = {}
31
- @row = {} # I do want row to be an array, but use a has for the moment
32
- # or maybe not, maybe I just want the data output to file to be array, internally it can be a hash
33
- end
34
-
35
- def add_variable(varname,varmeta)
36
-
37
- tmpvarlist = @variables.merge({ varname => varmeta })
38
-
39
- if not tmpvarlist[varname].has_key?(:type)
40
- raise ":type not defined for variable #{varname}"
41
- end
42
-
43
- @variables = tmpvarlist
44
- @row[varname] = nil
45
-
46
- end
47
-
48
- def []=(varname,value)
49
- if @row.has_key?(varname)
50
- @row[varname] = value
51
- else
52
- raise "Variable '#{varname}' not defined"
53
- end
54
- end
55
-
56
- def [](varname)
57
- @row[varname]
58
- end
59
-
60
- def output
61
-
62
- printf "|"
63
- @row.each do |key,value|
64
- printf "#{key}=#{value}|"
65
- end
66
- printf "\n"
67
-
68
- # puts "#{@row.inspect}"
69
- end
70
-
71
- end
72
-
73
- mydata = Dataset.new()
74
-
75
- mydata.add_variable(:n, { :type => "num" })
76
- mydata.add_variable(:retailer_id, { :type => "string", :length => 40 })
77
-
78
- for num in 1..3
79
- mydata[:n] = num
80
- mydata[:retailer_id] = "R#{mydata[:n]}"
81
- mydata.output
82
- end
83
-
84
- # want to do someting like this
85
- # the datastep is what opens and closes the files
86
- # It also advances through the datastep
87
- # It also does an automatic "output" unless disabled
88
- =begin
89
-
90
- datastep mydata do |d1|
91
-
92
- d1.add_variable({:msg => {:type => :string, :length => 20}})
93
-
94
- for num in 1..3
95
- d1.var(:msg) = "Hello #{num}"
96
- d1.output
97
- end
98
-
99
- end
100
-
101
-
102
- =end
103
-
104
- =begin
105
-
106
- class Dataset
107
-
108
- def initialize(columns)
109
- @columns = columns
110
- @row = []
111
- @position = 0
112
- @file = ""
113
- end
114
-
115
- def add_column
116
- end
117
-
118
- def output
119
- end
120
-
121
- columns = {}
122
-
123
- def show_columns
124
-
125
-
126
- if block_given?
127
- @columns.each do |key,val|
128
- yield val
129
- end
130
- end
131
-
132
- @columns.values if not block_given?
133
-
134
- end
135
-
136
-
137
-
138
- end
139
-
140
-
141
- mydata = Dataset.new({
142
- 1 => "distributor_id",
143
- 2 => "retailer_id",
144
- 3 => "physical_cases"
145
- });
146
-
147
- puts mydata.inspect
148
-
149
-
150
- mydata.show_columns do |f|
151
- puts f
152
- end
153
-
154
- x = mydata.show_columns
155
- puts "x = #{x.inspect}"
156
-
157
- =begin
158
- data mydata;
159
- set input_set;
160
-
161
- x = x**2;
162
- run;
163
- =end
data/test/test_NAME.rb DELETED
@@ -1,19 +0,0 @@
1
- #!/usr/bin/ruby
2
-
3
- require 'test/unit'
4
-
5
- class MyUnitTests < Test::Unit::TestCase
6
-
7
- def setup
8
- puts "setup!"
9
- end
10
-
11
- def teardown
12
- puts "teardown!"
13
- end
14
-
15
- def test_basic
16
- puts "I RAN!"
17
- end
18
-
19
- end