remi 0.0.1 → 0.2.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.bundle/config +2 -0
  3. data/.gitignore +3 -2
  4. data/.rspec +2 -0
  5. data/.ruby-version +1 -0
  6. data/Gemfile +4 -0
  7. data/Gemfile.lock +123 -0
  8. data/LICENSE.txt +21 -0
  9. data/README.md +94 -3
  10. data/bin/remi +8 -0
  11. data/doc/install-rbenv-os_x.md +47 -0
  12. data/lib/remi.rb +56 -9
  13. data/lib/remi/cli.rb +56 -0
  14. data/lib/remi/core/daru.rb +28 -0
  15. data/lib/remi/core/refinements.rb +21 -0
  16. data/lib/remi/core/string.rb +8 -0
  17. data/lib/remi/cucumber.rb +7 -0
  18. data/lib/remi/cucumber/business_rules.rb +504 -0
  19. data/lib/remi/cucumber/data_source.rb +63 -0
  20. data/lib/remi/data_source.rb +13 -0
  21. data/lib/remi/data_source/csv_file.rb +79 -0
  22. data/lib/remi/data_source/data_frame.rb +10 -0
  23. data/lib/remi/data_source/postgres.rb +58 -0
  24. data/lib/remi/data_source/salesforce.rb +78 -0
  25. data/lib/remi/data_subject.rb +25 -0
  26. data/lib/remi/data_target.rb +15 -0
  27. data/lib/remi/data_target/csv_file.rb +49 -0
  28. data/lib/remi/data_target/data_frame.rb +14 -0
  29. data/lib/remi/data_target/salesforce.rb +49 -0
  30. data/lib/remi/extractor/sftp_file.rb +84 -0
  31. data/lib/remi/field_symbolizers.rb +17 -0
  32. data/lib/remi/job.rb +200 -0
  33. data/lib/remi/lookup/regex_sieve.rb +55 -0
  34. data/lib/remi/project/features/examples.feature +24 -0
  35. data/lib/remi/project/features/formulas.feature +64 -0
  36. data/lib/remi/project/features/sample_job.feature +304 -0
  37. data/lib/remi/project/features/step_definitions/remi_step.rb +310 -0
  38. data/lib/remi/project/features/support/env.rb +10 -0
  39. data/lib/remi/project/features/support/env_app.rb +3 -0
  40. data/lib/remi/project/features/transforms/date_diff.feature +50 -0
  41. data/lib/remi/project/features/transforms/parse_date.feature +34 -0
  42. data/lib/remi/project/features/transforms/prefix.feature +15 -0
  43. data/lib/remi/project/jobs/all_jobs_shared.rb +25 -0
  44. data/lib/remi/project/jobs/copy_source_job.rb +12 -0
  45. data/lib/remi/project/jobs/sample_job.rb +164 -0
  46. data/lib/remi/project/jobs/transforms/date_diff_job.rb +17 -0
  47. data/lib/remi/project/jobs/transforms/parse_date_job.rb +18 -0
  48. data/lib/remi/project/jobs/transforms/prefix_job.rb +16 -0
  49. data/lib/remi/project/jobs/transforms/transform_jobs.rb +3 -0
  50. data/lib/remi/settings.rb +39 -0
  51. data/lib/remi/sf_bulk_helper.rb +265 -0
  52. data/lib/remi/source_to_target_map.rb +93 -0
  53. data/lib/remi/transform.rb +137 -0
  54. data/lib/remi/version.rb +3 -0
  55. data/remi.gemspec +25 -7
  56. data/workbooks/sample_workbook.ipynb +56 -0
  57. data/workbooks/workbook_helper.rb +1 -0
  58. metadata +234 -17
  59. data/lib/noodling.rb +0 -163
  60. data/test/test_NAME.rb +0 -19
data/lib/noodling.rb DELETED
@@ -1,163 +0,0 @@
1
- #!/usr/bin/ruby
2
-
3
- =begin
4
- * Think about a class structure for rows and fields
5
- * Read a csv, and create a set of fields (with user-extensible metadata)
6
- * Populate those fields from the csv file and define some sort of
7
- serialization method that could write out blocks of data to a dataset
8
- (that would hold the metadata in it)
9
- * Dataset class needs block methods to loop over rows
10
- * With group-by style first.dot last.dot processing and retain functionality
11
- (don't do lag, but do have a retain function - perhaps retain is the default
12
- for block variables)
13
- * Hash lookups
14
- * Build and process sub-datasets ? that might be cool!
15
-
16
-
17
- =end
18
-
19
-
20
- def datastep(dataset)
21
-
22
- end
23
-
24
- # I think I need a variable class too
25
- # then I could define an assignment operator for that
26
-
27
- class Dataset
28
-
29
- def initialize
30
- @variables = {}
31
- @row = {} # I do want row to be an array, but use a has for the moment
32
- # or maybe not, maybe I just want the data output to file to be array, internally it can be a hash
33
- end
34
-
35
- def add_variable(varname,varmeta)
36
-
37
- tmpvarlist = @variables.merge({ varname => varmeta })
38
-
39
- if not tmpvarlist[varname].has_key?(:type)
40
- raise ":type not defined for variable #{varname}"
41
- end
42
-
43
- @variables = tmpvarlist
44
- @row[varname] = nil
45
-
46
- end
47
-
48
- def []=(varname,value)
49
- if @row.has_key?(varname)
50
- @row[varname] = value
51
- else
52
- raise "Variable '#{varname}' not defined"
53
- end
54
- end
55
-
56
- def [](varname)
57
- @row[varname]
58
- end
59
-
60
- def output
61
-
62
- printf "|"
63
- @row.each do |key,value|
64
- printf "#{key}=#{value}|"
65
- end
66
- printf "\n"
67
-
68
- # puts "#{@row.inspect}"
69
- end
70
-
71
- end
72
-
73
- mydata = Dataset.new()
74
-
75
- mydata.add_variable(:n, { :type => "num" })
76
- mydata.add_variable(:retailer_id, { :type => "string", :length => 40 })
77
-
78
- for num in 1..3
79
- mydata[:n] = num
80
- mydata[:retailer_id] = "R#{mydata[:n]}"
81
- mydata.output
82
- end
83
-
84
- # want to do someting like this
85
- # the datastep is what opens and closes the files
86
- # It also advances through the datastep
87
- # It also does an automatic "output" unless disabled
88
- =begin
89
-
90
- datastep mydata do |d1|
91
-
92
- d1.add_variable({:msg => {:type => :string, :length => 20}})
93
-
94
- for num in 1..3
95
- d1.var(:msg) = "Hello #{num}"
96
- d1.output
97
- end
98
-
99
- end
100
-
101
-
102
- =end
103
-
104
- =begin
105
-
106
- class Dataset
107
-
108
- def initialize(columns)
109
- @columns = columns
110
- @row = []
111
- @position = 0
112
- @file = ""
113
- end
114
-
115
- def add_column
116
- end
117
-
118
- def output
119
- end
120
-
121
- columns = {}
122
-
123
- def show_columns
124
-
125
-
126
- if block_given?
127
- @columns.each do |key,val|
128
- yield val
129
- end
130
- end
131
-
132
- @columns.values if not block_given?
133
-
134
- end
135
-
136
-
137
-
138
- end
139
-
140
-
141
- mydata = Dataset.new({
142
- 1 => "distributor_id",
143
- 2 => "retailer_id",
144
- 3 => "physical_cases"
145
- });
146
-
147
- puts mydata.inspect
148
-
149
-
150
- mydata.show_columns do |f|
151
- puts f
152
- end
153
-
154
- x = mydata.show_columns
155
- puts "x = #{x.inspect}"
156
-
157
- =begin
158
- data mydata;
159
- set input_set;
160
-
161
- x = x**2;
162
- run;
163
- =end
data/test/test_NAME.rb DELETED
@@ -1,19 +0,0 @@
1
- #!/usr/bin/ruby
2
-
3
- require 'test/unit'
4
-
5
- class MyUnitTests < Test::Unit::TestCase
6
-
7
- def setup
8
- puts "setup!"
9
- end
10
-
11
- def teardown
12
- puts "teardown!"
13
- end
14
-
15
- def test_basic
16
- puts "I RAN!"
17
- end
18
-
19
- end