sycsvpro 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +21 -0
- data/.rspec +1 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +51 -0
- data/LICENSE +20 -0
- data/README.md +188 -0
- data/README.rdoc +44 -0
- data/Rakefile +44 -0
- data/bin/sycsvpro +208 -0
- data/features/step_definitions/sycsvpro_steps.rb +6 -0
- data/features/support/env.rb +15 -0
- data/features/sycsvpro.feature +8 -0
- data/html/Dsl.html +201 -0
- data/html/Object.html +116 -0
- data/html/README_rdoc.html +178 -0
- data/html/Sycsvpro/Analyzer.html +239 -0
- data/html/Sycsvpro/Calculator.html +354 -0
- data/html/Sycsvpro/Collector.html +281 -0
- data/html/Sycsvpro/ColumnFilter.html +165 -0
- data/html/Sycsvpro/Counter.html +397 -0
- data/html/Sycsvpro/Extractor.html +269 -0
- data/html/Sycsvpro/Filter.html +349 -0
- data/html/Sycsvpro/Header.html +228 -0
- data/html/Sycsvpro/Mapper.html +288 -0
- data/html/Sycsvpro/Profiler.html +234 -0
- data/html/Sycsvpro/RowFilter.html +162 -0
- data/html/Sycsvpro.html +141 -0
- data/html/created.rid +17 -0
- data/html/fonts/Lato-Light.ttf +0 -0
- data/html/fonts/Lato-LightItalic.ttf +0 -0
- data/html/fonts/Lato-Regular.ttf +0 -0
- data/html/fonts/Lato-RegularItalic.ttf +0 -0
- data/html/fonts/SourceCodePro-Bold.ttf +0 -0
- data/html/fonts/SourceCodePro-Regular.ttf +0 -0
- data/html/fonts.css +167 -0
- data/html/images/add.png +0 -0
- data/html/images/arrow_up.png +0 -0
- data/html/images/brick.png +0 -0
- data/html/images/brick_link.png +0 -0
- data/html/images/bug.png +0 -0
- data/html/images/bullet_black.png +0 -0
- data/html/images/bullet_toggle_minus.png +0 -0
- data/html/images/bullet_toggle_plus.png +0 -0
- data/html/images/date.png +0 -0
- data/html/images/delete.png +0 -0
- data/html/images/find.png +0 -0
- data/html/images/loadingAnimation.gif +0 -0
- data/html/images/macFFBgHack.png +0 -0
- data/html/images/package.png +0 -0
- data/html/images/page_green.png +0 -0
- data/html/images/page_white_text.png +0 -0
- data/html/images/page_white_width.png +0 -0
- data/html/images/plugin.png +0 -0
- data/html/images/ruby.png +0 -0
- data/html/images/tag_blue.png +0 -0
- data/html/images/tag_green.png +0 -0
- data/html/images/transparent.png +0 -0
- data/html/images/wrench.png +0 -0
- data/html/images/wrench_orange.png +0 -0
- data/html/images/zoom.png +0 -0
- data/html/index.html +202 -0
- data/html/js/darkfish.js +140 -0
- data/html/js/jquery.js +18 -0
- data/html/js/navigation.js +142 -0
- data/html/js/search.js +109 -0
- data/html/js/search_index.js +1 -0
- data/html/js/searcher.js +228 -0
- data/html/rdoc.css +580 -0
- data/html/table_of_contents.html +236 -0
- data/lib/sycsvpro/analyzer.rb +40 -0
- data/lib/sycsvpro/calculator.rb +94 -0
- data/lib/sycsvpro/collector.rb +60 -0
- data/lib/sycsvpro/column_filter.rb +23 -0
- data/lib/sycsvpro/counter.rb +74 -0
- data/lib/sycsvpro/dsl.rb +37 -0
- data/lib/sycsvpro/extractor.rb +39 -0
- data/lib/sycsvpro/filter.rb +98 -0
- data/lib/sycsvpro/header.rb +29 -0
- data/lib/sycsvpro/mapper.rb +53 -0
- data/lib/sycsvpro/profiler.rb +26 -0
- data/lib/sycsvpro/row_filter.rb +20 -0
- data/lib/sycsvpro/version.rb +5 -0
- data/lib/sycsvpro.rb +9 -0
- data/spec/sycsvpro/analyze_spec.rb +23 -0
- data/spec/sycsvpro/calculator_spec.rb +45 -0
- data/spec/sycsvpro/collector_spec.rb +27 -0
- data/spec/sycsvpro/counter_spec.rb +51 -0
- data/spec/sycsvpro/extractor_spec.rb +27 -0
- data/spec/sycsvpro/files/mappings +6 -0
- data/spec/sycsvpro/files/profile.rb +42 -0
- data/spec/sycsvpro/mapper_spec.rb +33 -0
- data/spec/sycsvpro/profiler_spec.rb +32 -0
- data/sycsvpro.gemspec +24 -0
- data/sycsvpro.rdoc +29 -0
- metadata +215 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
|
|
3
|
+
<html>
|
|
4
|
+
<head>
|
|
5
|
+
<meta charset="UTF-8">
|
|
6
|
+
|
|
7
|
+
<title>Table of Contents - Your application title</title>
|
|
8
|
+
|
|
9
|
+
<link href="./fonts.css" rel="stylesheet">
|
|
10
|
+
<link href="./rdoc.css" rel="stylesheet">
|
|
11
|
+
|
|
12
|
+
<script type="text/javascript">
|
|
13
|
+
var rdoc_rel_prefix = "./";
|
|
14
|
+
</script>
|
|
15
|
+
|
|
16
|
+
<script src="./js/jquery.js"></script>
|
|
17
|
+
<script src="./js/navigation.js"></script>
|
|
18
|
+
<script src="./js/search_index.js"></script>
|
|
19
|
+
<script src="./js/search.js"></script>
|
|
20
|
+
<script src="./js/searcher.js"></script>
|
|
21
|
+
<script src="./js/darkfish.js"></script>
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
<body id="top" class="table-of-contents">
|
|
25
|
+
<main role="main">
|
|
26
|
+
<h1 class="class">Table of Contents - Your application title</h1>
|
|
27
|
+
|
|
28
|
+
<h2 id="pages">Pages</h2>
|
|
29
|
+
<ul>
|
|
30
|
+
<li class="file">
|
|
31
|
+
<a href="README_rdoc.html">README</a>
|
|
32
|
+
|
|
33
|
+
<ul>
|
|
34
|
+
<li><a href="README_rdoc.html#label-sycsvpro">sycsvpro</a>
|
|
35
|
+
<li><a href="README_rdoc.html#label-Install">Install</a>
|
|
36
|
+
<li><a href="README_rdoc.html#label-Use">Use</a>
|
|
37
|
+
<li><a href="README_rdoc.html#label-Developing+for+%60sycsvpro%60">Developing for `sycsvpro`</a>
|
|
38
|
+
<li><a href="README_rdoc.html#label-sycsvpro">sycsvpro</a>
|
|
39
|
+
</ul>
|
|
40
|
+
</li>
|
|
41
|
+
|
|
42
|
+
</ul>
|
|
43
|
+
|
|
44
|
+
<h2 id="classes">Classes and Modules</h2>
|
|
45
|
+
<ul>
|
|
46
|
+
<li class="module">
|
|
47
|
+
<a href="Dsl.html">Dsl</a>
|
|
48
|
+
</li>
|
|
49
|
+
<li class="class">
|
|
50
|
+
<a href="Object.html">Object</a>
|
|
51
|
+
</li>
|
|
52
|
+
<li class="module">
|
|
53
|
+
<a href="Sycsvpro.html">Sycsvpro</a>
|
|
54
|
+
</li>
|
|
55
|
+
<li class="class">
|
|
56
|
+
<a href="Sycsvpro/Analyzer.html">Sycsvpro::Analyzer</a>
|
|
57
|
+
</li>
|
|
58
|
+
<li class="class">
|
|
59
|
+
<a href="Sycsvpro/Calculator.html">Sycsvpro::Calculator</a>
|
|
60
|
+
</li>
|
|
61
|
+
<li class="class">
|
|
62
|
+
<a href="Sycsvpro/Collector.html">Sycsvpro::Collector</a>
|
|
63
|
+
</li>
|
|
64
|
+
<li class="class">
|
|
65
|
+
<a href="Sycsvpro/ColumnFilter.html">Sycsvpro::ColumnFilter</a>
|
|
66
|
+
</li>
|
|
67
|
+
<li class="class">
|
|
68
|
+
<a href="Sycsvpro/Counter.html">Sycsvpro::Counter</a>
|
|
69
|
+
</li>
|
|
70
|
+
<li class="class">
|
|
71
|
+
<a href="Sycsvpro/Extractor.html">Sycsvpro::Extractor</a>
|
|
72
|
+
</li>
|
|
73
|
+
<li class="class">
|
|
74
|
+
<a href="Sycsvpro/Filter.html">Sycsvpro::Filter</a>
|
|
75
|
+
</li>
|
|
76
|
+
<li class="class">
|
|
77
|
+
<a href="Sycsvpro/Header.html">Sycsvpro::Header</a>
|
|
78
|
+
</li>
|
|
79
|
+
<li class="class">
|
|
80
|
+
<a href="Sycsvpro/Mapper.html">Sycsvpro::Mapper</a>
|
|
81
|
+
</li>
|
|
82
|
+
<li class="class">
|
|
83
|
+
<a href="Sycsvpro/Profiler.html">Sycsvpro::Profiler</a>
|
|
84
|
+
</li>
|
|
85
|
+
<li class="class">
|
|
86
|
+
<a href="Sycsvpro/RowFilter.html">Sycsvpro::RowFilter</a>
|
|
87
|
+
</li>
|
|
88
|
+
</ul>
|
|
89
|
+
|
|
90
|
+
<h2 id="methods">Methods</h2>
|
|
91
|
+
<ul>
|
|
92
|
+
|
|
93
|
+
<li class="method">
|
|
94
|
+
<a href="Sycsvpro/Analyzer.html#method-c-new">::new</a>
|
|
95
|
+
—
|
|
96
|
+
<span class="container">Sycsvpro::Analyzer</span>
|
|
97
|
+
|
|
98
|
+
<li class="method">
|
|
99
|
+
<a href="Sycsvpro/Extractor.html#method-c-new">::new</a>
|
|
100
|
+
—
|
|
101
|
+
<span class="container">Sycsvpro::Extractor</span>
|
|
102
|
+
|
|
103
|
+
<li class="method">
|
|
104
|
+
<a href="Sycsvpro/Calculator.html#method-c-new">::new</a>
|
|
105
|
+
—
|
|
106
|
+
<span class="container">Sycsvpro::Calculator</span>
|
|
107
|
+
|
|
108
|
+
<li class="method">
|
|
109
|
+
<a href="Sycsvpro/Profiler.html#method-c-new">::new</a>
|
|
110
|
+
—
|
|
111
|
+
<span class="container">Sycsvpro::Profiler</span>
|
|
112
|
+
|
|
113
|
+
<li class="method">
|
|
114
|
+
<a href="Sycsvpro/Header.html#method-c-new">::new</a>
|
|
115
|
+
—
|
|
116
|
+
<span class="container">Sycsvpro::Header</span>
|
|
117
|
+
|
|
118
|
+
<li class="method">
|
|
119
|
+
<a href="Sycsvpro/Collector.html#method-c-new">::new</a>
|
|
120
|
+
—
|
|
121
|
+
<span class="container">Sycsvpro::Collector</span>
|
|
122
|
+
|
|
123
|
+
<li class="method">
|
|
124
|
+
<a href="Sycsvpro/Mapper.html#method-c-new">::new</a>
|
|
125
|
+
—
|
|
126
|
+
<span class="container">Sycsvpro::Mapper</span>
|
|
127
|
+
|
|
128
|
+
<li class="method">
|
|
129
|
+
<a href="Sycsvpro/Counter.html#method-c-new">::new</a>
|
|
130
|
+
—
|
|
131
|
+
<span class="container">Sycsvpro::Counter</span>
|
|
132
|
+
|
|
133
|
+
<li class="method">
|
|
134
|
+
<a href="Sycsvpro/Filter.html#method-c-new">::new</a>
|
|
135
|
+
—
|
|
136
|
+
<span class="container">Sycsvpro::Filter</span>
|
|
137
|
+
|
|
138
|
+
<li class="method">
|
|
139
|
+
<a href="Sycsvpro/Counter.html#method-i-execute">#execute</a>
|
|
140
|
+
—
|
|
141
|
+
<span class="container">Sycsvpro::Counter</span>
|
|
142
|
+
|
|
143
|
+
<li class="method">
|
|
144
|
+
<a href="Sycsvpro/Collector.html#method-i-execute">#execute</a>
|
|
145
|
+
—
|
|
146
|
+
<span class="container">Sycsvpro::Collector</span>
|
|
147
|
+
|
|
148
|
+
<li class="method">
|
|
149
|
+
<a href="Sycsvpro/Calculator.html#method-i-execute">#execute</a>
|
|
150
|
+
—
|
|
151
|
+
<span class="container">Sycsvpro::Calculator</span>
|
|
152
|
+
|
|
153
|
+
<li class="method">
|
|
154
|
+
<a href="Sycsvpro/Profiler.html#method-i-execute">#execute</a>
|
|
155
|
+
—
|
|
156
|
+
<span class="container">Sycsvpro::Profiler</span>
|
|
157
|
+
|
|
158
|
+
<li class="method">
|
|
159
|
+
<a href="Sycsvpro/Mapper.html#method-i-execute">#execute</a>
|
|
160
|
+
—
|
|
161
|
+
<span class="container">Sycsvpro::Mapper</span>
|
|
162
|
+
|
|
163
|
+
<li class="method">
|
|
164
|
+
<a href="Sycsvpro/Extractor.html#method-i-execute">#execute</a>
|
|
165
|
+
—
|
|
166
|
+
<span class="container">Sycsvpro::Extractor</span>
|
|
167
|
+
|
|
168
|
+
<li class="method">
|
|
169
|
+
<a href="Sycsvpro/Filter.html#method-i-method_missing">#method_missing</a>
|
|
170
|
+
—
|
|
171
|
+
<span class="container">Sycsvpro::Filter</span>
|
|
172
|
+
|
|
173
|
+
<li class="method">
|
|
174
|
+
<a href="Sycsvpro/Calculator.html#method-i-method_missing">#method_missing</a>
|
|
175
|
+
—
|
|
176
|
+
<span class="container">Sycsvpro::Calculator</span>
|
|
177
|
+
|
|
178
|
+
<li class="method">
|
|
179
|
+
<a href="Sycsvpro/Filter.html#method-i-pivot_each_column">#pivot_each_column</a>
|
|
180
|
+
—
|
|
181
|
+
<span class="container">Sycsvpro::Filter</span>
|
|
182
|
+
|
|
183
|
+
<li class="method">
|
|
184
|
+
<a href="Sycsvpro/Header.html#method-i-process">#process</a>
|
|
185
|
+
—
|
|
186
|
+
<span class="container">Sycsvpro::Header</span>
|
|
187
|
+
|
|
188
|
+
<li class="method">
|
|
189
|
+
<a href="Sycsvpro/Filter.html#method-i-process">#process</a>
|
|
190
|
+
—
|
|
191
|
+
<span class="container">Sycsvpro::Filter</span>
|
|
192
|
+
|
|
193
|
+
<li class="method">
|
|
194
|
+
<a href="Sycsvpro/RowFilter.html#method-i-process">#process</a>
|
|
195
|
+
—
|
|
196
|
+
<span class="container">Sycsvpro::RowFilter</span>
|
|
197
|
+
|
|
198
|
+
<li class="method">
|
|
199
|
+
<a href="Sycsvpro/ColumnFilter.html#method-i-process">#process</a>
|
|
200
|
+
—
|
|
201
|
+
<span class="container">Sycsvpro::ColumnFilter</span>
|
|
202
|
+
|
|
203
|
+
<li class="method">
|
|
204
|
+
<a href="Sycsvpro/Counter.html#method-i-process_file">#process_file</a>
|
|
205
|
+
—
|
|
206
|
+
<span class="container">Sycsvpro::Counter</span>
|
|
207
|
+
|
|
208
|
+
<li class="method">
|
|
209
|
+
<a href="Sycsvpro/Analyzer.html#method-i-result">#result</a>
|
|
210
|
+
—
|
|
211
|
+
<span class="container">Sycsvpro::Analyzer</span>
|
|
212
|
+
|
|
213
|
+
<li class="method">
|
|
214
|
+
<a href="Dsl.html#method-i-rows">#rows</a>
|
|
215
|
+
—
|
|
216
|
+
<span class="container">Dsl</span>
|
|
217
|
+
|
|
218
|
+
<li class="method">
|
|
219
|
+
<a href="Sycsvpro/Counter.html#method-i-write_result">#write_result</a>
|
|
220
|
+
—
|
|
221
|
+
<span class="container">Sycsvpro::Counter</span>
|
|
222
|
+
|
|
223
|
+
<li class="method">
|
|
224
|
+
<a href="Dsl.html#method-i-write_to">#write_to</a>
|
|
225
|
+
—
|
|
226
|
+
<span class="container">Dsl</span>
|
|
227
|
+
</ul>
|
|
228
|
+
</main>
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
<footer id="validator-badges" role="contentinfo">
|
|
232
|
+
<p><a href="http://validator.w3.org/check/referer">Validate</a>
|
|
233
|
+
<p>Generated by <a href="http://rdoc.rubyforge.org">RDoc</a> 4.1.1.
|
|
234
|
+
<p>Based on <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish</a> by <a href="http://deveiate.org">Michael Granger</a>.
|
|
235
|
+
</footer>
|
|
236
|
+
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Operating csv files
|
|
2
|
+
module Sycsvpro
|
|
3
|
+
|
|
4
|
+
# Holds the analytics of the file
|
|
5
|
+
Result = Struct.new(:cols, :col_count, :row_count, :sample_row)
|
|
6
|
+
|
|
7
|
+
# Analyzes the file structure
|
|
8
|
+
class Analyzer
|
|
9
|
+
|
|
10
|
+
# File that is analyzed
|
|
11
|
+
attr_reader :file
|
|
12
|
+
|
|
13
|
+
# Creates a new analyzer
|
|
14
|
+
def initialize(file)
|
|
15
|
+
@file = file
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Analyzes the file and returns the result
|
|
19
|
+
def result
|
|
20
|
+
rows = File.readlines(file)
|
|
21
|
+
|
|
22
|
+
result = Result.new
|
|
23
|
+
unless rows.empty?
|
|
24
|
+
row_number = 0
|
|
25
|
+
row_number += 1 while rows[row_number].chomp.empty?
|
|
26
|
+
|
|
27
|
+
result.cols = rows[row_number].chomp.split(';')
|
|
28
|
+
result.col_count = result.cols.size
|
|
29
|
+
|
|
30
|
+
row_number += 1
|
|
31
|
+
row_number += 1 while rows[row_number].chomp.empty?
|
|
32
|
+
|
|
33
|
+
result.row_count = rows.size - 1
|
|
34
|
+
result.sample_row = rows[row_number].chomp
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
result
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
require_relative 'header'
|
|
3
|
+
|
|
4
|
+
# Operating csv files
|
|
5
|
+
module Sycsvpro
|
|
6
|
+
|
|
7
|
+
# Processes arithmetic operations on columns of a csv file. A column value has to be a number.
|
|
8
|
+
# Possible operations are +, -, * and /. It is also possible to use values of columns as an
|
|
9
|
+
# operator like c1*2 will multiply the value of column 1 with 2.
|
|
10
|
+
class Calculator
|
|
11
|
+
|
|
12
|
+
# infile contains the data that is operated on
|
|
13
|
+
attr_reader :infile
|
|
14
|
+
# outfile is the file where the result is written to
|
|
15
|
+
attr_reader :outfile
|
|
16
|
+
# filter that is used for rows
|
|
17
|
+
attr_reader :row_filter
|
|
18
|
+
# the operations on columns
|
|
19
|
+
attr_reader :formulae
|
|
20
|
+
# header of the outfile
|
|
21
|
+
attr_reader :header
|
|
22
|
+
# filter that is used for columns
|
|
23
|
+
attr_reader :columns
|
|
24
|
+
|
|
25
|
+
# Creates a new Calculator. Options expects :infile, :outfile, :rows and :columns. Optionally
|
|
26
|
+
# a header can be provided. The header can be supplemented with additional column names that
|
|
27
|
+
# are generated due to a arithmetic operation that creates new columns
|
|
28
|
+
def initialize(options={})
|
|
29
|
+
@infile = options[:infile]
|
|
30
|
+
@outfile = options[:outfile]
|
|
31
|
+
@row_filter = RowFilter.new(options[:rows])
|
|
32
|
+
@header = Header.new(options[:header])
|
|
33
|
+
@formulae = {}
|
|
34
|
+
create_calculator(options[:cols])
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Retrieves the values from a row as the result of a arithmetic operation
|
|
38
|
+
def method_missing(id, *args, &block)
|
|
39
|
+
to_number(columns[$1.to_i]) if id =~ /c(\d+)/
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Executes the calculator
|
|
43
|
+
def execute
|
|
44
|
+
processed_header = false
|
|
45
|
+
|
|
46
|
+
File.open(outfile, 'w') do |out|
|
|
47
|
+
File.open(infile).each_with_index do |line, index|
|
|
48
|
+
next if line.chomp.empty?
|
|
49
|
+
|
|
50
|
+
unless processed_header
|
|
51
|
+
header_row = header.process(line.chomp)
|
|
52
|
+
out.puts header_row unless header_row.empty?
|
|
53
|
+
processed_header = true
|
|
54
|
+
next
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
next if row_filter.process(line, row: index).nil?
|
|
58
|
+
|
|
59
|
+
@columns = line.chomp.split(';')
|
|
60
|
+
formulae.each do |col, formula|
|
|
61
|
+
@columns[col.to_i] = eval(formula)
|
|
62
|
+
end
|
|
63
|
+
out.puts @columns.join(';')
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
# given a csv file with a;b;c
|
|
71
|
+
# code is in the form of
|
|
72
|
+
# 1:*2,2:*c3-1,4:c1+1
|
|
73
|
+
# 1:*2 means multiply value from column 1 by 2 and assign it to column 1 c[1] = c[1]*2
|
|
74
|
+
# 2:*c3-1 means multiply value from column 2 with value from column 3, subtract 1 and assign
|
|
75
|
+
# the result to column 2 c[2] = c[2] * c[3] - 1
|
|
76
|
+
# 4:c1+1 means create a new column and assign to it the result of the sum of the value of
|
|
77
|
+
# column 1 + 1 c[4] = c[1] + 1
|
|
78
|
+
def create_calculator(code)
|
|
79
|
+
code.split(',').each do |operation|
|
|
80
|
+
col, term = operation.split(':')
|
|
81
|
+
term = "c#{col}#{term}" unless term =~ /^c\d+/
|
|
82
|
+
formulae[col] = term
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Casts a string to an integer or float depending whether the value has a decimal point
|
|
87
|
+
def to_number(value)
|
|
88
|
+
return value.to_i unless value =~ /\./
|
|
89
|
+
return value.to_f if value =~ /\./
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
require_relative 'column_filter'
|
|
3
|
+
|
|
4
|
+
# Operating csv files
|
|
5
|
+
module Sycsvpro
|
|
6
|
+
|
|
7
|
+
# Collects values from rows and groups them in categories
|
|
8
|
+
class Collector
|
|
9
|
+
|
|
10
|
+
# infile contains the data that is operated on
|
|
11
|
+
attr_reader :infile
|
|
12
|
+
# outfile is the file where the result is written to
|
|
13
|
+
attr_reader :outfile
|
|
14
|
+
# filter that is used for rows
|
|
15
|
+
attr_reader :row_filter
|
|
16
|
+
# collected values assigned to categories
|
|
17
|
+
attr_reader :collection
|
|
18
|
+
|
|
19
|
+
# Creates a new Collector
|
|
20
|
+
def initialize(options={})
|
|
21
|
+
@infile = options[:infile]
|
|
22
|
+
@outfile = options[:outfile]
|
|
23
|
+
@row_filter = RowFilter.new(options[:rows])
|
|
24
|
+
@collection = {}
|
|
25
|
+
init_collection(options[:cols])
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Execute the collector
|
|
29
|
+
def execute
|
|
30
|
+
File.new(infile).each_with_index do |line, index|
|
|
31
|
+
row = row_filter.process(line, row: index)
|
|
32
|
+
next if row.nil? or row.chomp.empty?
|
|
33
|
+
collection.each do |category, elements|
|
|
34
|
+
values = elements[:filter].process(row)
|
|
35
|
+
values.chomp.split(';').each do |value|
|
|
36
|
+
elements[:entries] << value.chomp if elements[:entries].index(value.chomp).nil?
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
File.open(outfile, 'w') do |out|
|
|
42
|
+
collection.each do |category, elements|
|
|
43
|
+
out.puts "[#{category}]"
|
|
44
|
+
elements[:entries].sort.each { |c| out.puts c }
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
# Initializes the collection
|
|
52
|
+
def init_collection(column_filter)
|
|
53
|
+
column_filter.split('+').each do |f|
|
|
54
|
+
category, filter = f.split(':')
|
|
55
|
+
collection[category] = { entries: [], filter: ColumnFilter.new(filter) }
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
require_relative 'filter'
|
|
2
|
+
|
|
3
|
+
# Operating csv files
|
|
4
|
+
module Sycsvpro
|
|
5
|
+
|
|
6
|
+
# Creates a new column filter
|
|
7
|
+
class ColumnFilter < Filter
|
|
8
|
+
|
|
9
|
+
# Processes the filter and returns the values that respect the filter
|
|
10
|
+
def process(object, options={})
|
|
11
|
+
return nil if object.nil? or object.empty?
|
|
12
|
+
object = object.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
|
|
13
|
+
return object if filter.empty? and pivot.empty?
|
|
14
|
+
filtered = object.split(';').values_at(*filter.flatten.uniq)
|
|
15
|
+
pivot_each_column(object.split(';')) do |column, match|
|
|
16
|
+
filtered << column if match
|
|
17
|
+
end
|
|
18
|
+
filtered.compact.join(';')
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
require_relative 'column_filter'
|
|
3
|
+
|
|
4
|
+
# Operating csv files
|
|
5
|
+
module Sycsvpro
|
|
6
|
+
|
|
7
|
+
# Creates a new counter that counts values and uses the values as column names and uses the count
|
|
8
|
+
# as the column value
|
|
9
|
+
class Counter
|
|
10
|
+
|
|
11
|
+
# infile contains the data that is operated on
|
|
12
|
+
attr_reader :infile
|
|
13
|
+
# outfile is the file where the result is written to
|
|
14
|
+
attr_reader :outfile
|
|
15
|
+
# values are assigned to the key column
|
|
16
|
+
attr_reader :key_column
|
|
17
|
+
# filter that is used for rows
|
|
18
|
+
attr_reader :row_filter
|
|
19
|
+
# filter that is used for columns
|
|
20
|
+
attr_reader :col_filter
|
|
21
|
+
# values that are assigned to the key column
|
|
22
|
+
attr_reader :customers
|
|
23
|
+
# header of the out file
|
|
24
|
+
attr_reader :heading
|
|
25
|
+
|
|
26
|
+
# Creates a new counter
|
|
27
|
+
def initialize(options={})
|
|
28
|
+
@infile = options[:infile]
|
|
29
|
+
@outfile = options[:outfile]
|
|
30
|
+
@key_column = options[:key].to_i
|
|
31
|
+
@row_filter = RowFilter.new(options[:rows])
|
|
32
|
+
@col_filter = ColumnFilter.new(options[:cols], df: options[:df])
|
|
33
|
+
@customers = {}
|
|
34
|
+
@heading = []
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Executes the counter
|
|
38
|
+
def execute
|
|
39
|
+
process_file
|
|
40
|
+
write_result
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Processes the counting on the in file
|
|
44
|
+
def process_file
|
|
45
|
+
File.new(infile).each_with_index do |line, index|
|
|
46
|
+
result = col_filter.process(row_filter.process(line.chomp, row: index))
|
|
47
|
+
unless result.nil? or result.empty?
|
|
48
|
+
key = line.split(';')[key_column]
|
|
49
|
+
customer = customers[key] || customers[key] = { name: key, products: Hash.new(0) }
|
|
50
|
+
result.chomp.split(';').each do |column|
|
|
51
|
+
heading << column if heading.index(column).nil?
|
|
52
|
+
customer[:products][column] += 1
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Writes the results
|
|
59
|
+
def write_result
|
|
60
|
+
File.open(outfile, 'w') do |out|
|
|
61
|
+
out.puts (["customer"] + heading.sort).join(';')
|
|
62
|
+
customers.each do |k,v|
|
|
63
|
+
line = [k]
|
|
64
|
+
heading.sort.each do |h|
|
|
65
|
+
line << v[:products][h]
|
|
66
|
+
end
|
|
67
|
+
out.puts line.join(';')
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
end
|
data/lib/sycsvpro/dsl.rb
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
|
|
3
|
+
# Methods to be used in customer specific script files
|
|
4
|
+
module Dsl
|
|
5
|
+
|
|
6
|
+
# Retrieves rows and columns from the file and returns them to the block provided by the caller
|
|
7
|
+
def rows(options={})
|
|
8
|
+
infile = File.expand_path(options[:infile])
|
|
9
|
+
row_filter = Sycsvpro::RowFilter.new(options[:row_filter]) if options[:row_filter]
|
|
10
|
+
|
|
11
|
+
File.new(infile).each_with_index do |line, index|
|
|
12
|
+
next if line.chomp.empty?
|
|
13
|
+
next if !row_filter.nil? and row_filter.process(line.chomp, row: index).nil?
|
|
14
|
+
|
|
15
|
+
values = line.chomp.split(';')
|
|
16
|
+
params = []
|
|
17
|
+
options.each { |k,v| params << extract_values(values, k, v) if k =~ /column$|columns$/ }
|
|
18
|
+
|
|
19
|
+
yield *params
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# writes values provided by a block to the given file
|
|
24
|
+
def write_to(file)
|
|
25
|
+
File.open(file, 'w') do |out|
|
|
26
|
+
yield out
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
# Assigns values to keys that are used in rows and yielded to the block
|
|
33
|
+
def extract_values(values, key, position)
|
|
34
|
+
return values[position] if key =~ /column$/
|
|
35
|
+
return values.values_at(*position) if key =~ /columns$/
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
require_relative 'column_filter'
|
|
3
|
+
|
|
4
|
+
# Operating csv files
|
|
5
|
+
module Sycsvpro
|
|
6
|
+
|
|
7
|
+
# Extracts rows and columns from a csv file
|
|
8
|
+
class Extractor
|
|
9
|
+
|
|
10
|
+
# infile contains the data that is operated on
|
|
11
|
+
attr_reader :in_file
|
|
12
|
+
# outfile is the file where the result is written to
|
|
13
|
+
attr_reader :out_file
|
|
14
|
+
# filter that is used for rows
|
|
15
|
+
attr_reader :row_filter
|
|
16
|
+
# filter that is used for columns
|
|
17
|
+
attr_reader :col_filter
|
|
18
|
+
|
|
19
|
+
# Creates a new extractor
|
|
20
|
+
def initialize(options={})
|
|
21
|
+
@in_file = options[:infile]
|
|
22
|
+
@out_file = options[:outfile]
|
|
23
|
+
@row_filter = RowFilter.new(options[:rows])
|
|
24
|
+
@col_filter = ColumnFilter.new(options[:cols])
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Executes the extractor
|
|
28
|
+
def execute
|
|
29
|
+
File.open(out_file, 'w') do |o|
|
|
30
|
+
File.new(in_file, 'r').each_with_index do |line, index|
|
|
31
|
+
extraction = col_filter.process(row_filter.process(line.chomp, row: index))
|
|
32
|
+
o.puts extraction unless extraction.nil?
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
end
|