sycsvpro 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +21 -0
- data/.rspec +1 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +51 -0
- data/LICENSE +20 -0
- data/README.md +188 -0
- data/README.rdoc +44 -0
- data/Rakefile +44 -0
- data/bin/sycsvpro +208 -0
- data/features/step_definitions/sycsvpro_steps.rb +6 -0
- data/features/support/env.rb +15 -0
- data/features/sycsvpro.feature +8 -0
- data/html/Dsl.html +201 -0
- data/html/Object.html +116 -0
- data/html/README_rdoc.html +178 -0
- data/html/Sycsvpro/Analyzer.html +239 -0
- data/html/Sycsvpro/Calculator.html +354 -0
- data/html/Sycsvpro/Collector.html +281 -0
- data/html/Sycsvpro/ColumnFilter.html +165 -0
- data/html/Sycsvpro/Counter.html +397 -0
- data/html/Sycsvpro/Extractor.html +269 -0
- data/html/Sycsvpro/Filter.html +349 -0
- data/html/Sycsvpro/Header.html +228 -0
- data/html/Sycsvpro/Mapper.html +288 -0
- data/html/Sycsvpro/Profiler.html +234 -0
- data/html/Sycsvpro/RowFilter.html +162 -0
- data/html/Sycsvpro.html +141 -0
- data/html/created.rid +17 -0
- data/html/fonts/Lato-Light.ttf +0 -0
- data/html/fonts/Lato-LightItalic.ttf +0 -0
- data/html/fonts/Lato-Regular.ttf +0 -0
- data/html/fonts/Lato-RegularItalic.ttf +0 -0
- data/html/fonts/SourceCodePro-Bold.ttf +0 -0
- data/html/fonts/SourceCodePro-Regular.ttf +0 -0
- data/html/fonts.css +167 -0
- data/html/images/add.png +0 -0
- data/html/images/arrow_up.png +0 -0
- data/html/images/brick.png +0 -0
- data/html/images/brick_link.png +0 -0
- data/html/images/bug.png +0 -0
- data/html/images/bullet_black.png +0 -0
- data/html/images/bullet_toggle_minus.png +0 -0
- data/html/images/bullet_toggle_plus.png +0 -0
- data/html/images/date.png +0 -0
- data/html/images/delete.png +0 -0
- data/html/images/find.png +0 -0
- data/html/images/loadingAnimation.gif +0 -0
- data/html/images/macFFBgHack.png +0 -0
- data/html/images/package.png +0 -0
- data/html/images/page_green.png +0 -0
- data/html/images/page_white_text.png +0 -0
- data/html/images/page_white_width.png +0 -0
- data/html/images/plugin.png +0 -0
- data/html/images/ruby.png +0 -0
- data/html/images/tag_blue.png +0 -0
- data/html/images/tag_green.png +0 -0
- data/html/images/transparent.png +0 -0
- data/html/images/wrench.png +0 -0
- data/html/images/wrench_orange.png +0 -0
- data/html/images/zoom.png +0 -0
- data/html/index.html +202 -0
- data/html/js/darkfish.js +140 -0
- data/html/js/jquery.js +18 -0
- data/html/js/navigation.js +142 -0
- data/html/js/search.js +109 -0
- data/html/js/search_index.js +1 -0
- data/html/js/searcher.js +228 -0
- data/html/rdoc.css +580 -0
- data/html/table_of_contents.html +236 -0
- data/lib/sycsvpro/analyzer.rb +40 -0
- data/lib/sycsvpro/calculator.rb +94 -0
- data/lib/sycsvpro/collector.rb +60 -0
- data/lib/sycsvpro/column_filter.rb +23 -0
- data/lib/sycsvpro/counter.rb +74 -0
- data/lib/sycsvpro/dsl.rb +37 -0
- data/lib/sycsvpro/extractor.rb +39 -0
- data/lib/sycsvpro/filter.rb +98 -0
- data/lib/sycsvpro/header.rb +29 -0
- data/lib/sycsvpro/mapper.rb +53 -0
- data/lib/sycsvpro/profiler.rb +26 -0
- data/lib/sycsvpro/row_filter.rb +20 -0
- data/lib/sycsvpro/version.rb +5 -0
- data/lib/sycsvpro.rb +9 -0
- data/spec/sycsvpro/analyze_spec.rb +23 -0
- data/spec/sycsvpro/calculator_spec.rb +45 -0
- data/spec/sycsvpro/collector_spec.rb +27 -0
- data/spec/sycsvpro/counter_spec.rb +51 -0
- data/spec/sycsvpro/extractor_spec.rb +27 -0
- data/spec/sycsvpro/files/mappings +6 -0
- data/spec/sycsvpro/files/profile.rb +42 -0
- data/spec/sycsvpro/mapper_spec.rb +33 -0
- data/spec/sycsvpro/profiler_spec.rb +32 -0
- data/sycsvpro.gemspec +24 -0
- data/sycsvpro.rdoc +29 -0
- metadata +215 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
|
|
3
|
+
<html>
|
|
4
|
+
<head>
|
|
5
|
+
<meta charset="UTF-8">
|
|
6
|
+
|
|
7
|
+
<title>Table of Contents - Your application title</title>
|
|
8
|
+
|
|
9
|
+
<link href="./fonts.css" rel="stylesheet">
|
|
10
|
+
<link href="./rdoc.css" rel="stylesheet">
|
|
11
|
+
|
|
12
|
+
<script type="text/javascript">
|
|
13
|
+
var rdoc_rel_prefix = "./";
|
|
14
|
+
</script>
|
|
15
|
+
|
|
16
|
+
<script src="./js/jquery.js"></script>
|
|
17
|
+
<script src="./js/navigation.js"></script>
|
|
18
|
+
<script src="./js/search_index.js"></script>
|
|
19
|
+
<script src="./js/search.js"></script>
|
|
20
|
+
<script src="./js/searcher.js"></script>
|
|
21
|
+
<script src="./js/darkfish.js"></script>
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
<body id="top" class="table-of-contents">
|
|
25
|
+
<main role="main">
|
|
26
|
+
<h1 class="class">Table of Contents - Your application title</h1>
|
|
27
|
+
|
|
28
|
+
<h2 id="pages">Pages</h2>
|
|
29
|
+
<ul>
|
|
30
|
+
<li class="file">
|
|
31
|
+
<a href="README_rdoc.html">README</a>
|
|
32
|
+
|
|
33
|
+
<ul>
|
|
34
|
+
<li><a href="README_rdoc.html#label-sycsvpro">sycsvpro</a>
|
|
35
|
+
<li><a href="README_rdoc.html#label-Install">Install</a>
|
|
36
|
+
<li><a href="README_rdoc.html#label-Use">Use</a>
|
|
37
|
+
<li><a href="README_rdoc.html#label-Developing+for+%60sycsvpro%60">Developing for `sycsvpro`</a>
|
|
38
|
+
<li><a href="README_rdoc.html#label-sycsvpro">sycsvpro</a>
|
|
39
|
+
</ul>
|
|
40
|
+
</li>
|
|
41
|
+
|
|
42
|
+
</ul>
|
|
43
|
+
|
|
44
|
+
<h2 id="classes">Classes and Modules</h2>
|
|
45
|
+
<ul>
|
|
46
|
+
<li class="module">
|
|
47
|
+
<a href="Dsl.html">Dsl</a>
|
|
48
|
+
</li>
|
|
49
|
+
<li class="class">
|
|
50
|
+
<a href="Object.html">Object</a>
|
|
51
|
+
</li>
|
|
52
|
+
<li class="module">
|
|
53
|
+
<a href="Sycsvpro.html">Sycsvpro</a>
|
|
54
|
+
</li>
|
|
55
|
+
<li class="class">
|
|
56
|
+
<a href="Sycsvpro/Analyzer.html">Sycsvpro::Analyzer</a>
|
|
57
|
+
</li>
|
|
58
|
+
<li class="class">
|
|
59
|
+
<a href="Sycsvpro/Calculator.html">Sycsvpro::Calculator</a>
|
|
60
|
+
</li>
|
|
61
|
+
<li class="class">
|
|
62
|
+
<a href="Sycsvpro/Collector.html">Sycsvpro::Collector</a>
|
|
63
|
+
</li>
|
|
64
|
+
<li class="class">
|
|
65
|
+
<a href="Sycsvpro/ColumnFilter.html">Sycsvpro::ColumnFilter</a>
|
|
66
|
+
</li>
|
|
67
|
+
<li class="class">
|
|
68
|
+
<a href="Sycsvpro/Counter.html">Sycsvpro::Counter</a>
|
|
69
|
+
</li>
|
|
70
|
+
<li class="class">
|
|
71
|
+
<a href="Sycsvpro/Extractor.html">Sycsvpro::Extractor</a>
|
|
72
|
+
</li>
|
|
73
|
+
<li class="class">
|
|
74
|
+
<a href="Sycsvpro/Filter.html">Sycsvpro::Filter</a>
|
|
75
|
+
</li>
|
|
76
|
+
<li class="class">
|
|
77
|
+
<a href="Sycsvpro/Header.html">Sycsvpro::Header</a>
|
|
78
|
+
</li>
|
|
79
|
+
<li class="class">
|
|
80
|
+
<a href="Sycsvpro/Mapper.html">Sycsvpro::Mapper</a>
|
|
81
|
+
</li>
|
|
82
|
+
<li class="class">
|
|
83
|
+
<a href="Sycsvpro/Profiler.html">Sycsvpro::Profiler</a>
|
|
84
|
+
</li>
|
|
85
|
+
<li class="class">
|
|
86
|
+
<a href="Sycsvpro/RowFilter.html">Sycsvpro::RowFilter</a>
|
|
87
|
+
</li>
|
|
88
|
+
</ul>
|
|
89
|
+
|
|
90
|
+
<h2 id="methods">Methods</h2>
|
|
91
|
+
<ul>
|
|
92
|
+
|
|
93
|
+
<li class="method">
|
|
94
|
+
<a href="Sycsvpro/Analyzer.html#method-c-new">::new</a>
|
|
95
|
+
—
|
|
96
|
+
<span class="container">Sycsvpro::Analyzer</span>
|
|
97
|
+
|
|
98
|
+
<li class="method">
|
|
99
|
+
<a href="Sycsvpro/Extractor.html#method-c-new">::new</a>
|
|
100
|
+
—
|
|
101
|
+
<span class="container">Sycsvpro::Extractor</span>
|
|
102
|
+
|
|
103
|
+
<li class="method">
|
|
104
|
+
<a href="Sycsvpro/Calculator.html#method-c-new">::new</a>
|
|
105
|
+
—
|
|
106
|
+
<span class="container">Sycsvpro::Calculator</span>
|
|
107
|
+
|
|
108
|
+
<li class="method">
|
|
109
|
+
<a href="Sycsvpro/Profiler.html#method-c-new">::new</a>
|
|
110
|
+
—
|
|
111
|
+
<span class="container">Sycsvpro::Profiler</span>
|
|
112
|
+
|
|
113
|
+
<li class="method">
|
|
114
|
+
<a href="Sycsvpro/Header.html#method-c-new">::new</a>
|
|
115
|
+
—
|
|
116
|
+
<span class="container">Sycsvpro::Header</span>
|
|
117
|
+
|
|
118
|
+
<li class="method">
|
|
119
|
+
<a href="Sycsvpro/Collector.html#method-c-new">::new</a>
|
|
120
|
+
—
|
|
121
|
+
<span class="container">Sycsvpro::Collector</span>
|
|
122
|
+
|
|
123
|
+
<li class="method">
|
|
124
|
+
<a href="Sycsvpro/Mapper.html#method-c-new">::new</a>
|
|
125
|
+
—
|
|
126
|
+
<span class="container">Sycsvpro::Mapper</span>
|
|
127
|
+
|
|
128
|
+
<li class="method">
|
|
129
|
+
<a href="Sycsvpro/Counter.html#method-c-new">::new</a>
|
|
130
|
+
—
|
|
131
|
+
<span class="container">Sycsvpro::Counter</span>
|
|
132
|
+
|
|
133
|
+
<li class="method">
|
|
134
|
+
<a href="Sycsvpro/Filter.html#method-c-new">::new</a>
|
|
135
|
+
—
|
|
136
|
+
<span class="container">Sycsvpro::Filter</span>
|
|
137
|
+
|
|
138
|
+
<li class="method">
|
|
139
|
+
<a href="Sycsvpro/Counter.html#method-i-execute">#execute</a>
|
|
140
|
+
—
|
|
141
|
+
<span class="container">Sycsvpro::Counter</span>
|
|
142
|
+
|
|
143
|
+
<li class="method">
|
|
144
|
+
<a href="Sycsvpro/Collector.html#method-i-execute">#execute</a>
|
|
145
|
+
—
|
|
146
|
+
<span class="container">Sycsvpro::Collector</span>
|
|
147
|
+
|
|
148
|
+
<li class="method">
|
|
149
|
+
<a href="Sycsvpro/Calculator.html#method-i-execute">#execute</a>
|
|
150
|
+
—
|
|
151
|
+
<span class="container">Sycsvpro::Calculator</span>
|
|
152
|
+
|
|
153
|
+
<li class="method">
|
|
154
|
+
<a href="Sycsvpro/Profiler.html#method-i-execute">#execute</a>
|
|
155
|
+
—
|
|
156
|
+
<span class="container">Sycsvpro::Profiler</span>
|
|
157
|
+
|
|
158
|
+
<li class="method">
|
|
159
|
+
<a href="Sycsvpro/Mapper.html#method-i-execute">#execute</a>
|
|
160
|
+
—
|
|
161
|
+
<span class="container">Sycsvpro::Mapper</span>
|
|
162
|
+
|
|
163
|
+
<li class="method">
|
|
164
|
+
<a href="Sycsvpro/Extractor.html#method-i-execute">#execute</a>
|
|
165
|
+
—
|
|
166
|
+
<span class="container">Sycsvpro::Extractor</span>
|
|
167
|
+
|
|
168
|
+
<li class="method">
|
|
169
|
+
<a href="Sycsvpro/Filter.html#method-i-method_missing">#method_missing</a>
|
|
170
|
+
—
|
|
171
|
+
<span class="container">Sycsvpro::Filter</span>
|
|
172
|
+
|
|
173
|
+
<li class="method">
|
|
174
|
+
<a href="Sycsvpro/Calculator.html#method-i-method_missing">#method_missing</a>
|
|
175
|
+
—
|
|
176
|
+
<span class="container">Sycsvpro::Calculator</span>
|
|
177
|
+
|
|
178
|
+
<li class="method">
|
|
179
|
+
<a href="Sycsvpro/Filter.html#method-i-pivot_each_column">#pivot_each_column</a>
|
|
180
|
+
—
|
|
181
|
+
<span class="container">Sycsvpro::Filter</span>
|
|
182
|
+
|
|
183
|
+
<li class="method">
|
|
184
|
+
<a href="Sycsvpro/Header.html#method-i-process">#process</a>
|
|
185
|
+
—
|
|
186
|
+
<span class="container">Sycsvpro::Header</span>
|
|
187
|
+
|
|
188
|
+
<li class="method">
|
|
189
|
+
<a href="Sycsvpro/Filter.html#method-i-process">#process</a>
|
|
190
|
+
—
|
|
191
|
+
<span class="container">Sycsvpro::Filter</span>
|
|
192
|
+
|
|
193
|
+
<li class="method">
|
|
194
|
+
<a href="Sycsvpro/RowFilter.html#method-i-process">#process</a>
|
|
195
|
+
—
|
|
196
|
+
<span class="container">Sycsvpro::RowFilter</span>
|
|
197
|
+
|
|
198
|
+
<li class="method">
|
|
199
|
+
<a href="Sycsvpro/ColumnFilter.html#method-i-process">#process</a>
|
|
200
|
+
—
|
|
201
|
+
<span class="container">Sycsvpro::ColumnFilter</span>
|
|
202
|
+
|
|
203
|
+
<li class="method">
|
|
204
|
+
<a href="Sycsvpro/Counter.html#method-i-process_file">#process_file</a>
|
|
205
|
+
—
|
|
206
|
+
<span class="container">Sycsvpro::Counter</span>
|
|
207
|
+
|
|
208
|
+
<li class="method">
|
|
209
|
+
<a href="Sycsvpro/Analyzer.html#method-i-result">#result</a>
|
|
210
|
+
—
|
|
211
|
+
<span class="container">Sycsvpro::Analyzer</span>
|
|
212
|
+
|
|
213
|
+
<li class="method">
|
|
214
|
+
<a href="Dsl.html#method-i-rows">#rows</a>
|
|
215
|
+
—
|
|
216
|
+
<span class="container">Dsl</span>
|
|
217
|
+
|
|
218
|
+
<li class="method">
|
|
219
|
+
<a href="Sycsvpro/Counter.html#method-i-write_result">#write_result</a>
|
|
220
|
+
—
|
|
221
|
+
<span class="container">Sycsvpro::Counter</span>
|
|
222
|
+
|
|
223
|
+
<li class="method">
|
|
224
|
+
<a href="Dsl.html#method-i-write_to">#write_to</a>
|
|
225
|
+
—
|
|
226
|
+
<span class="container">Dsl</span>
|
|
227
|
+
</ul>
|
|
228
|
+
</main>
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
<footer id="validator-badges" role="contentinfo">
|
|
232
|
+
<p><a href="http://validator.w3.org/check/referer">Validate</a>
|
|
233
|
+
<p>Generated by <a href="http://rdoc.rubyforge.org">RDoc</a> 4.1.1.
|
|
234
|
+
<p>Based on <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish</a> by <a href="http://deveiate.org">Michael Granger</a>.
|
|
235
|
+
</footer>
|
|
236
|
+
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Operating csv files
|
|
2
|
+
module Sycsvpro
|
|
3
|
+
|
|
4
|
+
# Holds the analytics of the file
|
|
5
|
+
Result = Struct.new(:cols, :col_count, :row_count, :sample_row)
|
|
6
|
+
|
|
7
|
+
# Analyzes the file structure
|
|
8
|
+
class Analyzer
|
|
9
|
+
|
|
10
|
+
# File that is analyzed
|
|
11
|
+
attr_reader :file
|
|
12
|
+
|
|
13
|
+
# Creates a new analyzer
|
|
14
|
+
def initialize(file)
|
|
15
|
+
@file = file
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Analyzes the file and returns the result
|
|
19
|
+
def result
|
|
20
|
+
rows = File.readlines(file)
|
|
21
|
+
|
|
22
|
+
result = Result.new
|
|
23
|
+
unless rows.empty?
|
|
24
|
+
row_number = 0
|
|
25
|
+
row_number += 1 while rows[row_number].chomp.empty?
|
|
26
|
+
|
|
27
|
+
result.cols = rows[row_number].chomp.split(';')
|
|
28
|
+
result.col_count = result.cols.size
|
|
29
|
+
|
|
30
|
+
row_number += 1
|
|
31
|
+
row_number += 1 while rows[row_number].chomp.empty?
|
|
32
|
+
|
|
33
|
+
result.row_count = rows.size - 1
|
|
34
|
+
result.sample_row = rows[row_number].chomp
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
result
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
require_relative 'header'
|
|
3
|
+
|
|
4
|
+
# Operating csv files
|
|
5
|
+
module Sycsvpro
|
|
6
|
+
|
|
7
|
+
# Processes arithmetic operations on columns of a csv file. A column value has to be a number.
|
|
8
|
+
# Possible operations are +, -, * and /. It is also possible to use values of columns as an
|
|
9
|
+
# operator like c1*2 will multiply the value of column 1 with 2.
|
|
10
|
+
class Calculator
|
|
11
|
+
|
|
12
|
+
# infile contains the data that is operated on
|
|
13
|
+
attr_reader :infile
|
|
14
|
+
# outfile is the file where the result is written to
|
|
15
|
+
attr_reader :outfile
|
|
16
|
+
# filter that is used for rows
|
|
17
|
+
attr_reader :row_filter
|
|
18
|
+
# the operations on columns
|
|
19
|
+
attr_reader :formulae
|
|
20
|
+
# header of the outfile
|
|
21
|
+
attr_reader :header
|
|
22
|
+
# filter that is used for columns
|
|
23
|
+
attr_reader :columns
|
|
24
|
+
|
|
25
|
+
# Creates a new Calculator. Options expects :infile, :outfile, :rows and :columns. Optionally
|
|
26
|
+
# a header can be provided. The header can be supplemented with additional column names that
|
|
27
|
+
# are generated due to a arithmetic operation that creates new columns
|
|
28
|
+
def initialize(options={})
|
|
29
|
+
@infile = options[:infile]
|
|
30
|
+
@outfile = options[:outfile]
|
|
31
|
+
@row_filter = RowFilter.new(options[:rows])
|
|
32
|
+
@header = Header.new(options[:header])
|
|
33
|
+
@formulae = {}
|
|
34
|
+
create_calculator(options[:cols])
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Retrieves the values from a row as the result of a arithmetic operation
|
|
38
|
+
def method_missing(id, *args, &block)
|
|
39
|
+
to_number(columns[$1.to_i]) if id =~ /c(\d+)/
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Executes the calculator
|
|
43
|
+
def execute
|
|
44
|
+
processed_header = false
|
|
45
|
+
|
|
46
|
+
File.open(outfile, 'w') do |out|
|
|
47
|
+
File.open(infile).each_with_index do |line, index|
|
|
48
|
+
next if line.chomp.empty?
|
|
49
|
+
|
|
50
|
+
unless processed_header
|
|
51
|
+
header_row = header.process(line.chomp)
|
|
52
|
+
out.puts header_row unless header_row.empty?
|
|
53
|
+
processed_header = true
|
|
54
|
+
next
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
next if row_filter.process(line, row: index).nil?
|
|
58
|
+
|
|
59
|
+
@columns = line.chomp.split(';')
|
|
60
|
+
formulae.each do |col, formula|
|
|
61
|
+
@columns[col.to_i] = eval(formula)
|
|
62
|
+
end
|
|
63
|
+
out.puts @columns.join(';')
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
# given a csv file with a;b;c
|
|
71
|
+
# code is in the form of
|
|
72
|
+
# 1:*2,2:*c3-1,4:c1+1
|
|
73
|
+
# 1:*2 means multiply value from column 1 by 2 and assign it to column 1 c[1] = c[1]*2
|
|
74
|
+
# 2:*c3-1 means multiply value from column 2 with value from column 3, subtract 1 and assign
|
|
75
|
+
# the result to column 2 c[2] = c[2] * c[3] - 1
|
|
76
|
+
# 4:c1+1 means create a new column and assign to it the result of the sum of the value of
|
|
77
|
+
# column 1 + 1 c[4] = c[1] + 1
|
|
78
|
+
def create_calculator(code)
|
|
79
|
+
code.split(',').each do |operation|
|
|
80
|
+
col, term = operation.split(':')
|
|
81
|
+
term = "c#{col}#{term}" unless term =~ /^c\d+/
|
|
82
|
+
formulae[col] = term
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Casts a string to an integer or float depending whether the value has a decimal point
|
|
87
|
+
def to_number(value)
|
|
88
|
+
return value.to_i unless value =~ /\./
|
|
89
|
+
return value.to_f if value =~ /\./
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
require_relative 'column_filter'
|
|
3
|
+
|
|
4
|
+
# Operating csv files
|
|
5
|
+
module Sycsvpro
|
|
6
|
+
|
|
7
|
+
# Collects values from rows and groups them in categories
|
|
8
|
+
class Collector
|
|
9
|
+
|
|
10
|
+
# infile contains the data that is operated on
|
|
11
|
+
attr_reader :infile
|
|
12
|
+
# outfile is the file where the result is written to
|
|
13
|
+
attr_reader :outfile
|
|
14
|
+
# filter that is used for rows
|
|
15
|
+
attr_reader :row_filter
|
|
16
|
+
# collected values assigned to categories
|
|
17
|
+
attr_reader :collection
|
|
18
|
+
|
|
19
|
+
# Creates a new Collector
|
|
20
|
+
def initialize(options={})
|
|
21
|
+
@infile = options[:infile]
|
|
22
|
+
@outfile = options[:outfile]
|
|
23
|
+
@row_filter = RowFilter.new(options[:rows])
|
|
24
|
+
@collection = {}
|
|
25
|
+
init_collection(options[:cols])
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Execute the collector
|
|
29
|
+
def execute
|
|
30
|
+
File.new(infile).each_with_index do |line, index|
|
|
31
|
+
row = row_filter.process(line, row: index)
|
|
32
|
+
next if row.nil? or row.chomp.empty?
|
|
33
|
+
collection.each do |category, elements|
|
|
34
|
+
values = elements[:filter].process(row)
|
|
35
|
+
values.chomp.split(';').each do |value|
|
|
36
|
+
elements[:entries] << value.chomp if elements[:entries].index(value.chomp).nil?
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
File.open(outfile, 'w') do |out|
|
|
42
|
+
collection.each do |category, elements|
|
|
43
|
+
out.puts "[#{category}]"
|
|
44
|
+
elements[:entries].sort.each { |c| out.puts c }
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
# Initializes the collection
|
|
52
|
+
def init_collection(column_filter)
|
|
53
|
+
column_filter.split('+').each do |f|
|
|
54
|
+
category, filter = f.split(':')
|
|
55
|
+
collection[category] = { entries: [], filter: ColumnFilter.new(filter) }
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
require_relative 'filter'
|
|
2
|
+
|
|
3
|
+
# Operating csv files
|
|
4
|
+
module Sycsvpro
|
|
5
|
+
|
|
6
|
+
# Creates a new column filter
|
|
7
|
+
class ColumnFilter < Filter
|
|
8
|
+
|
|
9
|
+
# Processes the filter and returns the values that respect the filter
|
|
10
|
+
def process(object, options={})
|
|
11
|
+
return nil if object.nil? or object.empty?
|
|
12
|
+
object = object.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
|
|
13
|
+
return object if filter.empty? and pivot.empty?
|
|
14
|
+
filtered = object.split(';').values_at(*filter.flatten.uniq)
|
|
15
|
+
pivot_each_column(object.split(';')) do |column, match|
|
|
16
|
+
filtered << column if match
|
|
17
|
+
end
|
|
18
|
+
filtered.compact.join(';')
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
require_relative 'column_filter'
|
|
3
|
+
|
|
4
|
+
# Operating csv files
|
|
5
|
+
module Sycsvpro
|
|
6
|
+
|
|
7
|
+
# Creates a new counter that counts values and uses the values as column names and uses the count
|
|
8
|
+
# as the column value
|
|
9
|
+
class Counter
|
|
10
|
+
|
|
11
|
+
# infile contains the data that is operated on
|
|
12
|
+
attr_reader :infile
|
|
13
|
+
# outfile is the file where the result is written to
|
|
14
|
+
attr_reader :outfile
|
|
15
|
+
# values are assigned to the key column
|
|
16
|
+
attr_reader :key_column
|
|
17
|
+
# filter that is used for rows
|
|
18
|
+
attr_reader :row_filter
|
|
19
|
+
# filter that is used for columns
|
|
20
|
+
attr_reader :col_filter
|
|
21
|
+
# values that are assigned to the key column
|
|
22
|
+
attr_reader :customers
|
|
23
|
+
# header of the out file
|
|
24
|
+
attr_reader :heading
|
|
25
|
+
|
|
26
|
+
# Creates a new counter
|
|
27
|
+
def initialize(options={})
|
|
28
|
+
@infile = options[:infile]
|
|
29
|
+
@outfile = options[:outfile]
|
|
30
|
+
@key_column = options[:key].to_i
|
|
31
|
+
@row_filter = RowFilter.new(options[:rows])
|
|
32
|
+
@col_filter = ColumnFilter.new(options[:cols], df: options[:df])
|
|
33
|
+
@customers = {}
|
|
34
|
+
@heading = []
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Executes the counter
|
|
38
|
+
def execute
|
|
39
|
+
process_file
|
|
40
|
+
write_result
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Processes the counting on the in file
|
|
44
|
+
def process_file
|
|
45
|
+
File.new(infile).each_with_index do |line, index|
|
|
46
|
+
result = col_filter.process(row_filter.process(line.chomp, row: index))
|
|
47
|
+
unless result.nil? or result.empty?
|
|
48
|
+
key = line.split(';')[key_column]
|
|
49
|
+
customer = customers[key] || customers[key] = { name: key, products: Hash.new(0) }
|
|
50
|
+
result.chomp.split(';').each do |column|
|
|
51
|
+
heading << column if heading.index(column).nil?
|
|
52
|
+
customer[:products][column] += 1
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Writes the results
|
|
59
|
+
def write_result
|
|
60
|
+
File.open(outfile, 'w') do |out|
|
|
61
|
+
out.puts (["customer"] + heading.sort).join(';')
|
|
62
|
+
customers.each do |k,v|
|
|
63
|
+
line = [k]
|
|
64
|
+
heading.sort.each do |h|
|
|
65
|
+
line << v[:products][h]
|
|
66
|
+
end
|
|
67
|
+
out.puts line.join(';')
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
end
|
data/lib/sycsvpro/dsl.rb
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
|
|
3
|
+
# Methods to be used in customer specific script files
|
|
4
|
+
module Dsl
|
|
5
|
+
|
|
6
|
+
# Retrieves rows and columns from the file and returns them to the block provided by the caller
|
|
7
|
+
def rows(options={})
|
|
8
|
+
infile = File.expand_path(options[:infile])
|
|
9
|
+
row_filter = Sycsvpro::RowFilter.new(options[:row_filter]) if options[:row_filter]
|
|
10
|
+
|
|
11
|
+
File.new(infile).each_with_index do |line, index|
|
|
12
|
+
next if line.chomp.empty?
|
|
13
|
+
next if !row_filter.nil? and row_filter.process(line.chomp, row: index).nil?
|
|
14
|
+
|
|
15
|
+
values = line.chomp.split(';')
|
|
16
|
+
params = []
|
|
17
|
+
options.each { |k,v| params << extract_values(values, k, v) if k =~ /column$|columns$/ }
|
|
18
|
+
|
|
19
|
+
yield *params
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# writes values provided by a block to the given file
|
|
24
|
+
def write_to(file)
|
|
25
|
+
File.open(file, 'w') do |out|
|
|
26
|
+
yield out
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
# Assigns values to keys that are used in rows and yielded to the block
|
|
33
|
+
def extract_values(values, key, position)
|
|
34
|
+
return values[position] if key =~ /column$/
|
|
35
|
+
return values.values_at(*position) if key =~ /columns$/
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
require_relative 'row_filter'
|
|
2
|
+
require_relative 'column_filter'
|
|
3
|
+
|
|
4
|
+
# Operating csv files
|
|
5
|
+
module Sycsvpro
|
|
6
|
+
|
|
7
|
+
# Extracts rows and columns from a csv file
|
|
8
|
+
class Extractor
|
|
9
|
+
|
|
10
|
+
# infile contains the data that is operated on
|
|
11
|
+
attr_reader :in_file
|
|
12
|
+
# outfile is the file where the result is written to
|
|
13
|
+
attr_reader :out_file
|
|
14
|
+
# filter that is used for rows
|
|
15
|
+
attr_reader :row_filter
|
|
16
|
+
# filter that is used for columns
|
|
17
|
+
attr_reader :col_filter
|
|
18
|
+
|
|
19
|
+
# Creates a new extractor
|
|
20
|
+
def initialize(options={})
|
|
21
|
+
@in_file = options[:infile]
|
|
22
|
+
@out_file = options[:outfile]
|
|
23
|
+
@row_filter = RowFilter.new(options[:rows])
|
|
24
|
+
@col_filter = ColumnFilter.new(options[:cols])
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Executes the extractor
|
|
28
|
+
def execute
|
|
29
|
+
File.open(out_file, 'w') do |o|
|
|
30
|
+
File.new(in_file, 'r').each_with_index do |line, index|
|
|
31
|
+
extraction = col_filter.process(row_filter.process(line.chomp, row: index))
|
|
32
|
+
o.puts extraction unless extraction.nil?
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
end
|