easy_sheet_io 0.4.9 → 0.4.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +11 -11
- data/.rspec +3 -3
- data/.rubocop.yml +10 -10
- data/CODE_OF_CONDUCT.md +84 -84
- data/Gemfile +16 -16
- data/Gemfile.lock +92 -92
- data/LICENSE.txt +21 -21
- data/README.md +43 -43
- data/Rakefile +12 -12
- data/bin/console +15 -15
- data/bin/setup +8 -8
- data/easy_sheet_io.gemspec +45 -45
- data/lib/easy_sheet_io/version.rb +5 -5
- data/lib/easy_sheet_io.rb +256 -256
- data/lib/longest_line.rb +3 -3
- data/lib/to_csv.rb +72 -72
- metadata +6 -6
data/bin/setup
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
set -euo pipefail
|
|
3
|
-
IFS=$'\n\t'
|
|
4
|
-
set -vx
|
|
5
|
-
|
|
6
|
-
bundle install
|
|
7
|
-
|
|
8
|
-
# Do any other automated setup that you need to do here
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
IFS=$'\n\t'
|
|
4
|
+
set -vx
|
|
5
|
+
|
|
6
|
+
bundle install
|
|
7
|
+
|
|
8
|
+
# Do any other automated setup that you need to do here
|
data/easy_sheet_io.gemspec
CHANGED
|
@@ -1,45 +1,45 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require_relative "lib/easy_sheet_io/version"
|
|
4
|
-
|
|
5
|
-
Gem::Specification.new do |spec|
|
|
6
|
-
spec.name = "easy_sheet_io"
|
|
7
|
-
spec.version = EasySheetIo::VERSION
|
|
8
|
-
spec.authors = ["show-o-atakun"]
|
|
9
|
-
spec.email = ["shun_yamaguchi_tc@live.jp"]
|
|
10
|
-
|
|
11
|
-
spec.summary = "
|
|
12
|
-
spec.description = "A simple way to Open .csv, .xls, .xlsx files. You can convert it to 2D array, hash, data frame."
|
|
13
|
-
spec.homepage = "https://github.com/show-o-atakun/easy_sheet_io"
|
|
14
|
-
spec.license = "MIT"
|
|
15
|
-
spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
|
|
16
|
-
|
|
17
|
-
# spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
|
|
18
|
-
|
|
19
|
-
spec.metadata["homepage_uri"] = spec.homepage
|
|
20
|
-
spec.metadata["source_code_uri"] = "https://github.com/show-o-atakun/easy_sheet_io"
|
|
21
|
-
## spec.metadata["changelog_uri"] = "https://github.com/show-o-atakun/easy_sheet_io_gemspec"
|
|
22
|
-
|
|
23
|
-
# Specify which files should be added to the gem when it is released.
|
|
24
|
-
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
25
|
-
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
26
|
-
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
|
27
|
-
end
|
|
28
|
-
spec.bindir = "exe"
|
|
29
|
-
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
30
|
-
spec.require_paths = ["lib"]
|
|
31
|
-
|
|
32
|
-
# Uncomment to register a new dependency of your gem
|
|
33
|
-
# spec.add_dependency "example-gem", "~> 1.0"
|
|
34
|
-
## spec.add_dependency "rake", "~> 13.0"
|
|
35
|
-
## spec.add_dependency "rspec", "~> 3.0"
|
|
36
|
-
## spec.add_dependency "rubocop", "~> 0.80"
|
|
37
|
-
|
|
38
|
-
spec.add_dependency "daru", ">= 0.3"
|
|
39
|
-
spec.add_dependency "rover-df", ">= 0.2.7"
|
|
40
|
-
spec.add_dependency "smarter_csv", ">= 1.4.2"
|
|
41
|
-
spec.add_dependency "roo-xls", ">= 1.2.0"
|
|
42
|
-
spec.add_dependency "spreadsheet", ">= 1.3.0"
|
|
43
|
-
# For more information and examples about making a new gem, checkout our
|
|
44
|
-
# guide at: https://bundler.io/guides/creating_gem.html
|
|
45
|
-
end
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "lib/easy_sheet_io/version"
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = "easy_sheet_io"
|
|
7
|
+
spec.version = EasySheetIo::VERSION
|
|
8
|
+
spec.authors = ["show-o-atakun"]
|
|
9
|
+
spec.email = ["shun_yamaguchi_tc@live.jp"]
|
|
10
|
+
|
|
11
|
+
spec.summary = "**Deprecated** Use yomise instead."
|
|
12
|
+
spec.description = "**Deprecated** Use yomise instead. A simple way to Open .csv, .xls, .xlsx files. You can convert it to 2D array, hash, data frame."
|
|
13
|
+
spec.homepage = "https://github.com/show-o-atakun/easy_sheet_io"
|
|
14
|
+
spec.license = "MIT"
|
|
15
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
|
|
16
|
+
|
|
17
|
+
# spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
|
|
18
|
+
|
|
19
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
|
20
|
+
spec.metadata["source_code_uri"] = "https://github.com/show-o-atakun/easy_sheet_io"
|
|
21
|
+
## spec.metadata["changelog_uri"] = "https://github.com/show-o-atakun/easy_sheet_io_gemspec"
|
|
22
|
+
|
|
23
|
+
# Specify which files should be added to the gem when it is released.
|
|
24
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
25
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
26
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
|
27
|
+
end
|
|
28
|
+
spec.bindir = "exe"
|
|
29
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
30
|
+
spec.require_paths = ["lib"]
|
|
31
|
+
|
|
32
|
+
# Uncomment to register a new dependency of your gem
|
|
33
|
+
# spec.add_dependency "example-gem", "~> 1.0"
|
|
34
|
+
## spec.add_dependency "rake", "~> 13.0"
|
|
35
|
+
## spec.add_dependency "rspec", "~> 3.0"
|
|
36
|
+
## spec.add_dependency "rubocop", "~> 0.80"
|
|
37
|
+
|
|
38
|
+
spec.add_dependency "daru", ">= 0.3"
|
|
39
|
+
spec.add_dependency "rover-df", ">= 0.2.7"
|
|
40
|
+
spec.add_dependency "smarter_csv", ">= 1.4.2"
|
|
41
|
+
spec.add_dependency "roo-xls", ">= 1.2.0"
|
|
42
|
+
spec.add_dependency "spreadsheet", ">= 1.3.0"
|
|
43
|
+
# For more information and examples about making a new gem, checkout our
|
|
44
|
+
# guide at: https://bundler.io/guides/creating_gem.html
|
|
45
|
+
end
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module EasySheetIo
|
|
4
|
-
VERSION = "0.4.9"
|
|
5
|
-
end
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EasySheetIo
|
|
4
|
+
VERSION = "0.4.9.2"
|
|
5
|
+
end
|
data/lib/easy_sheet_io.rb
CHANGED
|
@@ -1,256 +1,256 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
require "csv"
|
|
3
|
-
require "roo-xls"
|
|
4
|
-
require "spreadsheet"
|
|
5
|
-
require "rover"
|
|
6
|
-
require "daru"
|
|
7
|
-
require_relative "./to_csv"
|
|
8
|
-
require_relative "./longest_line"
|
|
9
|
-
require_relative "easy_sheet_io/version"
|
|
10
|
-
|
|
11
|
-
module EasySheetIo
|
|
12
|
-
class Error < StandardError; end
|
|
13
|
-
|
|
14
|
-
module_function
|
|
15
|
-
|
|
16
|
-
def read(path, **opt)
|
|
17
|
-
return /csv$/ === path ? read_csv(path, **opt) : read_excel(path, **opt)
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
# ##Generate Array from CSV File, and convert it to Hash or DataFrame.
|
|
21
|
-
# **opt candidate= line_from: 1, header: 0
|
|
22
|
-
# ver. 0.3.8~ default format=:daru
|
|
23
|
-
def read_csv(path, format: :daru, encoding: "utf-8", col_sep: ",", index: nil, **opt)
|
|
24
|
-
## TODO.. index: option that designate column number to generate DF index.
|
|
25
|
-
## That is, revicing set_index method.
|
|
26
|
-
|
|
27
|
-
# Get 2D Array
|
|
28
|
-
begin
|
|
29
|
-
csv = CSV.parse(File.open(path, encoding: encoding, &:read), col_sep: col_sep)
|
|
30
|
-
rescue
|
|
31
|
-
# Try Another Encoding
|
|
32
|
-
## puts "Fail Encoding #{encoding}. Trying cp932..."
|
|
33
|
-
csv = CSV.parse(File.open(path, encoding: "cp932", &:read), col_sep: col_sep)
|
|
34
|
-
encoding = "cp932"
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
if format.to_s == "array"
|
|
38
|
-
return csv
|
|
39
|
-
elsif format.to_s == "hash"
|
|
40
|
-
h, i = to_hash(csv, **opt)
|
|
41
|
-
return h
|
|
42
|
-
else # include format.nil? (in this case, convert to Daru::DF).
|
|
43
|
-
|
|
44
|
-
h, ind_orig = to_hash(csv, index: index, **opt)
|
|
45
|
-
ans = to_df(h, format: format)
|
|
46
|
-
|
|
47
|
-
# Converting Encode and Setting index.. rover not supported yet
|
|
48
|
-
if format.to_s == "daru" || format.nil?
|
|
49
|
-
ans.convert_enc!(from: encoding, to: "utf-8")
|
|
50
|
-
begin
|
|
51
|
-
ans.index = ind_orig if index
|
|
52
|
-
rescue
|
|
53
|
-
warn "Indexing failed (Parhaps due to duplicated index)."
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
return ans
|
|
58
|
-
end
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
# ##Generate Array from EXCEL File, and convert it to Hash or DataFrame.
|
|
62
|
-
# **opt candidate= line_from: 1, header: 0)
|
|
63
|
-
def read_excel(path, sheet_i: 0, format: :daru, encoding: "utf-8", index: nil, **opt)
|
|
64
|
-
a2d = open_excel(path, sheet_i, encoding: encoding) # Get 2D Array
|
|
65
|
-
|
|
66
|
-
if format.to_s == "array"
|
|
67
|
-
return a2d
|
|
68
|
-
elsif format.to_s == "hash"
|
|
69
|
-
h, i = to_hash(a2d, **opt)
|
|
70
|
-
return h
|
|
71
|
-
else # include format.nil?
|
|
72
|
-
h, ind_orig = to_hash(a2d, index: index, **opt)
|
|
73
|
-
ans = to_df(h, format: format)
|
|
74
|
-
if format.to_s == "daru" || format.nil?
|
|
75
|
-
begin
|
|
76
|
-
ans.index = ind_orig if index
|
|
77
|
-
rescue
|
|
78
|
-
warn "Indexing failed (Parhaps due to duplicated index)."
|
|
79
|
-
end
|
|
80
|
-
end
|
|
81
|
-
return ans
|
|
82
|
-
end
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
# Convert 2d Array to Hash
|
|
86
|
-
## header: nil -> Default Headers(:column1, column2,...) are generated.
|
|
87
|
-
## Option line_ignored, is not implemented yet.
|
|
88
|
-
def to_hash(array2d, line_from: 1, line_until: nil, line_ignored: nil,
|
|
89
|
-
column_from: nil, column_until: nil,
|
|
90
|
-
header: 0, symbol_header: false,
|
|
91
|
-
replaced_by_nil: [], analyze_type: true,
|
|
92
|
-
index: nil)
|
|
93
|
-
## TODO.. column_from: , column_until:
|
|
94
|
-
|
|
95
|
-
# Define Read Range------------
|
|
96
|
-
lfrom, luntil = line_from, line_until
|
|
97
|
-
lf_reg, lu_reg = line_from.kind_of?(Regexp), line_until.kind_of?(Regexp)
|
|
98
|
-
|
|
99
|
-
if lf_reg || lu_reg
|
|
100
|
-
lines_ary = array2d.map{ _1.join "," }
|
|
101
|
-
lfrom = lines_ary.find_index{ line_from === _1 } if lf_reg
|
|
102
|
-
luntil = (lines_ary.length-1) - lines_ary.reverse.find_index{ line_until === _1 } if lu_reg
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
# And get originally array-----
|
|
106
|
-
output = array2d[lfrom...luntil]
|
|
107
|
-
# -----------------------------
|
|
108
|
-
|
|
109
|
-
# Then get data of index-------
|
|
110
|
-
ind_orig = index ? output.map{ _1[index] } : nil
|
|
111
|
-
# -----------------------------
|
|
112
|
-
|
|
113
|
-
# Selecct Column---------------
|
|
114
|
-
output = output.map { _1[column_from...column_until] } if column_from || column_until
|
|
115
|
-
|
|
116
|
-
# Define Data Array------------
|
|
117
|
-
output_transpose = output[0].zip(*output[1..])
|
|
118
|
-
output_transpose = fix_array(output_transpose, replaced_by_nil, analyze_type)
|
|
119
|
-
# -----------------------------
|
|
120
|
-
|
|
121
|
-
# Define Header----------------
|
|
122
|
-
if header
|
|
123
|
-
hd = check_header(array2d[header])[column_from...column_until]
|
|
124
|
-
else
|
|
125
|
-
hd = [*0...(output.longest_line)].map{"column#{_1}"}
|
|
126
|
-
end
|
|
127
|
-
# hd = header.nil? ? [*0...(output.longest_line)].map{"column#{_1}"} : check_header(array2d[header])
|
|
128
|
-
|
|
129
|
-
hd = hd.map { _1.intern } if symbol_header
|
|
130
|
-
# -----------------------------
|
|
131
|
-
|
|
132
|
-
# Make Hash(Header => Data Array)
|
|
133
|
-
return hd.each_with_object({}).with_index {|(hdr, hash), i| hash[hdr]=output_transpose[i]}, ind_orig
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
# Convert Hash to DataFrame
|
|
137
|
-
def to_df(d, format: :daru)
|
|
138
|
-
if format.to_s == "daru" || format.nil?
|
|
139
|
-
Daru::DataFrame.new(d)
|
|
140
|
-
else
|
|
141
|
-
Rover::DataFrame.new(d)
|
|
142
|
-
end
|
|
143
|
-
end
|
|
144
|
-
|
|
145
|
-
#----------------------------
|
|
146
|
-
# Private metods from here
|
|
147
|
-
#----------------------------
|
|
148
|
-
|
|
149
|
-
# Genarate Array from excel file
|
|
150
|
-
def open_excel(path, sheet_i, encoding: "utf-8")
|
|
151
|
-
if /xlsx$/ === path
|
|
152
|
-
puts "Sorry, encoding option is not supported yet for xlsx file." if encoding != "utf-8"
|
|
153
|
-
|
|
154
|
-
book = Roo::Excelx.new(path)
|
|
155
|
-
s = book.sheet(sheet_i)
|
|
156
|
-
|
|
157
|
-
## bottole neck
|
|
158
|
-
return s.to_a
|
|
159
|
-
|
|
160
|
-
# xls
|
|
161
|
-
else
|
|
162
|
-
begin
|
|
163
|
-
Spreadsheet.client_encoding = encoding
|
|
164
|
-
ss = Spreadsheet.open(path)
|
|
165
|
-
rescue Encoding::InvalidByteSequenceError
|
|
166
|
-
puts "Fail Encoding #{encoding}. Trying Windows-31J..."
|
|
167
|
-
Spreadsheet.client_encoding = "Windows-31J"
|
|
168
|
-
ss = Spreadsheet.open(path)
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
a2d = []
|
|
172
|
-
ss.worksheets[sheet_i].rows.each do |row|
|
|
173
|
-
a1d = []
|
|
174
|
-
row.each {|cell| a1d.push cell}
|
|
175
|
-
a2d.push a1d
|
|
176
|
-
end
|
|
177
|
-
|
|
178
|
-
return a2d
|
|
179
|
-
end
|
|
180
|
-
end
|
|
181
|
-
|
|
182
|
-
# Fix Array (Replace specific values to nil, recognize value type and cast values to the type.)
|
|
183
|
-
def fix_array(array2d, replaced_by_nil, analyze_type)
|
|
184
|
-
ans = array2d
|
|
185
|
-
|
|
186
|
-
## Replace Blank or User-Selected Value
|
|
187
|
-
ans = ans.map do |column|
|
|
188
|
-
column.map { |cell| replaced_by_nil.include?(cell) || /^\s*$/ === cell ? nil : cell }
|
|
189
|
-
end
|
|
190
|
-
|
|
191
|
-
## Replace Number Values to Integer or Float
|
|
192
|
-
if analyze_type
|
|
193
|
-
ans = ans.map.with_index do |column, i|
|
|
194
|
-
type_of_column = :any
|
|
195
|
-
column.each { |cell| type_of_column = recognize_type(cell, type_of_column) }
|
|
196
|
-
|
|
197
|
-
# p type_of_column
|
|
198
|
-
case type_of_column
|
|
199
|
-
when :int
|
|
200
|
-
column.map { _1.nil? ? nil : _1.to_i }
|
|
201
|
-
when :float
|
|
202
|
-
column.map { _1.nil? ? nil : _1.to_f }
|
|
203
|
-
else
|
|
204
|
-
column
|
|
205
|
-
end
|
|
206
|
-
end
|
|
207
|
-
end
|
|
208
|
-
|
|
209
|
-
return ans
|
|
210
|
-
end
|
|
211
|
-
|
|
212
|
-
def recognize_type(str, expected)
|
|
213
|
-
return expected if str.nil?
|
|
214
|
-
|
|
215
|
-
order = {:any => 0, :int => 1, :float => 2, :string => 3}
|
|
216
|
-
if /^\s*(-|\+)?\d+\s*$/ === str
|
|
217
|
-
type_of_str = :int
|
|
218
|
-
elsif /^\s*(-|\+)?\d*\.\d*\s*$/ === str || /^\s*(-|\+)?(\d*\.\d+|\d+)(e|E)(-|\+)?\d+\s*$/ === str
|
|
219
|
-
type_of_str = :float
|
|
220
|
-
else
|
|
221
|
-
type_of_str = :string
|
|
222
|
-
end
|
|
223
|
-
|
|
224
|
-
# p "#{type_of_str}, #{str}" if order[type_of_str] > order[expected]
|
|
225
|
-
|
|
226
|
-
return order[type_of_str] > order[expected] ? type_of_str : expected
|
|
227
|
-
end
|
|
228
|
-
|
|
229
|
-
# Fix blank or duplicated header
|
|
230
|
-
def check_header(header_array)
|
|
231
|
-
# Check Blank
|
|
232
|
-
ans = header_array.map.with_index do |item, i|
|
|
233
|
-
if item.nil?
|
|
234
|
-
"column#{i}"
|
|
235
|
-
elsif item.kind_of?(String)
|
|
236
|
-
temp = /^\s*$/ === item ? "column#{i}" : item.gsub(/\s+/, "")
|
|
237
|
-
/^\d+$/ === temp ? "column#{i}" : temp
|
|
238
|
-
else
|
|
239
|
-
item.to_s
|
|
240
|
-
end
|
|
241
|
-
end
|
|
242
|
-
|
|
243
|
-
# Check Duplicated Value
|
|
244
|
-
dup_check = (0...(header_array.length)).group_by {|i| ans[i]}
|
|
245
|
-
dup_check.each do |item, i_s|
|
|
246
|
-
if i_s.length > 1
|
|
247
|
-
i_s.each_with_index {|i, index_in_i_s| ans[i] = "#{ans[i]}_#{index_in_i_s}"}
|
|
248
|
-
end
|
|
249
|
-
end
|
|
250
|
-
|
|
251
|
-
return ans
|
|
252
|
-
end
|
|
253
|
-
|
|
254
|
-
private_class_method :open_excel, :fix_array, :check_header
|
|
255
|
-
|
|
256
|
-
end
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "csv"
|
|
3
|
+
require "roo-xls"
|
|
4
|
+
require "spreadsheet"
|
|
5
|
+
require "rover"
|
|
6
|
+
require "daru"
|
|
7
|
+
require_relative "./to_csv"
|
|
8
|
+
require_relative "./longest_line"
|
|
9
|
+
require_relative "easy_sheet_io/version"
|
|
10
|
+
|
|
11
|
+
module EasySheetIo
|
|
12
|
+
class Error < StandardError; end
|
|
13
|
+
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
def read(path, **opt)
|
|
17
|
+
return /csv$/ === path ? read_csv(path, **opt) : read_excel(path, **opt)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# ##Generate Array from CSV File, and convert it to Hash or DataFrame.
|
|
21
|
+
# **opt candidate= line_from: 1, header: 0
|
|
22
|
+
# ver. 0.3.8~ default format=:daru
|
|
23
|
+
def read_csv(path, format: :daru, encoding: "utf-8", col_sep: ",", index: nil, **opt)
|
|
24
|
+
## TODO.. index: option that designate column number to generate DF index.
|
|
25
|
+
## That is, revicing set_index method.
|
|
26
|
+
|
|
27
|
+
# Get 2D Array
|
|
28
|
+
begin
|
|
29
|
+
csv = CSV.parse(File.open(path, encoding: encoding, &:read), col_sep: col_sep)
|
|
30
|
+
rescue
|
|
31
|
+
# Try Another Encoding
|
|
32
|
+
## puts "Fail Encoding #{encoding}. Trying cp932..."
|
|
33
|
+
csv = CSV.parse(File.open(path, encoding: "cp932", &:read), col_sep: col_sep)
|
|
34
|
+
encoding = "cp932"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
if format.to_s == "array"
|
|
38
|
+
return csv
|
|
39
|
+
elsif format.to_s == "hash"
|
|
40
|
+
h, i = to_hash(csv, **opt)
|
|
41
|
+
return h
|
|
42
|
+
else # include format.nil? (in this case, convert to Daru::DF).
|
|
43
|
+
|
|
44
|
+
h, ind_orig = to_hash(csv, index: index, **opt)
|
|
45
|
+
ans = to_df(h, format: format)
|
|
46
|
+
|
|
47
|
+
# Converting Encode and Setting index.. rover not supported yet
|
|
48
|
+
if format.to_s == "daru" || format.nil?
|
|
49
|
+
ans.convert_enc!(from: encoding, to: "utf-8")
|
|
50
|
+
begin
|
|
51
|
+
ans.index = ind_orig if index
|
|
52
|
+
rescue
|
|
53
|
+
warn "Indexing failed (Parhaps due to duplicated index)."
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
return ans
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# ##Generate Array from EXCEL File, and convert it to Hash or DataFrame.
|
|
62
|
+
# **opt candidate= line_from: 1, header: 0)
|
|
63
|
+
def read_excel(path, sheet_i: 0, format: :daru, encoding: "utf-8", index: nil, **opt)
|
|
64
|
+
a2d = open_excel(path, sheet_i, encoding: encoding) # Get 2D Array
|
|
65
|
+
|
|
66
|
+
if format.to_s == "array"
|
|
67
|
+
return a2d
|
|
68
|
+
elsif format.to_s == "hash"
|
|
69
|
+
h, i = to_hash(a2d, **opt)
|
|
70
|
+
return h
|
|
71
|
+
else # include format.nil?
|
|
72
|
+
h, ind_orig = to_hash(a2d, index: index, **opt)
|
|
73
|
+
ans = to_df(h, format: format)
|
|
74
|
+
if format.to_s == "daru" || format.nil?
|
|
75
|
+
begin
|
|
76
|
+
ans.index = ind_orig if index
|
|
77
|
+
rescue
|
|
78
|
+
warn "Indexing failed (Parhaps due to duplicated index)."
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
return ans
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Convert 2d Array to Hash
|
|
86
|
+
## header: nil -> Default Headers(:column1, column2,...) are generated.
|
|
87
|
+
## Option line_ignored, is not implemented yet.
|
|
88
|
+
def to_hash(array2d, line_from: 1, line_until: nil, line_ignored: nil,
|
|
89
|
+
column_from: nil, column_until: nil,
|
|
90
|
+
header: 0, symbol_header: false,
|
|
91
|
+
replaced_by_nil: [], analyze_type: true,
|
|
92
|
+
index: nil)
|
|
93
|
+
## TODO.. column_from: , column_until:
|
|
94
|
+
|
|
95
|
+
# Define Read Range------------
|
|
96
|
+
lfrom, luntil = line_from, line_until
|
|
97
|
+
lf_reg, lu_reg = line_from.kind_of?(Regexp), line_until.kind_of?(Regexp)
|
|
98
|
+
|
|
99
|
+
if lf_reg || lu_reg
|
|
100
|
+
lines_ary = array2d.map{ _1.join "," }
|
|
101
|
+
lfrom = lines_ary.find_index{ line_from === _1 } if lf_reg
|
|
102
|
+
luntil = (lines_ary.length-1) - lines_ary.reverse.find_index{ line_until === _1 } if lu_reg
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# And get originally array-----
|
|
106
|
+
output = array2d[lfrom...luntil]
|
|
107
|
+
# -----------------------------
|
|
108
|
+
|
|
109
|
+
# Then get data of index-------
|
|
110
|
+
ind_orig = index ? output.map{ _1[index] } : nil
|
|
111
|
+
# -----------------------------
|
|
112
|
+
|
|
113
|
+
# Selecct Column---------------
|
|
114
|
+
output = output.map { _1[column_from...column_until] } if column_from || column_until
|
|
115
|
+
|
|
116
|
+
# Define Data Array------------
|
|
117
|
+
output_transpose = output[0].zip(*output[1..])
|
|
118
|
+
output_transpose = fix_array(output_transpose, replaced_by_nil, analyze_type)
|
|
119
|
+
# -----------------------------
|
|
120
|
+
|
|
121
|
+
# Define Header----------------
|
|
122
|
+
if header
|
|
123
|
+
hd = check_header(array2d[header])[column_from...column_until]
|
|
124
|
+
else
|
|
125
|
+
hd = [*0...(output.longest_line)].map{"column#{_1}"}
|
|
126
|
+
end
|
|
127
|
+
# hd = header.nil? ? [*0...(output.longest_line)].map{"column#{_1}"} : check_header(array2d[header])
|
|
128
|
+
|
|
129
|
+
hd = hd.map { _1.intern } if symbol_header
|
|
130
|
+
# -----------------------------
|
|
131
|
+
|
|
132
|
+
# Make Hash(Header => Data Array)
|
|
133
|
+
return hd.each_with_object({}).with_index {|(hdr, hash), i| hash[hdr]=output_transpose[i]}, ind_orig
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Convert Hash to DataFrame
|
|
137
|
+
def to_df(d, format: :daru)
|
|
138
|
+
if format.to_s == "daru" || format.nil?
|
|
139
|
+
Daru::DataFrame.new(d)
|
|
140
|
+
else
|
|
141
|
+
Rover::DataFrame.new(d)
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
#----------------------------
|
|
146
|
+
# Private metods from here
|
|
147
|
+
#----------------------------
|
|
148
|
+
|
|
149
|
+
# Genarate Array from excel file
|
|
150
|
+
def open_excel(path, sheet_i, encoding: "utf-8")
|
|
151
|
+
if /xlsx$/ === path
|
|
152
|
+
puts "Sorry, encoding option is not supported yet for xlsx file." if encoding != "utf-8"
|
|
153
|
+
|
|
154
|
+
book = Roo::Excelx.new(path)
|
|
155
|
+
s = book.sheet(sheet_i)
|
|
156
|
+
|
|
157
|
+
## bottole neck
|
|
158
|
+
return s.to_a
|
|
159
|
+
|
|
160
|
+
# xls
|
|
161
|
+
else
|
|
162
|
+
begin
|
|
163
|
+
Spreadsheet.client_encoding = encoding
|
|
164
|
+
ss = Spreadsheet.open(path)
|
|
165
|
+
rescue Encoding::InvalidByteSequenceError
|
|
166
|
+
puts "Fail Encoding #{encoding}. Trying Windows-31J..."
|
|
167
|
+
Spreadsheet.client_encoding = "Windows-31J"
|
|
168
|
+
ss = Spreadsheet.open(path)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
a2d = []
|
|
172
|
+
ss.worksheets[sheet_i].rows.each do |row|
|
|
173
|
+
a1d = []
|
|
174
|
+
row.each {|cell| a1d.push cell}
|
|
175
|
+
a2d.push a1d
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
return a2d
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Fix Array (Replace specific values to nil, recognize value type and cast values to the type.)
|
|
183
|
+
def fix_array(array2d, replaced_by_nil, analyze_type)
|
|
184
|
+
ans = array2d
|
|
185
|
+
|
|
186
|
+
## Replace Blank or User-Selected Value
|
|
187
|
+
ans = ans.map do |column|
|
|
188
|
+
column.map { |cell| replaced_by_nil.include?(cell) || /^\s*$/ === cell ? nil : cell }
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
## Replace Number Values to Integer or Float
|
|
192
|
+
if analyze_type
|
|
193
|
+
ans = ans.map.with_index do |column, i|
|
|
194
|
+
type_of_column = :any
|
|
195
|
+
column.each { |cell| type_of_column = recognize_type(cell, type_of_column) }
|
|
196
|
+
|
|
197
|
+
# p type_of_column
|
|
198
|
+
case type_of_column
|
|
199
|
+
when :int
|
|
200
|
+
column.map { _1.nil? ? nil : _1.to_i }
|
|
201
|
+
when :float
|
|
202
|
+
column.map { _1.nil? ? nil : _1.to_f }
|
|
203
|
+
else
|
|
204
|
+
column
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
return ans
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def recognize_type(str, expected)
|
|
213
|
+
return expected if str.nil?
|
|
214
|
+
|
|
215
|
+
order = {:any => 0, :int => 1, :float => 2, :string => 3}
|
|
216
|
+
if /^\s*(-|\+)?\d+\s*$/ === str
|
|
217
|
+
type_of_str = :int
|
|
218
|
+
elsif /^\s*(-|\+)?\d*\.\d*\s*$/ === str || /^\s*(-|\+)?(\d*\.\d+|\d+)(e|E)(-|\+)?\d+\s*$/ === str
|
|
219
|
+
type_of_str = :float
|
|
220
|
+
else
|
|
221
|
+
type_of_str = :string
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
# p "#{type_of_str}, #{str}" if order[type_of_str] > order[expected]
|
|
225
|
+
|
|
226
|
+
return order[type_of_str] > order[expected] ? type_of_str : expected
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Fix blank or duplicated header
|
|
230
|
+
def check_header(header_array)
|
|
231
|
+
# Check Blank
|
|
232
|
+
ans = header_array.map.with_index do |item, i|
|
|
233
|
+
if item.nil?
|
|
234
|
+
"column#{i}"
|
|
235
|
+
elsif item.kind_of?(String)
|
|
236
|
+
temp = /^\s*$/ === item ? "column#{i}" : item.gsub(/\s+/, "")
|
|
237
|
+
/^\d+$/ === temp ? "column#{i}" : temp
|
|
238
|
+
else
|
|
239
|
+
item.to_s
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Check Duplicated Value
|
|
244
|
+
dup_check = (0...(header_array.length)).group_by {|i| ans[i]}
|
|
245
|
+
dup_check.each do |item, i_s|
|
|
246
|
+
if i_s.length > 1
|
|
247
|
+
i_s.each_with_index {|i, index_in_i_s| ans[i] = "#{ans[i]}_#{index_in_i_s}"}
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
return ans
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
private_class_method :open_excel, :fix_array, :check_header
|
|
255
|
+
|
|
256
|
+
end
|
data/lib/longest_line.rb
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
class Array
|
|
2
|
-
# Returns line No. of longest line(not the length of it).
|
|
3
|
-
def longest_line = self.map(&:length).max
|
|
1
|
+
class Array
|
|
2
|
+
# Returns line No. of longest line(not the length of it).
|
|
3
|
+
def longest_line = self.map(&:length).max
|
|
4
4
|
end
|