datafy 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/datafy/BaseHarvester.rb +138 -0
- data/lib/datafy/CSVHarvester.rb +528 -0
- data/lib/datafy/FileProvider.rb +109 -0
- data/lib/datafy.rb +9 -0
- metadata +48 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: a444f118259af2c44e08b0f8a9b12922162219ab41576fc8a0a73c232667c02b
|
4
|
+
data.tar.gz: 6089e37cb5a274c7953e17126cea75bc7e6cc8b77427b8063c56b765e1e8c19d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 55ce44b9c30c29feed0d09fcdc8c8bb06fd4b568f87cc3ce802b146d812f92e1d85f4b7f9c31055e9255074507a6a3c39832b18727f37d2508d61f7fe3a724d6
|
7
|
+
data.tar.gz: 110b9d262147d5c117f5bedb818ca4f46f5b80de9414854080531de8b9e6221c06abeb4304f1134da5a9d028ff386264248fc53dfe0d63aa0b800013aa54ae77
|
@@ -0,0 +1,138 @@
|
|
1
|
+
# print "Executing: #{__FILE__}"
|
2
|
+
|
3
|
+
require 'date'
|
4
|
+
require 'logger'
|
5
|
+
require_relative 'fileprovider'
|
6
|
+
|
7
|
+
module BaseHarvester
|
8
|
+
|
9
|
+
include FileProvider
|
10
|
+
|
11
|
+
# attr_accessor :errorMessage, :logFile, :logFileName, :showLog
|
12
|
+
|
13
|
+
class HarvestError < StandardError
|
14
|
+
attr_reader :message, :source
|
15
|
+
def initialize(msg="default message", thing="BaseHarvester")
|
16
|
+
@message = msg
|
17
|
+
@thing = thing
|
18
|
+
super(msg)
|
19
|
+
end
|
20
|
+
end # class HarvestError
|
21
|
+
|
22
|
+
# ================== Data Fields Section ==================
|
23
|
+
|
24
|
+
v, $VERBOSE = $VERBOSE, nil
|
25
|
+
BASE_HARVEST_DATA_FIELDS = {
|
26
|
+
'Harvest When' => { property: :harvestWhen, comment: 'Date/Time of data harvesting' },
|
27
|
+
'Harvest Successful?' => { property: :harvestSuccessful, comment: 'Boolean - was harvesting successful?' },
|
28
|
+
'Harvest Message' => { property: :harvestMessage, comment: 'Usually indicates harvesting problem' },
|
29
|
+
'Persist When' => { property: :persistWhen, comment: 'Date/Time of data persistence to CSV file' },
|
30
|
+
}
|
31
|
+
# --
|
32
|
+
DETAILS_PROPERTY_FIELDS = {
|
33
|
+
'Class' => { property: :klass, comment: 'The class the details are being recorded for. (Base)' },
|
34
|
+
'Detail' => { property: :detail, comment: 'The name of the field being recorded.' },
|
35
|
+
'Index' => { property: :index, comment: 'Recording sequence, i.e. 1: first, 2: second, etc.' },
|
36
|
+
'Line' => { property: :line, comment: 'The specific detail.' },
|
37
|
+
}
|
38
|
+
$VERBOSE = v
|
39
|
+
|
40
|
+
class << self
|
41
|
+
def included(base)
|
42
|
+
base.extend DataFieldsClassMethods
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
module DataFieldsClassMethods
|
47
|
+
|
48
|
+
def logger
|
49
|
+
@logger ||= Logger.new(logFileName,5,1024000)
|
50
|
+
end
|
51
|
+
|
52
|
+
def logFileName
|
53
|
+
@logFileName ||= FileProvider.getDataFileName("#{self.class}.log")
|
54
|
+
end
|
55
|
+
|
56
|
+
def attr_fields fields
|
57
|
+
logger.debug "#{self.class}::#{__method__}"
|
58
|
+
logger.debug "fields: #{fields}"
|
59
|
+
return if fields.nil?
|
60
|
+
props = properties
|
61
|
+
fields.each do |name,hash|
|
62
|
+
prop = hash[:property]
|
63
|
+
props[prop] = name
|
64
|
+
self.instance_eval do
|
65
|
+
attr_accessor hash[:property].to_s
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def prop_accessor *prop_groups
|
71
|
+
logger.debug "#{self}::#{__method__}"
|
72
|
+
logger.debug *prop_groups.inspect
|
73
|
+
prop_groups.each do |props|
|
74
|
+
logger.debug props
|
75
|
+
attr_fields props
|
76
|
+
register_fields props
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def properties
|
81
|
+
@properties ||= Hash.new
|
82
|
+
end
|
83
|
+
|
84
|
+
def register_fields fields
|
85
|
+
logger.debug "#{self.class}::#{__method__} fields: #{fields}"
|
86
|
+
return if fields.nil?
|
87
|
+
@data_fields = {} if @data_fields.nil?
|
88
|
+
@data_fields.merge!(fields) unless fields.nil?
|
89
|
+
@data_fields.merge!(BASE_HARVEST_DATA_FIELDS)
|
90
|
+
attr_fields fields
|
91
|
+
end
|
92
|
+
|
93
|
+
# def register_details_fields fields
|
94
|
+
# return if fields.nil?
|
95
|
+
# @details_fields = {} if @details_fields.nil?
|
96
|
+
# @details_fields.merge!(fields)
|
97
|
+
# @details_fields.merge!(BASE_HARVEST_DATA_FIELDS)
|
98
|
+
# attr_fields @details_fields
|
99
|
+
# end
|
100
|
+
|
101
|
+
def details_fields
|
102
|
+
# puts "#{self.class}::#{__method__}"
|
103
|
+
if @details_fields.nil?
|
104
|
+
register_fields(nil)
|
105
|
+
end
|
106
|
+
@details_fields
|
107
|
+
end
|
108
|
+
|
109
|
+
end # module DataFieldsClassMethods
|
110
|
+
|
111
|
+
def logger
|
112
|
+
self.class.logger
|
113
|
+
end
|
114
|
+
|
115
|
+
def data_fields
|
116
|
+
self.class.data_fields
|
117
|
+
end
|
118
|
+
|
119
|
+
def details_fields
|
120
|
+
self.class.details_fields
|
121
|
+
end
|
122
|
+
|
123
|
+
def now
|
124
|
+
DateTime.now.strftime('%l:%M:%S%P %b %-d, %Y ').strip
|
125
|
+
end
|
126
|
+
|
127
|
+
def flattenTextLines lines
|
128
|
+
lines.is_a?(Enumerable) ? lines.join(' ') : lines.to_s
|
129
|
+
end
|
130
|
+
|
131
|
+
def numify str
|
132
|
+
number = str.is_a?(String) ? str : str.to_s
|
133
|
+
number.gsub!(/[^0-9]/,'')
|
134
|
+
num_groups = number.to_s.chars.to_a.reverse.each_slice(3)
|
135
|
+
num_groups.map(&:join).join(',').reverse
|
136
|
+
end
|
137
|
+
|
138
|
+
end # module BaseHarvester
|
@@ -0,0 +1,528 @@
|
|
1
|
+
require 'set'
|
2
|
+
require 'csv'
|
3
|
+
require_relative 'baseharvester'
|
4
|
+
|
5
|
+
module CSVHarvester
|
6
|
+
|
7
|
+
include BaseHarvester
|
8
|
+
include FileProvider
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def included(base)
|
12
|
+
base.extend CSVHarvesterClassMethods
|
13
|
+
end
|
14
|
+
end # class << self
|
15
|
+
|
16
|
+
module CSVHarvesterClassMethods
|
17
|
+
|
18
|
+
def logger
|
19
|
+
@logger ||= Logger.new(logFileName,5,1024000)
|
20
|
+
end
|
21
|
+
|
22
|
+
def logFileName
|
23
|
+
@logFileName ||= "#{FileProvider.logDir}/#{self}.log"
|
24
|
+
end
|
25
|
+
|
26
|
+
def data_fields
|
27
|
+
# @data_fields ||= KEY_FIELDS.merge( PROPERTY_FIELDS, BaseHarvester::BASE_HARVEST_DATA_FIELDS )
|
28
|
+
@data_fields ||= register_fields :data
|
29
|
+
end
|
30
|
+
|
31
|
+
def set_data_fields fields
|
32
|
+
@data_fields = fields
|
33
|
+
end
|
34
|
+
|
35
|
+
def details_fields
|
36
|
+
# @details_fields ||= KEY_FIELD.merge( BaseHarvester::DETAILS_PROPERTY_FIELDS, BaseHarvester::BASE_HARVEST_DATA_FIELDS )
|
37
|
+
@details_fields ||= register_fields :details
|
38
|
+
end
|
39
|
+
|
40
|
+
def details_fields= fields
|
41
|
+
@details_fields = fields
|
42
|
+
end
|
43
|
+
|
44
|
+
def set_details_fields fields
|
45
|
+
@details_fields = fields
|
46
|
+
end
|
47
|
+
|
48
|
+
def attr_fields fields
|
49
|
+
logger.debug "#{self.class}::#{__method__}"
|
50
|
+
logger.debug "fields: #{fields}"
|
51
|
+
return if fields.nil?
|
52
|
+
props = properties
|
53
|
+
fields.each do |name,hash|
|
54
|
+
prop = hash[:property]
|
55
|
+
props[prop] = name
|
56
|
+
self.instance_eval do
|
57
|
+
attr_accessor hash[:property].to_s
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def properties
|
63
|
+
@properties ||= Hash.new
|
64
|
+
end
|
65
|
+
|
66
|
+
def prop_accessor *prop_groups
|
67
|
+
logger.debug "#{self}::#{__method__}"
|
68
|
+
logger.debug *prop_groups.inspect
|
69
|
+
prop_groups.each do |props|
|
70
|
+
logger.debug props
|
71
|
+
attr_fields props
|
72
|
+
end
|
73
|
+
register_fields
|
74
|
+
end
|
75
|
+
|
76
|
+
def register_fields type=:data
|
77
|
+
logger.debug "#{self}::#{__method__}"
|
78
|
+
@data_fields = self.const_defined?(:KEY_FIELDS) ? self::KEY_FIELDS.clone : {}
|
79
|
+
@details_fields = @data_fields.clone
|
80
|
+
# --
|
81
|
+
data_props = self.const_defined?(:PROPERTY_FIELDS) ? self::PROPERTY_FIELDS : {}
|
82
|
+
@data_fields.merge!( data_props, BaseHarvester::BASE_HARVEST_DATA_FIELDS )
|
83
|
+
@details_fields.merge!( BaseHarvester::DETAILS_PROPERTY_FIELDS, BaseHarvester::BASE_HARVEST_DATA_FIELDS )
|
84
|
+
attr_fields @data_fields
|
85
|
+
attr_fields @details_fields
|
86
|
+
return type.eql?(:data) ? @data_fields : @details_fields
|
87
|
+
end
|
88
|
+
|
89
|
+
def printProperties
|
90
|
+
logger.debug "#{self}::#{__method__}"
|
91
|
+
if properties.empty?
|
92
|
+
puts "no properties to print"
|
93
|
+
return
|
94
|
+
end
|
95
|
+
maxLen = properties.keys.max_by(&:length).length
|
96
|
+
maxLen += 2
|
97
|
+
properties.each do |f,v|
|
98
|
+
puts " %-#{maxLen}s -> '%s' " % [f.inspect,v]
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def print_fields type=:data
|
103
|
+
logger.debug "#{self}::#{__method__}"
|
104
|
+
label = "#{self} #{type.capitalize} Fields"
|
105
|
+
puts label
|
106
|
+
puts '-' * label.length
|
107
|
+
fields = type.eql?(:details) ? details_fields : data_fields
|
108
|
+
maxKeyLen = fields.keys.max_by(&:length).length
|
109
|
+
maxSynLen = fields.values.map{|x| x[:property]}.max_by(&:length).length + 2
|
110
|
+
comments = fields.values.map{|x| x[:comment]}.compact
|
111
|
+
maxComLen = comments.empty? ? "Comment".length : comments.max_by(&:length).length
|
112
|
+
logger.debug " class: #{self}\n #{type} Fields [CSV]"
|
113
|
+
logger.debug " %-#{maxKeyLen}s -> %-#{maxSynLen}s %-#{maxComLen}s" % ['Field Name', 'Symbol', "Comment"]
|
114
|
+
logger.debug " %-#{maxKeyLen}s %-#{maxSynLen}s %-#{maxComLen}s" % ['-' * maxKeyLen, '-' * maxSynLen, '-' * maxComLen ]
|
115
|
+
# puts '-' * (maxKeyLen +3)
|
116
|
+
fields.each do |f,v|
|
117
|
+
puts " %-#{maxKeyLen}s -> %-#{maxSynLen}s %s" % [f,":#{v[:property]}",v[:comment]]
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def printFields *args
|
122
|
+
logger.debug "#{self}::#{__method__} -c- args: #{args} empty? #{args.empty?}"
|
123
|
+
if args.empty?
|
124
|
+
print_fields :data
|
125
|
+
print_fields :details
|
126
|
+
return
|
127
|
+
end
|
128
|
+
args.each do |type|
|
129
|
+
case type.to_s.downcase
|
130
|
+
when 'data' then print_fields :data
|
131
|
+
when 'details' then print_fields :details
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
def printDataFields
|
137
|
+
print_fields :data
|
138
|
+
end
|
139
|
+
|
140
|
+
def printDetailsFields
|
141
|
+
print_fields :details
|
142
|
+
end
|
143
|
+
|
144
|
+
def getCSVFiles label=nil
|
145
|
+
logger.debug "#{self}::#{__method__}"
|
146
|
+
fqname = FileProvider.getDataFileName("")
|
147
|
+
logger.debug fqname
|
148
|
+
puts fqname
|
149
|
+
raise HarvestError.new "ERROR: method '#{self}::#{__method__}' not implemented source: CSVHarvester"
|
150
|
+
end
|
151
|
+
def prep_csv_file name, fields
|
152
|
+
logger.debug "#{self.class}::#{__method__} name:'#{name}' fields:#{fields}"
|
153
|
+
file = nil
|
154
|
+
if File.file?(name)
|
155
|
+
logger.debug 'Data file already exists'
|
156
|
+
FileProvider.backupFile(name)
|
157
|
+
fileFields = CSV.open(name, &:readline)
|
158
|
+
if fields == fileFields
|
159
|
+
logger.debug 'headers match'
|
160
|
+
file = CSV.open(name,'a', :headers => true)
|
161
|
+
else
|
162
|
+
logger.debug 'headers DO NOT match'
|
163
|
+
file = CSV.open(name,'w')
|
164
|
+
file << fields
|
165
|
+
end
|
166
|
+
else
|
167
|
+
logger.debug 'Data file DOES NOT already exist'
|
168
|
+
puts 'Data file DOES NOT already exist'
|
169
|
+
file = CSV.open(name,'w')
|
170
|
+
file << fields
|
171
|
+
end
|
172
|
+
return file
|
173
|
+
end
|
174
|
+
|
175
|
+
def baseFileName
|
176
|
+
logger.debug "#{self.class}::#{__method__}"
|
177
|
+
"#{self}".strip
|
178
|
+
end
|
179
|
+
|
180
|
+
def csv_files
|
181
|
+
@csv_files ||= loadCSVFiles
|
182
|
+
end
|
183
|
+
|
184
|
+
def csv_file type=:data
|
185
|
+
csv_files[type]
|
186
|
+
end
|
187
|
+
|
188
|
+
def loadCSVFiles type=:data
|
189
|
+
logger.debug "#{self.class}::#{__method__} type:'#{type}'"
|
190
|
+
name_data = FileProvider.getDataFileName("#{baseFileName} Data.csv")
|
191
|
+
name_details = FileProvider.getDataFileName("#{baseFileName} Details.csv")
|
192
|
+
logger.debug "base_name : #{baseFileName} "
|
193
|
+
logger.debug "name_data : #{name_data}"
|
194
|
+
logger.debug "name_details: #{name_details}"
|
195
|
+
unless @csv_files.nil?
|
196
|
+
@csv_files.each do |type,file|
|
197
|
+
begin
|
198
|
+
file.close
|
199
|
+
rescue => e
|
200
|
+
logger.error "Closing CSV file #{type} | #{file} failed"
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
@csv_files = {}
|
205
|
+
@csv_files[:data] = prep_csv_file(name_data, data_fields.keys)
|
206
|
+
@csv_files[:details] = prep_csv_file(name_details, details_fields.keys)
|
207
|
+
data_msg = "CSV file with data as Fields : #{name_data}"
|
208
|
+
flds_msg = "CSV file with data as Keys/Values: #{name_details}"
|
209
|
+
logger.debug data_msg
|
210
|
+
logger.debug flds_msg
|
211
|
+
return @csv_files
|
212
|
+
end
|
213
|
+
|
214
|
+
def printDataFiles show_fields=true
|
215
|
+
logger.debug "#{self.class}::#{__method__} show_fields:#{show_fields}"
|
216
|
+
puts "CSV Files\n---------"
|
217
|
+
type_len = show_fields ? 4 : 7
|
218
|
+
csv_files.each do |type,file|
|
219
|
+
puts "%#{type_len}s file -> %-s" % [type.to_s.capitalize,file.path]
|
220
|
+
if show_fields
|
221
|
+
printFields type
|
222
|
+
puts ''
|
223
|
+
end
|
224
|
+
end
|
225
|
+
puts "---------"
|
226
|
+
end
|
227
|
+
|
228
|
+
def closeCSVFiles
|
229
|
+
unless @csv_files.nil?
|
230
|
+
@csv_files.each do |type,file|
|
231
|
+
begin
|
232
|
+
file.close
|
233
|
+
rescue => e
|
234
|
+
logger.warn "Closing CSV file #{type} | #{file} failed"
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
@csv_files = nil
|
239
|
+
end
|
240
|
+
|
241
|
+
def persistWhen
|
242
|
+
@persistWhen ||= DateTime.now.strftime('%l:%M:%S%P %b %-d, %Y ').strip
|
243
|
+
end
|
244
|
+
def persistWhen= persist_when=now
|
245
|
+
@persistWhen = persist_when
|
246
|
+
end
|
247
|
+
|
248
|
+
end # module CSVHarvesterClassMethods
|
249
|
+
|
250
|
+
def logger
|
251
|
+
# puts "#{self.class}::#{__method__}"
|
252
|
+
self.class.logger
|
253
|
+
end
|
254
|
+
|
255
|
+
def logFileName
|
256
|
+
self.class.logFileName
|
257
|
+
end
|
258
|
+
|
259
|
+
def data_fields
|
260
|
+
self.class.data_fields
|
261
|
+
end
|
262
|
+
|
263
|
+
def details_fields
|
264
|
+
self.class.details_fields
|
265
|
+
end
|
266
|
+
|
267
|
+
def harvest
|
268
|
+
harvestNow
|
269
|
+
end
|
270
|
+
|
271
|
+
def harvestNow
|
272
|
+
record( property: :harvestWhen, value: now, append: false )
|
273
|
+
return now
|
274
|
+
end
|
275
|
+
|
276
|
+
def persistWhen
|
277
|
+
# persist_when = self.class.persistWhen
|
278
|
+
record( property: :persistWhen, value: now, append: false )
|
279
|
+
return now
|
280
|
+
end
|
281
|
+
|
282
|
+
def printDataFields
|
283
|
+
self.class.printFields :data
|
284
|
+
end
|
285
|
+
|
286
|
+
def printFields *args
|
287
|
+
# puts "#{self}::#{__method__} args: -i- #{args}"
|
288
|
+
self.class.printFields *args
|
289
|
+
end
|
290
|
+
|
291
|
+
def properties
|
292
|
+
logger.debug "#{self.class}::#{__method__} [i]"
|
293
|
+
self.class.properties
|
294
|
+
end
|
295
|
+
|
296
|
+
def printProperties
|
297
|
+
logger.debug "#{self.class}::#{__method__} [i]"
|
298
|
+
self.class.printProperties
|
299
|
+
end
|
300
|
+
|
301
|
+
def printDetailsFields
|
302
|
+
self.class.printFields :details
|
303
|
+
end
|
304
|
+
|
305
|
+
def record(property:, value:, append: true)
|
306
|
+
logger.debug "#{self.class}::#{__method__}"
|
307
|
+
begin
|
308
|
+
if ''.eql?(value) || value.nil?
|
309
|
+
logger.debug " !!! NOT setting nil or empty property"
|
310
|
+
return
|
311
|
+
end
|
312
|
+
prop_name = property.is_a?(String) ? property : property.to_s
|
313
|
+
prop_name.sub!(/[=]+$/,'')
|
314
|
+
curr_val = send(prop_name)
|
315
|
+
assign_prop = prop_name.concat('=')
|
316
|
+
assign_val = if append && !curr_val.nil? && !value.eql?(curr_val)
|
317
|
+
"#{curr_val} | #{value}"
|
318
|
+
else
|
319
|
+
value
|
320
|
+
end
|
321
|
+
send(assign_prop, assign_val)
|
322
|
+
register_detail(property: property, value: value)
|
323
|
+
rescue => e
|
324
|
+
puts "#{self.class}::#{__method__} ERROR:#{e}"
|
325
|
+
end
|
326
|
+
end
|
327
|
+
|
328
|
+
def resetDataFields
|
329
|
+
data_fields.each do |fieldName,parts|
|
330
|
+
methodName = "#{parts[:property].to_s}="
|
331
|
+
methodName.gsub!(/[=]+$/,'=')
|
332
|
+
send(methodName,nil)
|
333
|
+
end
|
334
|
+
@errorMessage = nil
|
335
|
+
end
|
336
|
+
|
337
|
+
def resetDetails
|
338
|
+
logger.debug "#{self.class}::#{__method__}"
|
339
|
+
@details = Hash.new { |k,v| k[v] = Set.new }
|
340
|
+
@detailsCSVRecords = nil
|
341
|
+
logger.debug "Details : #{@details.inspect}"
|
342
|
+
logger.debug "CSV recs: #{@detailsCSVRecords.inspect}"
|
343
|
+
end
|
344
|
+
|
345
|
+
def registerError error
|
346
|
+
logger.debug "#{self.class}::#{__method__}"
|
347
|
+
record( property: :harvestSuccessful, value: false )
|
348
|
+
record( property: :harvestMessage, value: error.message[0..150].gsub("\n", ' | ') )
|
349
|
+
@errorMessage = error.full_message.to_s # @errorMessage declared in BaseHarvester, useful for debugging
|
350
|
+
end
|
351
|
+
|
352
|
+
def csvRecord
|
353
|
+
logger.debug "#{self.class}::#{__method__}"
|
354
|
+
rec = CSV::Row.new( self.data_fields.keys, [] )
|
355
|
+
self.data_fields.each do |fieldName,parts|
|
356
|
+
value = send(parts[:property])
|
357
|
+
logger.debug "fieldName: '#{fieldName}' => :#{parts[:property]} ==> #{value} nil?#{value.nil?}"
|
358
|
+
rec[fieldName] = value
|
359
|
+
end
|
360
|
+
return rec
|
361
|
+
end
|
362
|
+
|
363
|
+
def printCSVRecord
|
364
|
+
logger.debug "#{self.class}::#{__method__}"
|
365
|
+
rec = csvRecord
|
366
|
+
maxLen = rec.headers.max_by(&:length).length
|
367
|
+
puts "CSV Record"
|
368
|
+
puts '=' * (maxLen)
|
369
|
+
unless rec.nil? || rec.empty?
|
370
|
+
rec.each do |f,v|
|
371
|
+
# logger.info " %-#{maxLen}s -> %s " % [f,v]
|
372
|
+
puts "%-#{maxLen}s : %s " % [f,v]
|
373
|
+
end
|
374
|
+
puts '-' * maxLen
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
378
|
+
def details
|
379
|
+
@details ||= Hash.new { |k,v| k[v] = Set.new }
|
380
|
+
end
|
381
|
+
|
382
|
+
# Use this method to only add a Details line, it does not set the property
|
383
|
+
def register_detail(property:, value:)
|
384
|
+
logger.debug "#{self.class}::#{__method__} prop:'#{property.inspect}' value:'#{value}'"
|
385
|
+
return if value.nil? || ''.eql?(value)
|
386
|
+
detail = properties.has_key?([property]) ? properties[property] : property
|
387
|
+
deets = details
|
388
|
+
logger.debug " - details has new? #{deets[detail].include?(value)}"
|
389
|
+
unless (deets.has_key?(detail) && deets[detail].include?(value))
|
390
|
+
deets[detail] << value.to_s.strip # .to_s handles cases of non-String value
|
391
|
+
end
|
392
|
+
end
|
393
|
+
|
394
|
+
def printDetails
|
395
|
+
logger.debug "#{self.class}::#{__method__}"
|
396
|
+
if details.empty?
|
397
|
+
msg = " - no details to print i.e. @details is nil or empty"
|
398
|
+
puts msg
|
399
|
+
logger.info msg
|
400
|
+
else
|
401
|
+
# maxKeyLen = properties.values.max_by(&:length).length
|
402
|
+
details.each do |type,lines|
|
403
|
+
# log true, type
|
404
|
+
# puts "%-#{maxKeyLen}s :: [%s]" % [properties[type],type]
|
405
|
+
puts "%s [%s]" % [properties[type],type]
|
406
|
+
lines.each do |line|
|
407
|
+
puts " - '#{line}'"
|
408
|
+
end
|
409
|
+
end
|
410
|
+
end
|
411
|
+
end #def printDetails
|
412
|
+
|
413
|
+
def detailsCSVRecords
|
414
|
+
logger.debug "#{self.class}::#{__method__}"
|
415
|
+
@detailsCSVRecords = Set.new
|
416
|
+
begin
|
417
|
+
details.each do |detail,lines|
|
418
|
+
@detail = detail
|
419
|
+
field = properties[detail]
|
420
|
+
lines.each_with_index do |line,index|
|
421
|
+
rec = CSV::Row.new( details_fields.keys, [] )
|
422
|
+
# rec = CSV::Row.new( detailsDataFields.keys, [] )
|
423
|
+
# DETAILS_DATA_FIELDS.each do |fieldName,parts|
|
424
|
+
details_fields.each do |fieldName,parts|
|
425
|
+
prop = parts[:property]
|
426
|
+
resp = respond_to?(prop)
|
427
|
+
if resp
|
428
|
+
value = send(prop)
|
429
|
+
rec[fieldName] = value
|
430
|
+
end
|
431
|
+
end
|
432
|
+
rec['Class'] = self.class # klass
|
433
|
+
rec['Detail'] = field
|
434
|
+
rec['Index'] = index + 1
|
435
|
+
rec['Line'] = line
|
436
|
+
rec['Harvest When'] = @harvestWhen.nil? ? now : detail.eql?(:harvestWhen) ? now : harvestWhen
|
437
|
+
rec['Persist When'] = @persistWhen.nil? ? now : persistWhen
|
438
|
+
@detailsCSVRecords << rec
|
439
|
+
end
|
440
|
+
end
|
441
|
+
rescue => e
|
442
|
+
registerError e
|
443
|
+
end
|
444
|
+
return @detailsCSVRecords
|
445
|
+
end
|
446
|
+
|
447
|
+
def printDataFiles show_fields=true
|
448
|
+
logger.debug "#{self.class}::#{__method__} show_fields:#{show_fields}"
|
449
|
+
self.class.printDataFiles show_fields
|
450
|
+
end
|
451
|
+
|
452
|
+
def printDetailsCSVRecords
|
453
|
+
logger.debug "#{self.class}::#{__method__}"
|
454
|
+
begin
|
455
|
+
puts "Details Records"
|
456
|
+
recs = detailsCSVRecords
|
457
|
+
if recs.nil? || recs.empty?
|
458
|
+
# log true, "No Details records exist to print."
|
459
|
+
puts " - no details records exist to print"
|
460
|
+
else
|
461
|
+
maxLen = recs.first.headers.max_by(&:length).length
|
462
|
+
puts '=' * maxLen
|
463
|
+
recs.each do |r|
|
464
|
+
r.entries.each do |e,v|
|
465
|
+
puts "%-#{maxLen}s : %s" % [e,v]
|
466
|
+
end
|
467
|
+
puts '-' * maxLen
|
468
|
+
end
|
469
|
+
end
|
470
|
+
rescue => e
|
471
|
+
registerError e
|
472
|
+
end
|
473
|
+
end
|
474
|
+
|
475
|
+
def persist_data
|
476
|
+
logger.debug "#{self.class}::#{__method__}"
|
477
|
+
begin
|
478
|
+
# @persistWhen = now
|
479
|
+
persistWhen
|
480
|
+
file = self.class.csv_files[:data]
|
481
|
+
$fields_file = file
|
482
|
+
unless file.nil?
|
483
|
+
file << csvRecord
|
484
|
+
end
|
485
|
+
rescue => e
|
486
|
+
logger.error "ERROR persisting CSV fields aka csvRecord:'#{e.message}'"
|
487
|
+
end
|
488
|
+
end
|
489
|
+
|
490
|
+
def persist_details
|
491
|
+
logger.debug "#{self.class}::#{__method__}"
|
492
|
+
begin
|
493
|
+
file = self.class.csv_files[:details]
|
494
|
+
$details_file = file
|
495
|
+
detailsCSVRecords.each do |rec|
|
496
|
+
file << rec
|
497
|
+
end
|
498
|
+
return
|
499
|
+
rescue => e
|
500
|
+
logger.error "ERROR persisting CSV data in details form:'#{e.message}'"
|
501
|
+
end
|
502
|
+
end
|
503
|
+
|
504
|
+
def persist(*args, persist_when: nil, append: true, close_on_exit: true)
|
505
|
+
logger.debug "#{self.class}::#{__method__} args:#{args}"
|
506
|
+
record( property: :persistWhen, value: persist_when.nil? ? now : persist_when, append: append )
|
507
|
+
persist_data
|
508
|
+
persist_details
|
509
|
+
self.class.closeCSVFiles if close_on_exit
|
510
|
+
end
|
511
|
+
|
512
|
+
def to_s
|
513
|
+
str = "#{self.class}:: "
|
514
|
+
consts = self.class.constants.grep(/KEY/)
|
515
|
+
consts.each do |c|
|
516
|
+
fields = self.class.const_get(c)
|
517
|
+
fields.each do |name,prop|
|
518
|
+
str+= " -#{name}-|#{send(prop[:property])}|"
|
519
|
+
end
|
520
|
+
# end
|
521
|
+
# key = key_field.first
|
522
|
+
# val = send(key_field.last[:property])
|
523
|
+
# str = "#{self.class}: '#{key}':'#{val}'"
|
524
|
+
end
|
525
|
+
return str
|
526
|
+
end
|
527
|
+
|
528
|
+
end # module CSVHarvester
|
@@ -0,0 +1,109 @@
|
|
1
|
+
# puts "Executing: #{__FILE__}"
|
2
|
+
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module FileProvider
|
6
|
+
|
7
|
+
def self.localDir
|
8
|
+
@local_dir ||= Dir.pwd
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.localDir= dir
|
12
|
+
@local_dir = dir
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.dataDir
|
16
|
+
@data_dir ||= getDataDir
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.getDataDir
|
20
|
+
getDir 'data'
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.logDir
|
24
|
+
@log_dir ||= getLogDir
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.getLogDir
|
28
|
+
getDir 'logs'
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.getDir dir='data'
|
32
|
+
dir_name = "#{localDir}/#{dir}".gsub(/[\/]+/,'/')
|
33
|
+
unless Dir.exists?(dir_name)
|
34
|
+
Dir.mkdir(dir_name)
|
35
|
+
end
|
36
|
+
return dir_name
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.dataDir= dir
|
40
|
+
@data_dir = dir
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.logDir= dir
|
44
|
+
@log_dir = dir
|
45
|
+
end
|
46
|
+
|
47
|
+
attr_reader :file_name
|
48
|
+
|
49
|
+
def self.getDataFileName fileName
|
50
|
+
normFileName = fileName.split('/').last
|
51
|
+
file_name = "#{dataDir}/#{normFileName}"
|
52
|
+
return file_name
|
53
|
+
end
|
54
|
+
|
55
|
+
def dataTextFile fileName, mode='a'
|
56
|
+
fqname = getDataFileName(fileName)
|
57
|
+
file = nil
|
58
|
+
begin
|
59
|
+
file = File.open(fqname,mode)
|
60
|
+
rescue => e
|
61
|
+
puts "FileProvider.dataTextFile(#{fileName}) problem: #{e.message}"
|
62
|
+
file = nil
|
63
|
+
end
|
64
|
+
# return HarvesterFile.new fileName, fqname, file
|
65
|
+
return file
|
66
|
+
end
|
67
|
+
|
68
|
+
def openLogFile fileName="#{self.class.to_s}_log.txt"
|
69
|
+
fqname = getDataFileName(fileName)
|
70
|
+
@logFile = File.open(fqname, 'a+')
|
71
|
+
end # def openLogFile
|
72
|
+
|
73
|
+
def removeFile fileName
|
74
|
+
fqname = getDataFileName(fileName)
|
75
|
+
status = ''
|
76
|
+
if File.file?(fqname)
|
77
|
+
# puts 'Data file already exists'
|
78
|
+
backupFile(fqname)
|
79
|
+
# puts "removing data file: '#{fileName}' -> '#{fqname}'"
|
80
|
+
result = system "rm '#{fqname}'"
|
81
|
+
status = result.is_a?(TrueClass) ? 'Success' : 'Failue'
|
82
|
+
else
|
83
|
+
status = 'File not found'
|
84
|
+
end
|
85
|
+
return status
|
86
|
+
end
|
87
|
+
|
88
|
+
def self.backupFile fileName
|
89
|
+
status = nil
|
90
|
+
if File.file?(fileName)
|
91
|
+
backupName = getBackupFileName fileName
|
92
|
+
FileUtils.cp(fileName,backupName)
|
93
|
+
status = { status: :backedUp, name: backupName }
|
94
|
+
else
|
95
|
+
status = { status: :originalFileNotFound }
|
96
|
+
end
|
97
|
+
return status
|
98
|
+
end
|
99
|
+
|
100
|
+
def backupFile fileName
|
101
|
+
self.class.backupFile fileName
|
102
|
+
end
|
103
|
+
|
104
|
+
def self.getBackupFileName fileName
|
105
|
+
fqname = getDataFileName fileName
|
106
|
+
"#{fileName}.#{DateTime.now}"
|
107
|
+
end
|
108
|
+
|
109
|
+
end # module FileProvider
|
data/lib/datafy.rb
ADDED
metadata
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: datafy
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.5.8
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Chris Gerrard
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-03-27 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Represent, record, and persist Ruby classes' attributes as CSV fields
|
14
|
+
and persist them as data fields into CSV files.
|
15
|
+
email:
|
16
|
+
- Chris@Gerrard.net
|
17
|
+
executables: []
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- lib/datafy.rb
|
22
|
+
- lib/datafy/BaseHarvester.rb
|
23
|
+
- lib/datafy/CSVHarvester.rb
|
24
|
+
- lib/datafy/FileProvider.rb
|
25
|
+
homepage: ''
|
26
|
+
licenses:
|
27
|
+
- MIT
|
28
|
+
metadata: {}
|
29
|
+
post_install_message:
|
30
|
+
rdoc_options: []
|
31
|
+
require_paths:
|
32
|
+
- lib
|
33
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
34
|
+
requirements:
|
35
|
+
- - ">="
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 3.1.2
|
38
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '0'
|
43
|
+
requirements: []
|
44
|
+
rubygems_version: 3.3.11
|
45
|
+
signing_key:
|
46
|
+
specification_version: 4
|
47
|
+
summary: Record and persist data as CSV fields.
|
48
|
+
test_files: []
|