filewatch-ext-excel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/JRubyFileExtension.jar +0 -0
- data/lib/filewatch/ext/filetail.rb +135 -0
- data/lib/filewatch/ext/tailbase.rb +189 -0
- data/lib/filewatch/ext/xlstail.rb +153 -0
- data/lib/filewatch/ext/xlsxtail.rb +151 -0
- data/lib/filewatch/winhelper.rb +70 -0
- metadata +93 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b10b9266fcaab0d8f520febe99e45846fd553664
|
4
|
+
data.tar.gz: 8101e8737b30f3ec592a1484d71921e621eab2ee
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 40c131395cd9ceb2fcc071b9589a6e000b6e2357bc1ef6cc1e28a712661ab74d3cb28898c20f9e458a86288d7183e1b5b77ff7e5319d625f682aeee42f5b11ca
|
7
|
+
data.tar.gz: 98661911c4a780638ea406d8d9622bf4192fad22c0d7e216b3b3dfe88cbb91044790d292bd7be83e29eb6566eefad4a963dc7c408d76025c2ddac27fb23622a9
|
Binary file
|
@@ -0,0 +1,135 @@
|
|
1
|
+
require "filewatch/ext/tailbase"
|
2
|
+
require "filewatch/buftok"
|
3
|
+
|
4
|
+
module FileWatch::Ext
|
5
|
+
class FileTail < FileWatch::Ext::TailBase
|
6
|
+
|
7
|
+
public
|
8
|
+
def subscribe(&block)
|
9
|
+
# subscribe(stat_interval = 1, discover_interval = 5, &block)
|
10
|
+
@watch.subscribe(@opts[:stat_interval],
|
11
|
+
@opts[:discover_interval]) do |event, path|
|
12
|
+
case event
|
13
|
+
when :create, :create_initial
|
14
|
+
if @files.member?(path)
|
15
|
+
@logger.debug("#{event} for #{path}: already exists in @files")
|
16
|
+
next
|
17
|
+
end
|
18
|
+
if _open_file(path, event)
|
19
|
+
_read_file(path, &block)
|
20
|
+
end
|
21
|
+
when :modify
|
22
|
+
_open_file(path, event) if @opts[:eof_close]
|
23
|
+
|
24
|
+
if !@files.member?(path)
|
25
|
+
@logger.debug(":modify for #{path}, does not exist in @files")
|
26
|
+
if _open_file(path, event)
|
27
|
+
_read_file(path, &block)
|
28
|
+
end
|
29
|
+
else
|
30
|
+
_read_file(path, &block)
|
31
|
+
end
|
32
|
+
when :delete
|
33
|
+
@logger.debug(":delete for #{path}, deleted from @files")
|
34
|
+
|
35
|
+
_read_file(path, &block) if !@opts[:eof_close]
|
36
|
+
_progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
|
37
|
+
|
38
|
+
@files[path].close if !@opts[:eof_close]
|
39
|
+
@files.delete(path)
|
40
|
+
_sincedb_delete(path)
|
41
|
+
@statcache.delete(path)
|
42
|
+
else
|
43
|
+
@logger.warn("unknown event type #{event} for #{path}")
|
44
|
+
end
|
45
|
+
end # @watch.subscribe
|
46
|
+
end # def each
|
47
|
+
|
48
|
+
private
|
49
|
+
def _open_file(path, event)
|
50
|
+
@logger.debug("_open_file: #{path}: opening")
|
51
|
+
begin
|
52
|
+
if @iswindows && defined? JRUBY_VERSION
|
53
|
+
@files[path] = Java::RubyFileExt::getRubyFile(path)
|
54
|
+
else
|
55
|
+
@files[path] = File.open(path)
|
56
|
+
end
|
57
|
+
rescue
|
58
|
+
# don't emit this message too often. if a file that we can't
|
59
|
+
# read is changing a lot, we'll try to open it more often,
|
60
|
+
# and might be spammy.
|
61
|
+
now = Time.now.to_i
|
62
|
+
if now - @lastwarn[path] > OPEN_WARN_INTERVAL
|
63
|
+
@logger.warn("failed to open #{path}: #{$!}")
|
64
|
+
@lastwarn[path] = now
|
65
|
+
else
|
66
|
+
@logger.debug("(warn supressed) failed to open #{path}: #{$!}")
|
67
|
+
end
|
68
|
+
@files.delete(path)
|
69
|
+
return false
|
70
|
+
end
|
71
|
+
|
72
|
+
stat = File::Stat.new(path)
|
73
|
+
|
74
|
+
if @iswindows
|
75
|
+
fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
|
76
|
+
inode = [fileId, stat.dev_major, stat.dev_minor]
|
77
|
+
else
|
78
|
+
inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
|
79
|
+
end
|
80
|
+
|
81
|
+
@statcache[path] = inode
|
82
|
+
if @sincedb.member?(inode)
|
83
|
+
last_size = @sincedb[inode][:pos]
|
84
|
+
@logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{stat.size}")
|
85
|
+
if last_size <= stat.size
|
86
|
+
@logger.debug("#{path}: sincedb: seeking to #{last_size}")
|
87
|
+
@files[path].sysseek(last_size, IO::SEEK_SET)
|
88
|
+
else
|
89
|
+
@logger.debug("#{path}: last value size is greater than current value, starting over")
|
90
|
+
@sincedb[inode] = {:size => stat.size, :pos => 0}
|
91
|
+
end
|
92
|
+
elsif (event == :create || event == :create_initial) && @files[path]
|
93
|
+
# TODO(sissel): Allow starting at beginning of the file.
|
94
|
+
if @opts[:start_new_files_at] == :beginning
|
95
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
|
96
|
+
@files[path].sysseek(0, IO::SEEK_SET)
|
97
|
+
@sincedb[inode] = {:size => stat.size, :pos => 0}
|
98
|
+
else
|
99
|
+
# seek to end
|
100
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to end #{stat.size}")
|
101
|
+
@files[path].sysseek(stat.size, IO::SEEK_SET)
|
102
|
+
@sincedb[inode] = {:size => stat.size, :pos => stat.size}
|
103
|
+
end
|
104
|
+
else
|
105
|
+
@logger.debug("#{path}: staying at position 0, no sincedb")
|
106
|
+
end
|
107
|
+
|
108
|
+
return true
|
109
|
+
end # def _open_file
|
110
|
+
|
111
|
+
private
|
112
|
+
def _read_file(path, &block)
|
113
|
+
# BufferedTokenizer is now in codec
|
114
|
+
|
115
|
+
changed = false
|
116
|
+
loop do
|
117
|
+
begin
|
118
|
+
data = @files[path].sysread(32768)
|
119
|
+
changed = true
|
120
|
+
yield(path, data, :log)
|
121
|
+
|
122
|
+
@sincedb[@statcache[path]][:pos] = @files[path].pos
|
123
|
+
_check_sincedb(false, path, &block)
|
124
|
+
rescue EOFError
|
125
|
+
_check_sincedb(true, path, &block) if changed
|
126
|
+
_close_file(path) if @opts[:eof_close]
|
127
|
+
@logger.debug("End of file reached for #{path}")
|
128
|
+
break
|
129
|
+
rescue Errno::EWOULDBLOCK, Errno::EINTR
|
130
|
+
break
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end # def _read_file
|
134
|
+
end # class Tail
|
135
|
+
end # module FileWatch
|
@@ -0,0 +1,189 @@
|
|
1
|
+
require "filewatch/watch"
|
2
|
+
if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
|
3
|
+
require "filewatch/winhelper"
|
4
|
+
end
|
5
|
+
require "logger"
|
6
|
+
require "rbconfig"
|
7
|
+
|
8
|
+
include Java if defined? JRUBY_VERSION
|
9
|
+
require "JRubyFileExtension.jar" if defined? JRUBY_VERSION
|
10
|
+
|
11
|
+
module FileWatch::Ext
|
12
|
+
class TailBase
|
13
|
+
# how often (in seconds) we @logger.warn a failed file open, per path.
|
14
|
+
OPEN_WARN_INTERVAL = ENV["FILEWATCH_OPEN_WARN_INTERVAL"] ?
|
15
|
+
ENV["FILEWATCH_OPEN_WARN_INTERVAL"].to_i : 300
|
16
|
+
|
17
|
+
attr_accessor :logger
|
18
|
+
|
19
|
+
class NoSinceDBPathGiven < StandardError; end
|
20
|
+
|
21
|
+
public
|
22
|
+
def initialize(opts={})
|
23
|
+
@iswindows = ((RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) != nil)
|
24
|
+
|
25
|
+
if opts[:logger]
|
26
|
+
@logger = opts[:logger]
|
27
|
+
else
|
28
|
+
@logger = Logger.new(STDERR)
|
29
|
+
@logger.level = Logger::INFO
|
30
|
+
end
|
31
|
+
@files = {}
|
32
|
+
@lastwarn = Hash.new { |h, k| h[k] = 0 }
|
33
|
+
@watch = FileWatch::Watch.new
|
34
|
+
@watch.logger = @logger
|
35
|
+
@sincedb = {}
|
36
|
+
@sincedb_last_write = 0
|
37
|
+
@statcache = {}
|
38
|
+
@opts = {
|
39
|
+
:sincedb_write_interval => 10,
|
40
|
+
:stat_interval => 1,
|
41
|
+
:discover_interval => 5,
|
42
|
+
:exclude => [],
|
43
|
+
:start_new_files_at => :end,
|
44
|
+
:progressdb => false,
|
45
|
+
:eof_close => false,
|
46
|
+
}.merge(opts)
|
47
|
+
if !@opts.include?(:sincedb_path)
|
48
|
+
@opts[:sincedb_path] = File.join(ENV["HOME"], ".sincedb") if ENV.include?("HOME")
|
49
|
+
@opts[:sincedb_path] = ENV["SINCEDB_PATH"] if ENV.include?("SINCEDB_PATH")
|
50
|
+
end
|
51
|
+
if !@opts.include?(:sincedb_path)
|
52
|
+
raise NoSinceDBPathGiven.new("No HOME or SINCEDB_PATH set in environment. I need one of these set so I can keep track of the files I am following.")
|
53
|
+
end
|
54
|
+
@watch.exclude(@opts[:exclude])
|
55
|
+
|
56
|
+
_sincedb_open
|
57
|
+
end # def initialize
|
58
|
+
|
59
|
+
public
|
60
|
+
def logger=(logger)
|
61
|
+
@logger = logger
|
62
|
+
@watch.logger = logger
|
63
|
+
end # def logger=
|
64
|
+
|
65
|
+
public
|
66
|
+
def tail(path)
|
67
|
+
@watch.watch(path)
|
68
|
+
end # def tail
|
69
|
+
|
70
|
+
public
|
71
|
+
def subscribe(&block)
|
72
|
+
# to be overwritten
|
73
|
+
end # def each
|
74
|
+
|
75
|
+
protected
|
76
|
+
def _open_file(path, event)
|
77
|
+
# to be overwritten
|
78
|
+
end # def _open_file
|
79
|
+
|
80
|
+
protected
|
81
|
+
def _read_file(path, &block)
|
82
|
+
# to be overwritten
|
83
|
+
end # def _read_file
|
84
|
+
|
85
|
+
private
|
86
|
+
def _close_file(path)
|
87
|
+
@files[path].close
|
88
|
+
end
|
89
|
+
|
90
|
+
protected
|
91
|
+
def _check_sincedb(eof, path, &block)
|
92
|
+
now = Time.now.to_i
|
93
|
+
delta = now - @sincedb_last_write
|
94
|
+
if eof || delta >= @opts[:sincedb_write_interval]
|
95
|
+
@logger.debug("writing sincedb (delta since last write = #{delta})") if @logger.debug?
|
96
|
+
_sincedb_write
|
97
|
+
_progressdb_write(path, eof, &block) if @opts[:progressdb]
|
98
|
+
@sincedb_last_write = now
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
public
|
103
|
+
def sincedb_write(reason=nil)
|
104
|
+
@logger.debug("caller requested sincedb write (#{reason})")
|
105
|
+
_sincedb_write
|
106
|
+
end
|
107
|
+
|
108
|
+
protected
|
109
|
+
def _sincedb_open
|
110
|
+
path = @opts[:sincedb_path]
|
111
|
+
begin
|
112
|
+
db = File.open(path)
|
113
|
+
rescue
|
114
|
+
@logger.debug("_sincedb_open: #{path}: #{$!}")
|
115
|
+
return
|
116
|
+
end
|
117
|
+
|
118
|
+
@logger.debug("_sincedb_open: reading from #{path}")
|
119
|
+
db.each do |line|
|
120
|
+
ino, dev_major, dev_minor, size, pos = line.split(" ", 5)
|
121
|
+
inode = [ino, dev_major.to_i, dev_minor.to_i]
|
122
|
+
@logger.debug("_sincedb_open: setting #{inode.inspect} to #{pos.to_i}")
|
123
|
+
@sincedb[inode] = {:size => size.to_i, :pos => pos.to_i}
|
124
|
+
end
|
125
|
+
end # def _sincedb_open
|
126
|
+
|
127
|
+
protected
|
128
|
+
def _sincedb_write
|
129
|
+
path = @opts[:sincedb_path]
|
130
|
+
tmp = "#{path}.new"
|
131
|
+
begin
|
132
|
+
db = File.open(tmp, "w")
|
133
|
+
rescue => e
|
134
|
+
@logger.warn("_sincedb_write failed: #{tmp}: #{e}")
|
135
|
+
return
|
136
|
+
end
|
137
|
+
|
138
|
+
@sincedb.each do |inode, meta|
|
139
|
+
db.puts([inode, meta[:size], meta[:pos]].flatten.join(" "))
|
140
|
+
end
|
141
|
+
db.close
|
142
|
+
|
143
|
+
begin
|
144
|
+
File.rename(tmp, path)
|
145
|
+
rescue => e
|
146
|
+
@logger.warn("_sincedb_write rename/sync failed: #{tmp} -> #{path}: #{e}")
|
147
|
+
end
|
148
|
+
end # def _sincedb_write
|
149
|
+
|
150
|
+
protected
|
151
|
+
def _sincedb_delete(path)
|
152
|
+
inode = @statcache[path]
|
153
|
+
@sincedb.delete(inode)
|
154
|
+
_sincedb_write
|
155
|
+
end
|
156
|
+
|
157
|
+
protected
|
158
|
+
def _progressdb_write(path, eof, &block)
|
159
|
+
if eof
|
160
|
+
inode = @statcache[path]
|
161
|
+
meta = @sincedb[@statcache[path]]
|
162
|
+
line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
|
163
|
+
yield(path, line, :progressdb)
|
164
|
+
else
|
165
|
+
@statcache.each do |path, inode|
|
166
|
+
meta = @sincedb[inode]
|
167
|
+
if meta[:size] != meta[:pos]
|
168
|
+
line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
|
169
|
+
yield(path, line, :progressdb)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
protected
|
176
|
+
def _progressdb_delete(path, &block)
|
177
|
+
inode = @statcache[path]
|
178
|
+
meta = @sincedb[inode]
|
179
|
+
|
180
|
+
line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
|
181
|
+
yield(path, line, :progressdb_del)
|
182
|
+
end
|
183
|
+
|
184
|
+
public
|
185
|
+
def quit
|
186
|
+
@watch.quit
|
187
|
+
end # def quit
|
188
|
+
end # class Tail
|
189
|
+
end # module FileWatch
|
@@ -0,0 +1,153 @@
|
|
1
|
+
require "filewatch/ext/tailbase"
|
2
|
+
require "spreadsheet"
|
3
|
+
|
4
|
+
module FileWatch::Ext
|
5
|
+
class XlsTail < FileWatch::Ext::TailBase
|
6
|
+
|
7
|
+
public
|
8
|
+
def initialize(opts={})
|
9
|
+
@pos = {}
|
10
|
+
super
|
11
|
+
end
|
12
|
+
|
13
|
+
public
|
14
|
+
def subscribe(&block)
|
15
|
+
# subscribe(stat_interval = 1, discover_interval = 5, &block)
|
16
|
+
@watch.subscribe(@opts[:stat_interval],
|
17
|
+
@opts[:discover_interval]) do |event, path|
|
18
|
+
case event
|
19
|
+
when :create, :create_initial
|
20
|
+
if @files.member?(path)
|
21
|
+
@logger.debug("#{event} for #{path}: already exists in @files")
|
22
|
+
next
|
23
|
+
end
|
24
|
+
if _open_file(path, event)
|
25
|
+
_read_file(path, &block)
|
26
|
+
end
|
27
|
+
when :modify
|
28
|
+
@logger.debug(":modify for #{path}")
|
29
|
+
when :delete
|
30
|
+
@logger.debug(":delete for #{path}, deleted from @files")
|
31
|
+
|
32
|
+
_progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
|
33
|
+
|
34
|
+
@files.delete(path)
|
35
|
+
_sincedb_delete(path)
|
36
|
+
@statcache.delete(path)
|
37
|
+
else
|
38
|
+
@logger.warn("unknown event type #{event} for #{path}")
|
39
|
+
end
|
40
|
+
end # @watch.subscribe
|
41
|
+
end # def each
|
42
|
+
|
43
|
+
private
|
44
|
+
def get_size(path)
|
45
|
+
workbook = Spreadsheet.open(path)
|
46
|
+
pos = 0
|
47
|
+
workbook.worksheets.each do |worksheet|
|
48
|
+
pos += worksheet.rows.length
|
49
|
+
end
|
50
|
+
@size = pos
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
def _open_file(path, event)
|
55
|
+
@logger.debug("_open_file: #{path}: opening")
|
56
|
+
begin
|
57
|
+
#Spreadsheet.client_encoding = 'UTF-8'
|
58
|
+
@files[path] = Spreadsheet.open(path)
|
59
|
+
rescue
|
60
|
+
# don't emit this message too often. if a file that we can't
|
61
|
+
# read is changing a lot, we'll try to open it more often,
|
62
|
+
# and might be spammy.
|
63
|
+
now = Time.now.to_i
|
64
|
+
if now - @lastwarn[path] > OPEN_WARN_INTERVAL
|
65
|
+
@logger.warn("failed to open #{path}: #{$!}")
|
66
|
+
@lastwarn[path] = now
|
67
|
+
else
|
68
|
+
@logger.debug("(warn supressed) failed to open #{path}: #{$!}")
|
69
|
+
end
|
70
|
+
#@files.delete(path)
|
71
|
+
return false
|
72
|
+
end
|
73
|
+
|
74
|
+
stat = File::Stat.new(path)
|
75
|
+
size = get_size(path)
|
76
|
+
|
77
|
+
if @iswindows
|
78
|
+
fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
|
79
|
+
inode = [fileId, stat.dev_major, stat.dev_minor]
|
80
|
+
else
|
81
|
+
inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
|
82
|
+
end
|
83
|
+
|
84
|
+
@statcache[path] = inode
|
85
|
+
|
86
|
+
if @sincedb.member?(inode)
|
87
|
+
last_size = @sincedb[inode][:pos]
|
88
|
+
@logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{size}")
|
89
|
+
if last_size <= size
|
90
|
+
@logger.debug("#{path}: sincedb: seeking to #{last_size}")
|
91
|
+
@pos[path] = last_size
|
92
|
+
else
|
93
|
+
@logger.debug("#{path}: last value size is greater than current value, starting over")
|
94
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
95
|
+
end
|
96
|
+
elsif event == :create_initial && @files[path]
|
97
|
+
# TODO(sissel): Allow starting at beginning of the file.
|
98
|
+
if @opts[:start_new_files_at] == :beginning
|
99
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
|
100
|
+
@pos[path] = 0
|
101
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
102
|
+
else
|
103
|
+
# seek to end
|
104
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to end #{size}")
|
105
|
+
@pos[path] = size
|
106
|
+
@sincedb[inode] = {:size => size, :pos => size}
|
107
|
+
end
|
108
|
+
elsif event == :create
|
109
|
+
@pos[path] = 0
|
110
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
111
|
+
else
|
112
|
+
@logger.debug("#{path}: staying at position 0, no sincedb")
|
113
|
+
end
|
114
|
+
|
115
|
+
return true
|
116
|
+
end # def _open_file
|
117
|
+
|
118
|
+
private
|
119
|
+
def _read_file(path, &block)
|
120
|
+
changed = false
|
121
|
+
pos = 0
|
122
|
+
|
123
|
+
worksheets = @files[path].worksheets
|
124
|
+
|
125
|
+
worksheets.each_with_index do |worksheet, index_sheet|
|
126
|
+
|
127
|
+
worksheet.each_with_index do |row, index_row|
|
128
|
+
pos += 1
|
129
|
+
|
130
|
+
if pos > @pos[path]
|
131
|
+
changed = true
|
132
|
+
|
133
|
+
sheet_name = worksheet.name.empty? ? "Sheet#{index_sheet+1}" : worksheet.name
|
134
|
+
|
135
|
+
if pos == @size #end of file is reached
|
136
|
+
data = {:row => row, :wsname => sheet_name, :eof => true}
|
137
|
+
else
|
138
|
+
data = {:row => row, :wsname => sheet_name, :eof => false}
|
139
|
+
end
|
140
|
+
|
141
|
+
yield(path, data, :log)
|
142
|
+
|
143
|
+
@pos[path] = pos
|
144
|
+
@sincedb[@statcache[path]][:pos] = pos
|
145
|
+
_check_sincedb(false, path, &block)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
_check_sincedb(true, path, &block) if changed
|
150
|
+
end # def _read_file
|
151
|
+
|
152
|
+
end # class Tail
|
153
|
+
end # module FileWatch
|
@@ -0,0 +1,151 @@
|
|
1
|
+
require "filewatch/ext/tailbase"
|
2
|
+
require "simple_xlsx_reader"
|
3
|
+
|
4
|
+
module FileWatch::Ext
|
5
|
+
class XlsxTail < FileWatch::Ext::TailBase
|
6
|
+
|
7
|
+
public
|
8
|
+
def initialize(opts={})
|
9
|
+
@pos = {}
|
10
|
+
super
|
11
|
+
end
|
12
|
+
|
13
|
+
public
|
14
|
+
def subscribe(&block)
|
15
|
+
# subscribe(stat_interval = 1, discover_interval = 5, &block)
|
16
|
+
@watch.subscribe(@opts[:stat_interval],
|
17
|
+
@opts[:discover_interval]) do |event, path|
|
18
|
+
case event
|
19
|
+
when :create, :create_initial
|
20
|
+
if @files.member?(path)
|
21
|
+
@logger.debug("#{event} for #{path}: already exists in @files")
|
22
|
+
next
|
23
|
+
end
|
24
|
+
if _open_file(path, event)
|
25
|
+
_read_file(path, &block)
|
26
|
+
end
|
27
|
+
when :modify
|
28
|
+
@logger.debug(":modify for #{path}")
|
29
|
+
when :delete
|
30
|
+
@logger.debug(":delete for #{path}, deleted from @files")
|
31
|
+
|
32
|
+
_progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
|
33
|
+
|
34
|
+
@files.delete(path)
|
35
|
+
_sincedb_delete(path)
|
36
|
+
@statcache.delete(path)
|
37
|
+
else
|
38
|
+
@logger.warn("unknown event type #{event} for #{path}")
|
39
|
+
end
|
40
|
+
end # @watch.subscribe
|
41
|
+
end # def each
|
42
|
+
|
43
|
+
private
|
44
|
+
def get_size(path)
|
45
|
+
doc = SimpleXlsxReader.open(path)
|
46
|
+
pos = 0
|
47
|
+
doc.sheets.each do |worksheet|
|
48
|
+
pos += worksheet.rows.length
|
49
|
+
end
|
50
|
+
@size = pos
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
def _open_file(path, event)
|
55
|
+
@logger.debug("_open_file: #{path}: opening")
|
56
|
+
begin
|
57
|
+
@files[path] = SimpleXlsxReader.open(path)
|
58
|
+
rescue
|
59
|
+
# don't emit this message too often. if a file that we can't
|
60
|
+
# read is changing a lot, we'll try to open it more often,
|
61
|
+
# and might be spammy.
|
62
|
+
now = Time.now.to_i
|
63
|
+
if now - @lastwarn[path] > OPEN_WARN_INTERVAL
|
64
|
+
@logger.warn("failed to open #{path}: #{$!}")
|
65
|
+
@lastwarn[path] = now
|
66
|
+
else
|
67
|
+
@logger.debug("(warn supressed) failed to open #{path}: #{$!}")
|
68
|
+
end
|
69
|
+
#@files.delete(path)
|
70
|
+
return false
|
71
|
+
end
|
72
|
+
|
73
|
+
stat = File::Stat.new(path)
|
74
|
+
size = get_size(path)
|
75
|
+
|
76
|
+
if @iswindows
|
77
|
+
fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
|
78
|
+
inode = [fileId, stat.dev_major, stat.dev_minor]
|
79
|
+
else
|
80
|
+
inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
|
81
|
+
end
|
82
|
+
|
83
|
+
@statcache[path] = inode
|
84
|
+
|
85
|
+
if @sincedb.member?(inode)
|
86
|
+
last_size = @sincedb[inode][:pos]
|
87
|
+
@logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{size}")
|
88
|
+
if last_size <= size
|
89
|
+
@logger.debug("#{path}: sincedb: seeking to #{last_size}")
|
90
|
+
@pos[path] = last_size
|
91
|
+
else
|
92
|
+
@logger.debug("#{path}: last value size is greater than current value, starting over")
|
93
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
94
|
+
end
|
95
|
+
elsif event == :create_initial && @files[path]
|
96
|
+
# TODO(sissel): Allow starting at beginning of the file.
|
97
|
+
if @opts[:start_new_files_at] == :beginning
|
98
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
|
99
|
+
@pos[path] = 0
|
100
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
101
|
+
else
|
102
|
+
# seek to end
|
103
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to end #{size}")
|
104
|
+
@pos[path] = size
|
105
|
+
@sincedb[inode] = {:size => size, :pos => size}
|
106
|
+
end
|
107
|
+
elsif event == :create
|
108
|
+
@pos[path] = 0
|
109
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
110
|
+
else
|
111
|
+
@logger.debug("#{path}: staying at position 0, no sincedb")
|
112
|
+
end
|
113
|
+
|
114
|
+
return true
|
115
|
+
end # def _open_file
|
116
|
+
|
117
|
+
private
|
118
|
+
def _read_file(path, &block)
|
119
|
+
changed = false
|
120
|
+
pos = 0
|
121
|
+
|
122
|
+
worksheets = @files[path].sheets
|
123
|
+
|
124
|
+
worksheets.each_with_index do |worksheet, index_sheet|
|
125
|
+
|
126
|
+
worksheet.rows.each_with_index do |row, index_row|
|
127
|
+
pos += 1
|
128
|
+
|
129
|
+
if pos > @pos[path]
|
130
|
+
changed = true
|
131
|
+
sheet_name = worksheet.name.empty? ? "Sheet#{index_sheet+1}" : worksheet.name
|
132
|
+
|
133
|
+
if pos == @size #end of file is reached
|
134
|
+
data = {:row => row, :wsname => sheet_name, :eof => true}
|
135
|
+
else
|
136
|
+
data = {:row => row, :wsname => sheet_name, :eof => false}
|
137
|
+
end
|
138
|
+
|
139
|
+
yield(path, data, :log)
|
140
|
+
|
141
|
+
@pos[path] = pos
|
142
|
+
@sincedb[@statcache[path]][:pos] = pos
|
143
|
+
_check_sincedb(false, path, &block)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
_check_sincedb(true, path, &block) if changed
|
148
|
+
end # def _read_file
|
149
|
+
|
150
|
+
end # class Tail
|
151
|
+
end # module FileWatch
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require "ffi"
|
2
|
+
|
3
|
+
module Winhelper
|
4
|
+
extend FFI::Library
|
5
|
+
|
6
|
+
ffi_lib 'kernel32'
|
7
|
+
ffi_convention :stdcall
|
8
|
+
class FileTime < FFI::Struct
|
9
|
+
layout :lowDateTime, :uint,
|
10
|
+
:highDateTime, :uint
|
11
|
+
end
|
12
|
+
|
13
|
+
#http://msdn.microsoft.com/en-us/library/windows/desktop/aa363788(v=vs.85).aspx
|
14
|
+
class FileInformation < FFI::Struct
|
15
|
+
def initialize()
|
16
|
+
createTime = FileTime.new
|
17
|
+
lastAccessTime = FileTime.new
|
18
|
+
lastWriteTime = FileTime.new
|
19
|
+
end
|
20
|
+
|
21
|
+
layout :fileAttributes, :uint, #DWORD dwFileAttributes;
|
22
|
+
:createTime, FileTime, #FILETIME ftCreationTime;
|
23
|
+
:lastAccessTime, FileTime, #FILETIME ftLastAccessTime;
|
24
|
+
:lastWriteTime, FileTime, #FILETIME ftLastWriteTime;
|
25
|
+
:volumeSerialNumber, :uint, #DWORD dwVolumeSerialNumber;
|
26
|
+
:fileSizeHigh, :uint, #DWORD nFileSizeHigh;
|
27
|
+
:fileSizeLow, :uint, #DWORD nFileSizeLow;
|
28
|
+
:numberOfLinks, :uint, #DWORD nNumberOfLinks;
|
29
|
+
:fileIndexHigh, :uint, #DWORD nFileIndexHigh;
|
30
|
+
:fileIndexLow, :uint #DWORD nFileIndexLow;
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
#http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx
|
35
|
+
#HANDLE WINAPI CreateFile(_In_ LPCTSTR lpFileName,_In_ DWORD dwDesiredAccess,_In_ DWORD dwShareMode,
|
36
|
+
# _In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes,_In_ DWORD dwCreationDisposition,
|
37
|
+
# _In_ DWORD dwFlagsAndAttributes,_In_opt_ HANDLE hTemplateFile);
|
38
|
+
attach_function :GetOpenFileHandle, :CreateFileA, [:pointer, :uint, :uint, :pointer, :uint, :uint, :pointer], :pointer
|
39
|
+
|
40
|
+
#http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952(v=vs.85).aspx
|
41
|
+
#BOOL WINAPI GetFileInformationByHandle(_In_ HANDLE hFile,_Out_ LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
|
42
|
+
attach_function :GetFileInformationByHandle, [:pointer, :pointer], :int
|
43
|
+
|
44
|
+
attach_function :CloseHandle, [:pointer], :int
|
45
|
+
|
46
|
+
|
47
|
+
def self.GetWindowsUniqueFileIdentifier(path)
|
48
|
+
handle = GetOpenFileHandle(path, 0, 7, nil, 3, 128, nil)
|
49
|
+
fileInfo = Winhelper::FileInformation.new
|
50
|
+
success = GetFileInformationByHandle(handle, fileInfo)
|
51
|
+
CloseHandle(handle)
|
52
|
+
if success == 1
|
53
|
+
#args = [
|
54
|
+
# fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow],
|
55
|
+
# fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow]
|
56
|
+
# ]
|
57
|
+
#p "Information: %u %u %u %u %u %u %u " % args
|
58
|
+
#this is only guaranteed on NTFS, for ReFS on windows 2012, GetFileInformationByHandleEx should be used with FILE_ID_INFO, which returns a 128 bit identifier
|
59
|
+
return "#{fileInfo[:volumeSerialNumber]}-#{fileInfo[:fileIndexLow]}-#{fileInfo[:fileIndexHigh]}"
|
60
|
+
else
|
61
|
+
#p "cannot retrieve file information, returning path"
|
62
|
+
return path;
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
#fileId = Winhelper.GetWindowsUniqueFileIdentifier('C:\inetpub\logs\LogFiles\W3SVC1\u_ex1fdsadfsadfasdf30612.log')
|
68
|
+
#p "FileId: " + fileId
|
69
|
+
#p "outside function, sleeping"
|
70
|
+
#sleep(10)
|
metadata
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: filewatch-ext-excel
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- duft
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-12-18 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: filewatch
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.5.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.5.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: simple_xlsx_reader
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.9.8
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.9.8
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: spreadsheet
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.9.7
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.9.7
|
55
|
+
description: Extensions for ruby-filewatcher, like xls, xlsx
|
56
|
+
email:
|
57
|
+
- dietmar@signifydata.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- lib/JRubyFileExtension.jar
|
63
|
+
- lib/filewatch/ext/filetail.rb
|
64
|
+
- lib/filewatch/ext/tailbase.rb
|
65
|
+
- lib/filewatch/ext/xlstail.rb
|
66
|
+
- lib/filewatch/ext/xlsxtail.rb
|
67
|
+
- lib/filewatch/winhelper.rb
|
68
|
+
homepage: https://bitbucket.org/signify/ruby-filewatch-ext-excel
|
69
|
+
licenses:
|
70
|
+
- MIT
|
71
|
+
metadata: {}
|
72
|
+
post_install_message:
|
73
|
+
rdoc_options: []
|
74
|
+
require_paths:
|
75
|
+
- lib
|
76
|
+
- lib
|
77
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: '0'
|
87
|
+
requirements: []
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 2.2.2
|
90
|
+
signing_key:
|
91
|
+
specification_version: 4
|
92
|
+
summary: filewatch extensions for excel file input
|
93
|
+
test_files: []
|