filewatch-ext-excel 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/JRubyFileExtension.jar +0 -0
- data/lib/filewatch/ext/filetail.rb +135 -0
- data/lib/filewatch/ext/tailbase.rb +189 -0
- data/lib/filewatch/ext/xlstail.rb +153 -0
- data/lib/filewatch/ext/xlsxtail.rb +151 -0
- data/lib/filewatch/winhelper.rb +70 -0
- metadata +93 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b10b9266fcaab0d8f520febe99e45846fd553664
|
4
|
+
data.tar.gz: 8101e8737b30f3ec592a1484d71921e621eab2ee
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 40c131395cd9ceb2fcc071b9589a6e000b6e2357bc1ef6cc1e28a712661ab74d3cb28898c20f9e458a86288d7183e1b5b77ff7e5319d625f682aeee42f5b11ca
|
7
|
+
data.tar.gz: 98661911c4a780638ea406d8d9622bf4192fad22c0d7e216b3b3dfe88cbb91044790d292bd7be83e29eb6566eefad4a963dc7c408d76025c2ddac27fb23622a9
|
Binary file
|
@@ -0,0 +1,135 @@
|
|
1
|
+
require "filewatch/ext/tailbase"
|
2
|
+
require "filewatch/buftok"
|
3
|
+
|
4
|
+
module FileWatch::Ext
|
5
|
+
class FileTail < FileWatch::Ext::TailBase
|
6
|
+
|
7
|
+
public
|
8
|
+
def subscribe(&block)
|
9
|
+
# subscribe(stat_interval = 1, discover_interval = 5, &block)
|
10
|
+
@watch.subscribe(@opts[:stat_interval],
|
11
|
+
@opts[:discover_interval]) do |event, path|
|
12
|
+
case event
|
13
|
+
when :create, :create_initial
|
14
|
+
if @files.member?(path)
|
15
|
+
@logger.debug("#{event} for #{path}: already exists in @files")
|
16
|
+
next
|
17
|
+
end
|
18
|
+
if _open_file(path, event)
|
19
|
+
_read_file(path, &block)
|
20
|
+
end
|
21
|
+
when :modify
|
22
|
+
_open_file(path, event) if @opts[:eof_close]
|
23
|
+
|
24
|
+
if !@files.member?(path)
|
25
|
+
@logger.debug(":modify for #{path}, does not exist in @files")
|
26
|
+
if _open_file(path, event)
|
27
|
+
_read_file(path, &block)
|
28
|
+
end
|
29
|
+
else
|
30
|
+
_read_file(path, &block)
|
31
|
+
end
|
32
|
+
when :delete
|
33
|
+
@logger.debug(":delete for #{path}, deleted from @files")
|
34
|
+
|
35
|
+
_read_file(path, &block) if !@opts[:eof_close]
|
36
|
+
_progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
|
37
|
+
|
38
|
+
@files[path].close if !@opts[:eof_close]
|
39
|
+
@files.delete(path)
|
40
|
+
_sincedb_delete(path)
|
41
|
+
@statcache.delete(path)
|
42
|
+
else
|
43
|
+
@logger.warn("unknown event type #{event} for #{path}")
|
44
|
+
end
|
45
|
+
end # @watch.subscribe
|
46
|
+
end # def each
|
47
|
+
|
48
|
+
private
|
49
|
+
def _open_file(path, event)
|
50
|
+
@logger.debug("_open_file: #{path}: opening")
|
51
|
+
begin
|
52
|
+
if @iswindows && defined? JRUBY_VERSION
|
53
|
+
@files[path] = Java::RubyFileExt::getRubyFile(path)
|
54
|
+
else
|
55
|
+
@files[path] = File.open(path)
|
56
|
+
end
|
57
|
+
rescue
|
58
|
+
# don't emit this message too often. if a file that we can't
|
59
|
+
# read is changing a lot, we'll try to open it more often,
|
60
|
+
# and might be spammy.
|
61
|
+
now = Time.now.to_i
|
62
|
+
if now - @lastwarn[path] > OPEN_WARN_INTERVAL
|
63
|
+
@logger.warn("failed to open #{path}: #{$!}")
|
64
|
+
@lastwarn[path] = now
|
65
|
+
else
|
66
|
+
@logger.debug("(warn supressed) failed to open #{path}: #{$!}")
|
67
|
+
end
|
68
|
+
@files.delete(path)
|
69
|
+
return false
|
70
|
+
end
|
71
|
+
|
72
|
+
stat = File::Stat.new(path)
|
73
|
+
|
74
|
+
if @iswindows
|
75
|
+
fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
|
76
|
+
inode = [fileId, stat.dev_major, stat.dev_minor]
|
77
|
+
else
|
78
|
+
inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
|
79
|
+
end
|
80
|
+
|
81
|
+
@statcache[path] = inode
|
82
|
+
if @sincedb.member?(inode)
|
83
|
+
last_size = @sincedb[inode][:pos]
|
84
|
+
@logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{stat.size}")
|
85
|
+
if last_size <= stat.size
|
86
|
+
@logger.debug("#{path}: sincedb: seeking to #{last_size}")
|
87
|
+
@files[path].sysseek(last_size, IO::SEEK_SET)
|
88
|
+
else
|
89
|
+
@logger.debug("#{path}: last value size is greater than current value, starting over")
|
90
|
+
@sincedb[inode] = {:size => stat.size, :pos => 0}
|
91
|
+
end
|
92
|
+
elsif (event == :create || event == :create_initial) && @files[path]
|
93
|
+
# TODO(sissel): Allow starting at beginning of the file.
|
94
|
+
if @opts[:start_new_files_at] == :beginning
|
95
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
|
96
|
+
@files[path].sysseek(0, IO::SEEK_SET)
|
97
|
+
@sincedb[inode] = {:size => stat.size, :pos => 0}
|
98
|
+
else
|
99
|
+
# seek to end
|
100
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to end #{stat.size}")
|
101
|
+
@files[path].sysseek(stat.size, IO::SEEK_SET)
|
102
|
+
@sincedb[inode] = {:size => stat.size, :pos => stat.size}
|
103
|
+
end
|
104
|
+
else
|
105
|
+
@logger.debug("#{path}: staying at position 0, no sincedb")
|
106
|
+
end
|
107
|
+
|
108
|
+
return true
|
109
|
+
end # def _open_file
|
110
|
+
|
111
|
+
private
|
112
|
+
def _read_file(path, &block)
|
113
|
+
# BufferedTokenizer is now in codec
|
114
|
+
|
115
|
+
changed = false
|
116
|
+
loop do
|
117
|
+
begin
|
118
|
+
data = @files[path].sysread(32768)
|
119
|
+
changed = true
|
120
|
+
yield(path, data, :log)
|
121
|
+
|
122
|
+
@sincedb[@statcache[path]][:pos] = @files[path].pos
|
123
|
+
_check_sincedb(false, path, &block)
|
124
|
+
rescue EOFError
|
125
|
+
_check_sincedb(true, path, &block) if changed
|
126
|
+
_close_file(path) if @opts[:eof_close]
|
127
|
+
@logger.debug("End of file reached for #{path}")
|
128
|
+
break
|
129
|
+
rescue Errno::EWOULDBLOCK, Errno::EINTR
|
130
|
+
break
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end # def _read_file
|
134
|
+
end # class Tail
|
135
|
+
end # module FileWatch
|
@@ -0,0 +1,189 @@
|
|
1
|
+
require "filewatch/watch"
|
2
|
+
if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
|
3
|
+
require "filewatch/winhelper"
|
4
|
+
end
|
5
|
+
require "logger"
|
6
|
+
require "rbconfig"
|
7
|
+
|
8
|
+
include Java if defined? JRUBY_VERSION
|
9
|
+
require "JRubyFileExtension.jar" if defined? JRUBY_VERSION
|
10
|
+
|
11
|
+
module FileWatch::Ext
|
12
|
+
class TailBase
|
13
|
+
# how often (in seconds) we @logger.warn a failed file open, per path.
|
14
|
+
OPEN_WARN_INTERVAL = ENV["FILEWATCH_OPEN_WARN_INTERVAL"] ?
|
15
|
+
ENV["FILEWATCH_OPEN_WARN_INTERVAL"].to_i : 300
|
16
|
+
|
17
|
+
attr_accessor :logger
|
18
|
+
|
19
|
+
class NoSinceDBPathGiven < StandardError; end
|
20
|
+
|
21
|
+
public
|
22
|
+
def initialize(opts={})
|
23
|
+
@iswindows = ((RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) != nil)
|
24
|
+
|
25
|
+
if opts[:logger]
|
26
|
+
@logger = opts[:logger]
|
27
|
+
else
|
28
|
+
@logger = Logger.new(STDERR)
|
29
|
+
@logger.level = Logger::INFO
|
30
|
+
end
|
31
|
+
@files = {}
|
32
|
+
@lastwarn = Hash.new { |h, k| h[k] = 0 }
|
33
|
+
@watch = FileWatch::Watch.new
|
34
|
+
@watch.logger = @logger
|
35
|
+
@sincedb = {}
|
36
|
+
@sincedb_last_write = 0
|
37
|
+
@statcache = {}
|
38
|
+
@opts = {
|
39
|
+
:sincedb_write_interval => 10,
|
40
|
+
:stat_interval => 1,
|
41
|
+
:discover_interval => 5,
|
42
|
+
:exclude => [],
|
43
|
+
:start_new_files_at => :end,
|
44
|
+
:progressdb => false,
|
45
|
+
:eof_close => false,
|
46
|
+
}.merge(opts)
|
47
|
+
if !@opts.include?(:sincedb_path)
|
48
|
+
@opts[:sincedb_path] = File.join(ENV["HOME"], ".sincedb") if ENV.include?("HOME")
|
49
|
+
@opts[:sincedb_path] = ENV["SINCEDB_PATH"] if ENV.include?("SINCEDB_PATH")
|
50
|
+
end
|
51
|
+
if !@opts.include?(:sincedb_path)
|
52
|
+
raise NoSinceDBPathGiven.new("No HOME or SINCEDB_PATH set in environment. I need one of these set so I can keep track of the files I am following.")
|
53
|
+
end
|
54
|
+
@watch.exclude(@opts[:exclude])
|
55
|
+
|
56
|
+
_sincedb_open
|
57
|
+
end # def initialize
|
58
|
+
|
59
|
+
public
|
60
|
+
def logger=(logger)
|
61
|
+
@logger = logger
|
62
|
+
@watch.logger = logger
|
63
|
+
end # def logger=
|
64
|
+
|
65
|
+
public
|
66
|
+
def tail(path)
|
67
|
+
@watch.watch(path)
|
68
|
+
end # def tail
|
69
|
+
|
70
|
+
public
|
71
|
+
def subscribe(&block)
|
72
|
+
# to be overwritten
|
73
|
+
end # def each
|
74
|
+
|
75
|
+
protected
|
76
|
+
def _open_file(path, event)
|
77
|
+
# to be overwritten
|
78
|
+
end # def _open_file
|
79
|
+
|
80
|
+
protected
|
81
|
+
def _read_file(path, &block)
|
82
|
+
# to be overwritten
|
83
|
+
end # def _read_file
|
84
|
+
|
85
|
+
private
|
86
|
+
def _close_file(path)
|
87
|
+
@files[path].close
|
88
|
+
end
|
89
|
+
|
90
|
+
protected
|
91
|
+
def _check_sincedb(eof, path, &block)
|
92
|
+
now = Time.now.to_i
|
93
|
+
delta = now - @sincedb_last_write
|
94
|
+
if eof || delta >= @opts[:sincedb_write_interval]
|
95
|
+
@logger.debug("writing sincedb (delta since last write = #{delta})") if @logger.debug?
|
96
|
+
_sincedb_write
|
97
|
+
_progressdb_write(path, eof, &block) if @opts[:progressdb]
|
98
|
+
@sincedb_last_write = now
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
public
|
103
|
+
def sincedb_write(reason=nil)
|
104
|
+
@logger.debug("caller requested sincedb write (#{reason})")
|
105
|
+
_sincedb_write
|
106
|
+
end
|
107
|
+
|
108
|
+
protected
|
109
|
+
def _sincedb_open
|
110
|
+
path = @opts[:sincedb_path]
|
111
|
+
begin
|
112
|
+
db = File.open(path)
|
113
|
+
rescue
|
114
|
+
@logger.debug("_sincedb_open: #{path}: #{$!}")
|
115
|
+
return
|
116
|
+
end
|
117
|
+
|
118
|
+
@logger.debug("_sincedb_open: reading from #{path}")
|
119
|
+
db.each do |line|
|
120
|
+
ino, dev_major, dev_minor, size, pos = line.split(" ", 5)
|
121
|
+
inode = [ino, dev_major.to_i, dev_minor.to_i]
|
122
|
+
@logger.debug("_sincedb_open: setting #{inode.inspect} to #{pos.to_i}")
|
123
|
+
@sincedb[inode] = {:size => size.to_i, :pos => pos.to_i}
|
124
|
+
end
|
125
|
+
end # def _sincedb_open
|
126
|
+
|
127
|
+
protected
|
128
|
+
def _sincedb_write
|
129
|
+
path = @opts[:sincedb_path]
|
130
|
+
tmp = "#{path}.new"
|
131
|
+
begin
|
132
|
+
db = File.open(tmp, "w")
|
133
|
+
rescue => e
|
134
|
+
@logger.warn("_sincedb_write failed: #{tmp}: #{e}")
|
135
|
+
return
|
136
|
+
end
|
137
|
+
|
138
|
+
@sincedb.each do |inode, meta|
|
139
|
+
db.puts([inode, meta[:size], meta[:pos]].flatten.join(" "))
|
140
|
+
end
|
141
|
+
db.close
|
142
|
+
|
143
|
+
begin
|
144
|
+
File.rename(tmp, path)
|
145
|
+
rescue => e
|
146
|
+
@logger.warn("_sincedb_write rename/sync failed: #{tmp} -> #{path}: #{e}")
|
147
|
+
end
|
148
|
+
end # def _sincedb_write
|
149
|
+
|
150
|
+
protected
|
151
|
+
def _sincedb_delete(path)
|
152
|
+
inode = @statcache[path]
|
153
|
+
@sincedb.delete(inode)
|
154
|
+
_sincedb_write
|
155
|
+
end
|
156
|
+
|
157
|
+
protected
|
158
|
+
def _progressdb_write(path, eof, &block)
|
159
|
+
if eof
|
160
|
+
inode = @statcache[path]
|
161
|
+
meta = @sincedb[@statcache[path]]
|
162
|
+
line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
|
163
|
+
yield(path, line, :progressdb)
|
164
|
+
else
|
165
|
+
@statcache.each do |path, inode|
|
166
|
+
meta = @sincedb[inode]
|
167
|
+
if meta[:size] != meta[:pos]
|
168
|
+
line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
|
169
|
+
yield(path, line, :progressdb)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
protected
|
176
|
+
def _progressdb_delete(path, &block)
|
177
|
+
inode = @statcache[path]
|
178
|
+
meta = @sincedb[inode]
|
179
|
+
|
180
|
+
line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
|
181
|
+
yield(path, line, :progressdb_del)
|
182
|
+
end
|
183
|
+
|
184
|
+
public
|
185
|
+
def quit
|
186
|
+
@watch.quit
|
187
|
+
end # def quit
|
188
|
+
end # class Tail
|
189
|
+
end # module FileWatch
|
@@ -0,0 +1,153 @@
|
|
1
|
+
require "filewatch/ext/tailbase"
|
2
|
+
require "spreadsheet"
|
3
|
+
|
4
|
+
module FileWatch::Ext
|
5
|
+
class XlsTail < FileWatch::Ext::TailBase
|
6
|
+
|
7
|
+
public
|
8
|
+
def initialize(opts={})
|
9
|
+
@pos = {}
|
10
|
+
super
|
11
|
+
end
|
12
|
+
|
13
|
+
public
|
14
|
+
def subscribe(&block)
|
15
|
+
# subscribe(stat_interval = 1, discover_interval = 5, &block)
|
16
|
+
@watch.subscribe(@opts[:stat_interval],
|
17
|
+
@opts[:discover_interval]) do |event, path|
|
18
|
+
case event
|
19
|
+
when :create, :create_initial
|
20
|
+
if @files.member?(path)
|
21
|
+
@logger.debug("#{event} for #{path}: already exists in @files")
|
22
|
+
next
|
23
|
+
end
|
24
|
+
if _open_file(path, event)
|
25
|
+
_read_file(path, &block)
|
26
|
+
end
|
27
|
+
when :modify
|
28
|
+
@logger.debug(":modify for #{path}")
|
29
|
+
when :delete
|
30
|
+
@logger.debug(":delete for #{path}, deleted from @files")
|
31
|
+
|
32
|
+
_progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
|
33
|
+
|
34
|
+
@files.delete(path)
|
35
|
+
_sincedb_delete(path)
|
36
|
+
@statcache.delete(path)
|
37
|
+
else
|
38
|
+
@logger.warn("unknown event type #{event} for #{path}")
|
39
|
+
end
|
40
|
+
end # @watch.subscribe
|
41
|
+
end # def each
|
42
|
+
|
43
|
+
private
|
44
|
+
def get_size(path)
|
45
|
+
workbook = Spreadsheet.open(path)
|
46
|
+
pos = 0
|
47
|
+
workbook.worksheets.each do |worksheet|
|
48
|
+
pos += worksheet.rows.length
|
49
|
+
end
|
50
|
+
@size = pos
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
def _open_file(path, event)
|
55
|
+
@logger.debug("_open_file: #{path}: opening")
|
56
|
+
begin
|
57
|
+
#Spreadsheet.client_encoding = 'UTF-8'
|
58
|
+
@files[path] = Spreadsheet.open(path)
|
59
|
+
rescue
|
60
|
+
# don't emit this message too often. if a file that we can't
|
61
|
+
# read is changing a lot, we'll try to open it more often,
|
62
|
+
# and might be spammy.
|
63
|
+
now = Time.now.to_i
|
64
|
+
if now - @lastwarn[path] > OPEN_WARN_INTERVAL
|
65
|
+
@logger.warn("failed to open #{path}: #{$!}")
|
66
|
+
@lastwarn[path] = now
|
67
|
+
else
|
68
|
+
@logger.debug("(warn supressed) failed to open #{path}: #{$!}")
|
69
|
+
end
|
70
|
+
#@files.delete(path)
|
71
|
+
return false
|
72
|
+
end
|
73
|
+
|
74
|
+
stat = File::Stat.new(path)
|
75
|
+
size = get_size(path)
|
76
|
+
|
77
|
+
if @iswindows
|
78
|
+
fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
|
79
|
+
inode = [fileId, stat.dev_major, stat.dev_minor]
|
80
|
+
else
|
81
|
+
inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
|
82
|
+
end
|
83
|
+
|
84
|
+
@statcache[path] = inode
|
85
|
+
|
86
|
+
if @sincedb.member?(inode)
|
87
|
+
last_size = @sincedb[inode][:pos]
|
88
|
+
@logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{size}")
|
89
|
+
if last_size <= size
|
90
|
+
@logger.debug("#{path}: sincedb: seeking to #{last_size}")
|
91
|
+
@pos[path] = last_size
|
92
|
+
else
|
93
|
+
@logger.debug("#{path}: last value size is greater than current value, starting over")
|
94
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
95
|
+
end
|
96
|
+
elsif event == :create_initial && @files[path]
|
97
|
+
# TODO(sissel): Allow starting at beginning of the file.
|
98
|
+
if @opts[:start_new_files_at] == :beginning
|
99
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
|
100
|
+
@pos[path] = 0
|
101
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
102
|
+
else
|
103
|
+
# seek to end
|
104
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to end #{size}")
|
105
|
+
@pos[path] = size
|
106
|
+
@sincedb[inode] = {:size => size, :pos => size}
|
107
|
+
end
|
108
|
+
elsif event == :create
|
109
|
+
@pos[path] = 0
|
110
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
111
|
+
else
|
112
|
+
@logger.debug("#{path}: staying at position 0, no sincedb")
|
113
|
+
end
|
114
|
+
|
115
|
+
return true
|
116
|
+
end # def _open_file
|
117
|
+
|
118
|
+
private
|
119
|
+
def _read_file(path, &block)
|
120
|
+
changed = false
|
121
|
+
pos = 0
|
122
|
+
|
123
|
+
worksheets = @files[path].worksheets
|
124
|
+
|
125
|
+
worksheets.each_with_index do |worksheet, index_sheet|
|
126
|
+
|
127
|
+
worksheet.each_with_index do |row, index_row|
|
128
|
+
pos += 1
|
129
|
+
|
130
|
+
if pos > @pos[path]
|
131
|
+
changed = true
|
132
|
+
|
133
|
+
sheet_name = worksheet.name.empty? ? "Sheet#{index_sheet+1}" : worksheet.name
|
134
|
+
|
135
|
+
if pos == @size #end of file is reached
|
136
|
+
data = {:row => row, :wsname => sheet_name, :eof => true}
|
137
|
+
else
|
138
|
+
data = {:row => row, :wsname => sheet_name, :eof => false}
|
139
|
+
end
|
140
|
+
|
141
|
+
yield(path, data, :log)
|
142
|
+
|
143
|
+
@pos[path] = pos
|
144
|
+
@sincedb[@statcache[path]][:pos] = pos
|
145
|
+
_check_sincedb(false, path, &block)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
_check_sincedb(true, path, &block) if changed
|
150
|
+
end # def _read_file
|
151
|
+
|
152
|
+
end # class Tail
|
153
|
+
end # module FileWatch
|
@@ -0,0 +1,151 @@
|
|
1
|
+
require "filewatch/ext/tailbase"
|
2
|
+
require "simple_xlsx_reader"
|
3
|
+
|
4
|
+
module FileWatch::Ext
|
5
|
+
class XlsxTail < FileWatch::Ext::TailBase
|
6
|
+
|
7
|
+
public
|
8
|
+
def initialize(opts={})
|
9
|
+
@pos = {}
|
10
|
+
super
|
11
|
+
end
|
12
|
+
|
13
|
+
public
|
14
|
+
def subscribe(&block)
|
15
|
+
# subscribe(stat_interval = 1, discover_interval = 5, &block)
|
16
|
+
@watch.subscribe(@opts[:stat_interval],
|
17
|
+
@opts[:discover_interval]) do |event, path|
|
18
|
+
case event
|
19
|
+
when :create, :create_initial
|
20
|
+
if @files.member?(path)
|
21
|
+
@logger.debug("#{event} for #{path}: already exists in @files")
|
22
|
+
next
|
23
|
+
end
|
24
|
+
if _open_file(path, event)
|
25
|
+
_read_file(path, &block)
|
26
|
+
end
|
27
|
+
when :modify
|
28
|
+
@logger.debug(":modify for #{path}")
|
29
|
+
when :delete
|
30
|
+
@logger.debug(":delete for #{path}, deleted from @files")
|
31
|
+
|
32
|
+
_progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
|
33
|
+
|
34
|
+
@files.delete(path)
|
35
|
+
_sincedb_delete(path)
|
36
|
+
@statcache.delete(path)
|
37
|
+
else
|
38
|
+
@logger.warn("unknown event type #{event} for #{path}")
|
39
|
+
end
|
40
|
+
end # @watch.subscribe
|
41
|
+
end # def each
|
42
|
+
|
43
|
+
private
|
44
|
+
def get_size(path)
|
45
|
+
doc = SimpleXlsxReader.open(path)
|
46
|
+
pos = 0
|
47
|
+
doc.sheets.each do |worksheet|
|
48
|
+
pos += worksheet.rows.length
|
49
|
+
end
|
50
|
+
@size = pos
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
def _open_file(path, event)
|
55
|
+
@logger.debug("_open_file: #{path}: opening")
|
56
|
+
begin
|
57
|
+
@files[path] = SimpleXlsxReader.open(path)
|
58
|
+
rescue
|
59
|
+
# don't emit this message too often. if a file that we can't
|
60
|
+
# read is changing a lot, we'll try to open it more often,
|
61
|
+
# and might be spammy.
|
62
|
+
now = Time.now.to_i
|
63
|
+
if now - @lastwarn[path] > OPEN_WARN_INTERVAL
|
64
|
+
@logger.warn("failed to open #{path}: #{$!}")
|
65
|
+
@lastwarn[path] = now
|
66
|
+
else
|
67
|
+
@logger.debug("(warn supressed) failed to open #{path}: #{$!}")
|
68
|
+
end
|
69
|
+
#@files.delete(path)
|
70
|
+
return false
|
71
|
+
end
|
72
|
+
|
73
|
+
stat = File::Stat.new(path)
|
74
|
+
size = get_size(path)
|
75
|
+
|
76
|
+
if @iswindows
|
77
|
+
fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
|
78
|
+
inode = [fileId, stat.dev_major, stat.dev_minor]
|
79
|
+
else
|
80
|
+
inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
|
81
|
+
end
|
82
|
+
|
83
|
+
@statcache[path] = inode
|
84
|
+
|
85
|
+
if @sincedb.member?(inode)
|
86
|
+
last_size = @sincedb[inode][:pos]
|
87
|
+
@logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{size}")
|
88
|
+
if last_size <= size
|
89
|
+
@logger.debug("#{path}: sincedb: seeking to #{last_size}")
|
90
|
+
@pos[path] = last_size
|
91
|
+
else
|
92
|
+
@logger.debug("#{path}: last value size is greater than current value, starting over")
|
93
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
94
|
+
end
|
95
|
+
elsif event == :create_initial && @files[path]
|
96
|
+
# TODO(sissel): Allow starting at beginning of the file.
|
97
|
+
if @opts[:start_new_files_at] == :beginning
|
98
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
|
99
|
+
@pos[path] = 0
|
100
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
101
|
+
else
|
102
|
+
# seek to end
|
103
|
+
@logger.debug("#{path}: initial create, no sincedb, seeking to end #{size}")
|
104
|
+
@pos[path] = size
|
105
|
+
@sincedb[inode] = {:size => size, :pos => size}
|
106
|
+
end
|
107
|
+
elsif event == :create
|
108
|
+
@pos[path] = 0
|
109
|
+
@sincedb[inode] = {:size => size, :pos => 0}
|
110
|
+
else
|
111
|
+
@logger.debug("#{path}: staying at position 0, no sincedb")
|
112
|
+
end
|
113
|
+
|
114
|
+
return true
|
115
|
+
end # def _open_file
|
116
|
+
|
117
|
+
private
|
118
|
+
def _read_file(path, &block)
|
119
|
+
changed = false
|
120
|
+
pos = 0
|
121
|
+
|
122
|
+
worksheets = @files[path].sheets
|
123
|
+
|
124
|
+
worksheets.each_with_index do |worksheet, index_sheet|
|
125
|
+
|
126
|
+
worksheet.rows.each_with_index do |row, index_row|
|
127
|
+
pos += 1
|
128
|
+
|
129
|
+
if pos > @pos[path]
|
130
|
+
changed = true
|
131
|
+
sheet_name = worksheet.name.empty? ? "Sheet#{index_sheet+1}" : worksheet.name
|
132
|
+
|
133
|
+
if pos == @size #end of file is reached
|
134
|
+
data = {:row => row, :wsname => sheet_name, :eof => true}
|
135
|
+
else
|
136
|
+
data = {:row => row, :wsname => sheet_name, :eof => false}
|
137
|
+
end
|
138
|
+
|
139
|
+
yield(path, data, :log)
|
140
|
+
|
141
|
+
@pos[path] = pos
|
142
|
+
@sincedb[@statcache[path]][:pos] = pos
|
143
|
+
_check_sincedb(false, path, &block)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
_check_sincedb(true, path, &block) if changed
|
148
|
+
end # def _read_file
|
149
|
+
|
150
|
+
end # class Tail
|
151
|
+
end # module FileWatch
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require "ffi"
|
2
|
+
|
3
|
+
module Winhelper
|
4
|
+
extend FFI::Library
|
5
|
+
|
6
|
+
ffi_lib 'kernel32'
|
7
|
+
ffi_convention :stdcall
|
8
|
+
class FileTime < FFI::Struct
|
9
|
+
layout :lowDateTime, :uint,
|
10
|
+
:highDateTime, :uint
|
11
|
+
end
|
12
|
+
|
13
|
+
#http://msdn.microsoft.com/en-us/library/windows/desktop/aa363788(v=vs.85).aspx
|
14
|
+
class FileInformation < FFI::Struct
|
15
|
+
def initialize()
|
16
|
+
createTime = FileTime.new
|
17
|
+
lastAccessTime = FileTime.new
|
18
|
+
lastWriteTime = FileTime.new
|
19
|
+
end
|
20
|
+
|
21
|
+
layout :fileAttributes, :uint, #DWORD dwFileAttributes;
|
22
|
+
:createTime, FileTime, #FILETIME ftCreationTime;
|
23
|
+
:lastAccessTime, FileTime, #FILETIME ftLastAccessTime;
|
24
|
+
:lastWriteTime, FileTime, #FILETIME ftLastWriteTime;
|
25
|
+
:volumeSerialNumber, :uint, #DWORD dwVolumeSerialNumber;
|
26
|
+
:fileSizeHigh, :uint, #DWORD nFileSizeHigh;
|
27
|
+
:fileSizeLow, :uint, #DWORD nFileSizeLow;
|
28
|
+
:numberOfLinks, :uint, #DWORD nNumberOfLinks;
|
29
|
+
:fileIndexHigh, :uint, #DWORD nFileIndexHigh;
|
30
|
+
:fileIndexLow, :uint #DWORD nFileIndexLow;
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
#http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx
|
35
|
+
#HANDLE WINAPI CreateFile(_In_ LPCTSTR lpFileName,_In_ DWORD dwDesiredAccess,_In_ DWORD dwShareMode,
|
36
|
+
# _In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes,_In_ DWORD dwCreationDisposition,
|
37
|
+
# _In_ DWORD dwFlagsAndAttributes,_In_opt_ HANDLE hTemplateFile);
|
38
|
+
attach_function :GetOpenFileHandle, :CreateFileA, [:pointer, :uint, :uint, :pointer, :uint, :uint, :pointer], :pointer
|
39
|
+
|
40
|
+
#http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952(v=vs.85).aspx
|
41
|
+
#BOOL WINAPI GetFileInformationByHandle(_In_ HANDLE hFile,_Out_ LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
|
42
|
+
attach_function :GetFileInformationByHandle, [:pointer, :pointer], :int
|
43
|
+
|
44
|
+
attach_function :CloseHandle, [:pointer], :int
|
45
|
+
|
46
|
+
|
47
|
+
def self.GetWindowsUniqueFileIdentifier(path)
|
48
|
+
handle = GetOpenFileHandle(path, 0, 7, nil, 3, 128, nil)
|
49
|
+
fileInfo = Winhelper::FileInformation.new
|
50
|
+
success = GetFileInformationByHandle(handle, fileInfo)
|
51
|
+
CloseHandle(handle)
|
52
|
+
if success == 1
|
53
|
+
#args = [
|
54
|
+
# fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow],
|
55
|
+
# fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow]
|
56
|
+
# ]
|
57
|
+
#p "Information: %u %u %u %u %u %u %u " % args
|
58
|
+
#this is only guaranteed on NTFS, for ReFS on windows 2012, GetFileInformationByHandleEx should be used with FILE_ID_INFO, which returns a 128 bit identifier
|
59
|
+
return "#{fileInfo[:volumeSerialNumber]}-#{fileInfo[:fileIndexLow]}-#{fileInfo[:fileIndexHigh]}"
|
60
|
+
else
|
61
|
+
#p "cannot retrieve file information, returning path"
|
62
|
+
return path;
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
#fileId = Winhelper.GetWindowsUniqueFileIdentifier('C:\inetpub\logs\LogFiles\W3SVC1\u_ex1fdsadfsadfasdf30612.log')
|
68
|
+
#p "FileId: " + fileId
|
69
|
+
#p "outside function, sleeping"
|
70
|
+
#sleep(10)
|
metadata
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: filewatch-ext-excel
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- duft
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-12-18 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: filewatch
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.5.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.5.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: simple_xlsx_reader
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.9.8
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.9.8
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: spreadsheet
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.9.7
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.9.7
|
55
|
+
description: Extensions for ruby-filewatcher, like xls, xlsx
|
56
|
+
email:
|
57
|
+
- dietmar@signifydata.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- lib/JRubyFileExtension.jar
|
63
|
+
- lib/filewatch/ext/filetail.rb
|
64
|
+
- lib/filewatch/ext/tailbase.rb
|
65
|
+
- lib/filewatch/ext/xlstail.rb
|
66
|
+
- lib/filewatch/ext/xlsxtail.rb
|
67
|
+
- lib/filewatch/winhelper.rb
|
68
|
+
homepage: https://bitbucket.org/signify/ruby-filewatch-ext-excel
|
69
|
+
licenses:
|
70
|
+
- MIT
|
71
|
+
metadata: {}
|
72
|
+
post_install_message:
|
73
|
+
rdoc_options: []
|
74
|
+
require_paths:
|
75
|
+
- lib
|
76
|
+
- lib
|
77
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: '0'
|
87
|
+
requirements: []
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 2.2.2
|
90
|
+
signing_key:
|
91
|
+
specification_version: 4
|
92
|
+
summary: filewatch extensions for excel file input
|
93
|
+
test_files: []
|