filewatch-ext 0.3.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7aaa9136a78754ff3bf796e41f7f90ebf8d85a0b
4
- data.tar.gz: f562eaacb52a89b0d0663ec0943cea657a0e606a
3
+ metadata.gz: cc02b1e8e969759f62b9962cea5a75e90166d878
4
+ data.tar.gz: c574c0b6b2c0afa7184bf9d83de1cfbece399298
5
5
  SHA512:
6
- metadata.gz: 6b5fde10a6257ba99fcdaf62ac89d3a4f2536f9b7cb556ca42b850ae44b90622e6c6a5a2fa57702d12cf5703eeb5acbeca4e9c783b7eca8a84ad3ce04b6819fc
7
- data.tar.gz: d03e3131588f827ab474e58c6f898f9e81bd9edf7d91ba9c8e184207baa612a1e4af0b633602d53d2fef5335ed29f00a62fe65f87d7ca34a044ac93666042487
6
+ metadata.gz: 6d53563ad238de688ebe97921dcb51ebab42301d27772929a37ca207582b9a5b4300b8e1fefddc30171d9572deeb44371a506f8cd68c3f7aaccb1a0b1273bfe0
7
+ data.tar.gz: cdb418c8ede09c47ad56370f460789232c85353e07bdf0ed3e2adc124b830567547a9e28fef915caea1f4f013410014417e30378fce6dc1f9599149f002e3723
@@ -0,0 +1,61 @@
1
+ require 'filewatch/observing_tail'
2
+
3
+ module FileWatch
4
+ module Ext
5
+ class ProgressTail < FileWatch::ObservingTail
6
+
7
+ private
8
+ def observe_read_file(watched_file, listener)
9
+ changed = false
10
+ loop do
11
+ begin
12
+ data = watched_file.file_read(32768)
13
+ changed = true
14
+ watched_file.buffer_extract(data).each do |line|
15
+ listener.accept(line)
16
+ @sincedb[watched_file.inode] += (line.bytesize + @delimiter_byte_size)
17
+ end
18
+ # watched_file bytes_read tracks the sincedb entry
19
+ # see TODO in watch.rb
20
+ watched_file.update_bytes_read(@sincedb[watched_file.inode])
21
+
22
+ _progressdb_send(watched_file, listener)
23
+ rescue EOFError
24
+ listener.eof
25
+ watched_file.unwatch if @opts[:eof_close]
26
+ break
27
+ rescue Errno::EWOULDBLOCK, Errno::EINTR
28
+ listener.error
29
+ break
30
+ rescue => e
31
+ @logger.debug? && @logger.debug("observe_read_file: general error reading #{watched_file.path} - error: #{e.inspect}")
32
+ listener.error
33
+ break
34
+ end
35
+ end
36
+
37
+ if changed
38
+ now = Time.now.to_i
39
+ delta = now - @sincedb_last_write
40
+ if delta >= @opts[:sincedb_write_interval]
41
+ @logger.debug? && @logger.debug("writing sincedb (delta since last write = #{delta})")
42
+ _sincedb_write
43
+ @sincedb_last_write = now
44
+ end
45
+ end
46
+ end # def _read_file
47
+
48
+
49
+ private
50
+ def _progressdb_send watched_file, listener
51
+ path = watched_file.path
52
+ sincedb_key = watched_file.inode
53
+ stat = watched_file.filestat
54
+
55
+ line = [sincedb_key, stat.size, @sincedb[sincedb_key]].flatten.join(" ")
56
+
57
+ listener.progress(line)
58
+ end
59
+ end # module ProgressTail
60
+ end # module Ext
61
+ end # module FileWatch
@@ -0,0 +1,22 @@
1
+ require "filewatch/ext/progress_tail"
2
+ require "forwardable"
3
+
4
+ module FileWatch
5
+ module Ext
6
+ class Tail
7
+ extend Forwardable
8
+
9
+ def_delegators :@target, :tail, :logger=, :subscribe, :sincedb_record_uid, :sincedb_write, :quit, :close_file
10
+
11
+ attr_writer :target
12
+
13
+ def self.new_observing_progress(opts = {})
14
+ new({}, ProgressTail.new(opts))
15
+ end
16
+
17
+ def initialize(opts = {}, target = YieldingTail.new(opts))
18
+ @target = target
19
+ end
20
+ end
21
+ end
22
+ end
metadata CHANGED
@@ -1,35 +1,49 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: filewatch-ext
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Signify
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-08-25 00:00:00.000000000 Z
11
+ date: 2016-08-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: filewatch
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - "~>"
18
- - !ruby/object:Gem::Version
19
- version: '0.6'
20
17
  - - ">="
21
18
  - !ruby/object:Gem::Version
22
- version: 0.6.5
19
+ version: 0.8.1
20
+ - - "~>"
21
+ - !ruby/object:Gem::Version
22
+ version: '0.8'
23
23
  type: :runtime
24
24
  prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 0.8.1
27
30
  - - "~>"
28
31
  - !ruby/object:Gem::Version
29
- version: '0.6'
32
+ version: '0.8'
33
+ - !ruby/object:Gem::Dependency
34
+ name: stud
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ">="
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ type: :development
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
30
44
  - - ">="
31
45
  - !ruby/object:Gem::Version
32
- version: 0.6.5
46
+ version: '0'
33
47
  description: Extensions for ruby-filewatcher
34
48
  email:
35
49
  - dietmar@signifydata.com
@@ -37,10 +51,8 @@ executables: []
37
51
  extensions: []
38
52
  extra_rdoc_files: []
39
53
  files:
40
- - lib/JRubyFileExtension.jar
41
- - lib/filewatch/ext/filetail.rb
42
- - lib/filewatch/ext/tailbase.rb
43
- - lib/filewatch/winhelper.rb
54
+ - lib/filewatch/ext/progress_tail.rb
55
+ - lib/filewatch/ext/tail.rb
44
56
  homepage: https://bitbucket.org/signify/ruby-filewatch-ext
45
57
  licenses:
46
58
  - MIT
Binary file
@@ -1,135 +0,0 @@
1
- require "filewatch/ext/tailbase"
2
- require "filewatch/buftok"
3
-
4
- module FileWatch::Ext
5
- class FileTail < FileWatch::Ext::TailBase
6
-
7
- public
8
- def subscribe(&block)
9
- # subscribe(stat_interval = 1, discover_interval = 5, &block)
10
- @watch.subscribe(@opts[:stat_interval],
11
- @opts[:discover_interval]) do |event, path|
12
- case event
13
- when :create, :create_initial
14
- if @files.member?(path)
15
- @logger.debug("#{event} for #{path}: already exists in @files")
16
- next
17
- end
18
- if _open_file(path, event)
19
- _read_file(path, &block)
20
- end
21
- when :modify
22
- _open_file(path, event) if @opts[:eof_close]
23
-
24
- if !@files.member?(path)
25
- @logger.debug(":modify for #{path}, does not exist in @files")
26
- if _open_file(path, event)
27
- _read_file(path, &block)
28
- end
29
- else
30
- _read_file(path, &block)
31
- end
32
- when :delete
33
- @logger.debug(":delete for #{path}, deleted from @files")
34
-
35
- _read_file(path, &block) if !@opts[:eof_close]
36
- _progressdb_delete(path, &block) if @opts[:progressdb] && @opts[:progressdb_del]
37
-
38
- @files[path].close if !@opts[:eof_close]
39
- @files.delete(path)
40
- _sincedb_delete(path)
41
- @statcache.delete(path)
42
- else
43
- @logger.warn("unknown event type #{event} for #{path}")
44
- end
45
- end # @watch.subscribe
46
- end # def each
47
-
48
- private
49
- def _open_file(path, event)
50
- @logger.debug("_open_file: #{path}: opening")
51
- begin
52
- if @iswindows && defined? JRUBY_VERSION
53
- @files[path] = Java::RubyFileExt::getRubyFile(path)
54
- else
55
- @files[path] = File.open(path)
56
- end
57
- rescue
58
- # don't emit this message too often. if a file that we can't
59
- # read is changing a lot, we'll try to open it more often,
60
- # and might be spammy.
61
- now = Time.now.to_i
62
- if now - @lastwarn[path] > OPEN_WARN_INTERVAL
63
- @logger.warn("failed to open #{path}: #{$!}")
64
- @lastwarn[path] = now
65
- else
66
- @logger.debug("(warn supressed) failed to open #{path}: #{$!}")
67
- end
68
- @files.delete(path)
69
- return false
70
- end
71
-
72
- stat = File::Stat.new(path)
73
-
74
- if @iswindows
75
- fileId = Winhelper.GetWindowsUniqueFileIdentifier(path)
76
- inode = [fileId, stat.dev_major, stat.dev_minor]
77
- else
78
- inode = [stat.ino.to_s, stat.dev_major, stat.dev_minor]
79
- end
80
-
81
- @statcache[path] = inode
82
- if @sincedb.member?(inode)
83
- last_size = @sincedb[inode][:pos]
84
- @logger.debug("#{path}: sincedb last value #{@sincedb[inode]}, cur size #{stat.size}")
85
- if last_size <= stat.size
86
- @logger.debug("#{path}: sincedb: seeking to #{last_size}")
87
- @files[path].sysseek(last_size, IO::SEEK_SET)
88
- else
89
- @logger.debug("#{path}: last value size is greater than current value, starting over")
90
- @sincedb[inode] = {:size => stat.size, :pos => 0}
91
- end
92
- elsif (event == :create || event == :create_initial) && @files[path]
93
- # TODO(sissel): Allow starting at beginning of the file.
94
- if @opts[:start_new_files_at] == :beginning
95
- @logger.debug("#{path}: initial create, no sincedb, seeking to beginning of file")
96
- @files[path].sysseek(0, IO::SEEK_SET)
97
- @sincedb[inode] = {:size => stat.size, :pos => 0}
98
- else
99
- # seek to end
100
- @logger.debug("#{path}: initial create, no sincedb, seeking to end #{stat.size}")
101
- @files[path].sysseek(stat.size, IO::SEEK_SET)
102
- @sincedb[inode] = {:size => stat.size, :pos => stat.size}
103
- end
104
- else
105
- @logger.debug("#{path}: staying at position 0, no sincedb")
106
- end
107
-
108
- return true
109
- end # def _open_file
110
-
111
- private
112
- def _read_file(path, &block)
113
- # BufferedTokenizer is now in codec
114
-
115
- changed = false
116
- loop do
117
- begin
118
- data = @files[path].sysread(32768)
119
- changed = true
120
- yield(path, data, :log)
121
-
122
- @sincedb[@statcache[path]][:pos] = @files[path].pos
123
- _check_sincedb(false, path, &block)
124
- rescue EOFError
125
- _check_sincedb(true, path, &block) if changed
126
- _close_file(path) if @opts[:eof_close]
127
- @logger.debug("End of file reached for #{path}")
128
- break
129
- rescue Errno::EWOULDBLOCK, Errno::EINTR
130
- break
131
- end
132
- end
133
- end # def _read_file
134
- end # class Tail
135
- end # module FileWatch
@@ -1,189 +0,0 @@
1
- require "filewatch/watch"
2
- if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
3
- require "filewatch/winhelper"
4
- end
5
- require "logger"
6
- require "rbconfig"
7
-
8
- include Java if defined? JRUBY_VERSION
9
- require "JRubyFileExtension.jar" if defined? JRUBY_VERSION
10
-
11
- module FileWatch::Ext
12
- class TailBase
13
- # how often (in seconds) we @logger.warn a failed file open, per path.
14
- OPEN_WARN_INTERVAL = ENV["FILEWATCH_OPEN_WARN_INTERVAL"] ?
15
- ENV["FILEWATCH_OPEN_WARN_INTERVAL"].to_i : 300
16
-
17
- attr_accessor :logger
18
-
19
- class NoSinceDBPathGiven < StandardError; end
20
-
21
- public
22
- def initialize(opts={})
23
- @iswindows = ((RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) != nil)
24
-
25
- if opts[:logger]
26
- @logger = opts[:logger]
27
- else
28
- @logger = Logger.new(STDERR)
29
- @logger.level = Logger::INFO
30
- end
31
- @files = {}
32
- @lastwarn = Hash.new { |h, k| h[k] = 0 }
33
- @watch = FileWatch::Watch.new
34
- @watch.logger = @logger
35
- @sincedb = {}
36
- @sincedb_last_write = 0
37
- @statcache = {}
38
- @opts = {
39
- :sincedb_write_interval => 10,
40
- :stat_interval => 1,
41
- :discover_interval => 5,
42
- :exclude => [],
43
- :start_new_files_at => :end,
44
- :progressdb => false,
45
- :eof_close => false,
46
- }.merge(opts)
47
- if !@opts.include?(:sincedb_path)
48
- @opts[:sincedb_path] = File.join(ENV["HOME"], ".sincedb") if ENV.include?("HOME")
49
- @opts[:sincedb_path] = ENV["SINCEDB_PATH"] if ENV.include?("SINCEDB_PATH")
50
- end
51
- if !@opts.include?(:sincedb_path)
52
- raise NoSinceDBPathGiven.new("No HOME or SINCEDB_PATH set in environment. I need one of these set so I can keep track of the files I am following.")
53
- end
54
- @watch.exclude(@opts[:exclude])
55
-
56
- _sincedb_open
57
- end # def initialize
58
-
59
- public
60
- def logger=(logger)
61
- @logger = logger
62
- @watch.logger = logger
63
- end # def logger=
64
-
65
- public
66
- def tail(path)
67
- @watch.watch(path)
68
- end # def tail
69
-
70
- public
71
- def subscribe(&block)
72
- # to be overwritten
73
- end # def each
74
-
75
- protected
76
- def _open_file(path, event)
77
- # to be overwritten
78
- end # def _open_file
79
-
80
- protected
81
- def _read_file(path, &block)
82
- # to be overwritten
83
- end # def _read_file
84
-
85
- private
86
- def _close_file(path)
87
- @files[path].close
88
- end
89
-
90
- protected
91
- def _check_sincedb(eof, path, &block)
92
- now = Time.now.to_i
93
- delta = now - @sincedb_last_write
94
- if eof || delta >= @opts[:sincedb_write_interval]
95
- @logger.debug("writing sincedb (delta since last write = #{delta})") if @logger.debug?
96
- _sincedb_write
97
- _progressdb_write(path, eof, &block) if @opts[:progressdb]
98
- @sincedb_last_write = now
99
- end
100
- end
101
-
102
- public
103
- def sincedb_write(reason=nil)
104
- @logger.debug("caller requested sincedb write (#{reason})")
105
- _sincedb_write
106
- end
107
-
108
- protected
109
- def _sincedb_open
110
- path = @opts[:sincedb_path]
111
- begin
112
- db = File.open(path)
113
- rescue
114
- @logger.debug("_sincedb_open: #{path}: #{$!}")
115
- return
116
- end
117
-
118
- @logger.debug("_sincedb_open: reading from #{path}")
119
- db.each do |line|
120
- ino, dev_major, dev_minor, size, pos = line.split(" ", 5)
121
- inode = [ino, dev_major.to_i, dev_minor.to_i]
122
- @logger.debug("_sincedb_open: setting #{inode.inspect} to #{pos.to_i}")
123
- @sincedb[inode] = {:size => size.to_i, :pos => pos.to_i}
124
- end
125
- end # def _sincedb_open
126
-
127
- protected
128
- def _sincedb_write
129
- path = @opts[:sincedb_path]
130
- tmp = "#{path}.new"
131
- begin
132
- db = File.open(tmp, "w")
133
- rescue => e
134
- @logger.warn("_sincedb_write failed: #{tmp}: #{e}")
135
- return
136
- end
137
-
138
- @sincedb.each do |inode, meta|
139
- db.puts([inode, meta[:size], meta[:pos]].flatten.join(" "))
140
- end
141
- db.close
142
-
143
- begin
144
- File.rename(tmp, path)
145
- rescue => e
146
- @logger.warn("_sincedb_write rename/sync failed: #{tmp} -> #{path}: #{e}")
147
- end
148
- end # def _sincedb_write
149
-
150
- protected
151
- def _sincedb_delete(path)
152
- inode = @statcache[path]
153
- @sincedb.delete(inode)
154
- _sincedb_write
155
- end
156
-
157
- protected
158
- def _progressdb_write(path, eof, &block)
159
- if eof
160
- inode = @statcache[path]
161
- meta = @sincedb[@statcache[path]]
162
- line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
163
- yield(path, line, :progressdb)
164
- else
165
- @statcache.each do |path, inode|
166
- meta = @sincedb[inode]
167
- if meta[:size] != meta[:pos]
168
- line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
169
- yield(path, line, :progressdb)
170
- end
171
- end
172
- end
173
- end
174
-
175
- protected
176
- def _progressdb_delete(path, &block)
177
- inode = @statcache[path]
178
- meta = @sincedb[inode]
179
-
180
- line = [inode, meta[:size], meta[:pos]].flatten.join(" ")
181
- yield(path, line, :progressdb_del)
182
- end
183
-
184
- public
185
- def quit
186
- @watch.quit
187
- end # def quit
188
- end # class Tail
189
- end # module FileWatch
@@ -1,70 +0,0 @@
1
- require "ffi"
2
-
3
- module Winhelper
4
- extend FFI::Library
5
-
6
- ffi_lib 'kernel32'
7
- ffi_convention :stdcall
8
- class FileTime < FFI::Struct
9
- layout :lowDateTime, :uint,
10
- :highDateTime, :uint
11
- end
12
-
13
- #http://msdn.microsoft.com/en-us/library/windows/desktop/aa363788(v=vs.85).aspx
14
- class FileInformation < FFI::Struct
15
- def initialize()
16
- createTime = FileTime.new
17
- lastAccessTime = FileTime.new
18
- lastWriteTime = FileTime.new
19
- end
20
-
21
- layout :fileAttributes, :uint, #DWORD dwFileAttributes;
22
- :createTime, FileTime, #FILETIME ftCreationTime;
23
- :lastAccessTime, FileTime, #FILETIME ftLastAccessTime;
24
- :lastWriteTime, FileTime, #FILETIME ftLastWriteTime;
25
- :volumeSerialNumber, :uint, #DWORD dwVolumeSerialNumber;
26
- :fileSizeHigh, :uint, #DWORD nFileSizeHigh;
27
- :fileSizeLow, :uint, #DWORD nFileSizeLow;
28
- :numberOfLinks, :uint, #DWORD nNumberOfLinks;
29
- :fileIndexHigh, :uint, #DWORD nFileIndexHigh;
30
- :fileIndexLow, :uint #DWORD nFileIndexLow;
31
- end
32
-
33
-
34
- #http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx
35
- #HANDLE WINAPI CreateFile(_In_ LPCTSTR lpFileName,_In_ DWORD dwDesiredAccess,_In_ DWORD dwShareMode,
36
- # _In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes,_In_ DWORD dwCreationDisposition,
37
- # _In_ DWORD dwFlagsAndAttributes,_In_opt_ HANDLE hTemplateFile);
38
- attach_function :GetOpenFileHandle, :CreateFileA, [:pointer, :uint, :uint, :pointer, :uint, :uint, :pointer], :pointer
39
-
40
- #http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952(v=vs.85).aspx
41
- #BOOL WINAPI GetFileInformationByHandle(_In_ HANDLE hFile,_Out_ LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
42
- attach_function :GetFileInformationByHandle, [:pointer, :pointer], :int
43
-
44
- attach_function :CloseHandle, [:pointer], :int
45
-
46
-
47
- def self.GetWindowsUniqueFileIdentifier(path)
48
- handle = GetOpenFileHandle(path, 0, 7, nil, 3, 128, nil)
49
- fileInfo = Winhelper::FileInformation.new
50
- success = GetFileInformationByHandle(handle, fileInfo)
51
- CloseHandle(handle)
52
- if success == 1
53
- #args = [
54
- # fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow],
55
- # fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow]
56
- # ]
57
- #p "Information: %u %u %u %u %u %u %u " % args
58
- #this is only guaranteed on NTFS, for ReFS on windows 2012, GetFileInformationByHandleEx should be used with FILE_ID_INFO, which returns a 128 bit identifier
59
- return "#{fileInfo[:volumeSerialNumber]}-#{fileInfo[:fileIndexLow]}-#{fileInfo[:fileIndexHigh]}"
60
- else
61
- #p "cannot retrieve file information, returning path"
62
- return path;
63
- end
64
- end
65
- end
66
-
67
- #fileId = Winhelper.GetWindowsUniqueFileIdentifier('C:\inetpub\logs\LogFiles\W3SVC1\u_ex1fdsadfsadfasdf30612.log')
68
- #p "FileId: " + fileId
69
- #p "outside function, sleeping"
70
- #sleep(10)