rflow-components-file 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f44728c001d40ee5a90876c3f70b0a43115b5a1
-  data.tar.gz: f218d0e4de80964236ee3bdfb182e9dfb8c47aba
+  metadata.gz: 433fded31e018ca0cc18cb19a6a53f8b95aa4aea
+  data.tar.gz: 4c849d8a12ecdb55a50871de7c2747ba4c10af3f
 SHA512:
-  metadata.gz: efd36259e737fb46c7151ea1d09fc5b0eac4fd68f1b94f51e665785cecae6c88780a4554e7c8989d4cf2607a3f5d5b035a5559a762a38a12f4391e1c26e9dd8b
-  data.tar.gz: 3fb7bca5c53bf8be089343758892a59d411a85a382466784bc38decd56525d4bfd3295ed02c7a95aead6b6a56242c81aab768a66c80aee6670866bba0f6170b4
+  metadata.gz: 375510343e6c6a0c69025c630df3c6305d04db791f3f14d398f42b954bf72d28ecd7820e9f09fa30d8d00dfc836c18ad2e7ca45edd1743c22f3b526b857c8b0f
+  data.tar.gz: 1dbd5d3d0b33db272d80d6e31a3088ba522278fb46bd7fcefe91bf417796b26789bd41e4d685c422a81302ece2b8653efc40c37c453730eae9f7bcedc288dce6

data/.travis.yml

@@ -8,6 +8,7 @@ rvm:
 before_install:
   - sudo apt-get install libtool autoconf automake uuid-dev build-essential
   - wget http://download.zeromq.org/zeromq-3.2.4.tar.gz && tar zxvf zeromq-3.2.4.tar.gz && cd zeromq-3.2.4 && ./configure && make && sudo make install && cd ..
+  - gem update bundler
 # Only has 4.0.4, need 3.2 version due to old em-zeromq
 #  - sudo add-apt-repository -y ppa:chris-lea/zeromq
 #  - sudo apt-get update

data/lib/rflow-components-file.rb

@@ -1,2 +1,3 @@
 require 'rflow'
 require 'rflow/components/file'
+require 'rflow/components/file/output_to_disk'

data/lib/rflow/components/file/directory_watcher.rb

@@ -41,12 +41,21 @@ class RFlow
         def run!
           timer = EventMachine::PeriodicTimer.new(poll_interval) do
             RFlow.logger.debug { "#{name}: Polling for files in #{::File.join(@directory_path, @file_name_glob)}" }
-            # Sort by last modified, which will process the earliest
-            # modified file first
-            file_paths = Dir.glob(::File.join(@directory_path, @file_name_glob)).sort_by {|f| test(?M, f)}
+            file_paths = Dir.glob(::File.join(@directory_path, @file_name_glob)).
+              sort_by {|f| test(?M, f)}. # sort by last modified to process the earliest modified file first
+              select {|f| shard.count == 1 || ((f.sum % shard.count) + 1 == worker.index) } # for multiple copies, share the load equally
 
             file_paths.first(@files_per_poll).each do |path|
               RFlow.logger.debug { "#{name}: Importing #{path}" }
+              unless ::File.readable?(path)
+                RFlow.logger.warn "#{name}: Unable to read file #{path}, skipping it"
+                next
+              end
+              if @remove_files && !::File.writable?(path)
+                RFlow.logger.warn "#{name}: Unable to remove file #{path}, skipping it"
+                next
+              end
+
               ::File.open(path, 'r:BINARY') do |file|
                 content = file.read
 
@@ -67,7 +76,7 @@ class RFlow
               end
 
               if @remove_files
-                RFlow.logger.debug { "#{name}: Removing #{::File.join(@directory_path, path)}" }
+                RFlow.logger.debug { "#{name}: Removing #{path}" }
                 ::File.delete path
               end
             end

data/lib/rflow/components/file/output_raw_to_files.rb

@@ -1,76 +1,18 @@
 require 'eventmachine'
 require 'rflow/component'
 require 'digest/md5'
+require 'rflow/components/file/output_to_disk'
 
 class RFlow
   module Components
     module File
       class OutputRawToFiles < RFlow::Component
+        include RFlow::Components::File::OutputToDisk
         input_port :raw_port
 
-        DEFAULT_CONFIG = {
-          'directory_path'  => '/tmp',
-          'file_name_prefix' => 'output.',
-          'file_name_suffix' => '.out',
-        }
-
-        attr_accessor :config, :directory_path, :file_name_prefix, :file_name_suffix
-
-        def configure!(config)
-          @config = DEFAULT_CONFIG.merge config
-          @directory_path  = ::File.expand_path(@config['directory_path'])
-          @file_name_prefix = @config['file_name_prefix']
-          @file_name_suffix = @config['file_name_suffix']
-
-          unless ::File.directory?(@directory_path)
-            raise ArgumentError, "Invalid directory '#{@directory_path}'"
-          end
-
-          unless ::File.writable?(@directory_path)
-            raise ArgumentError, "Unable to read from directory '#{@directory_path}'"
-          end
-
-          # TODO: more error checking of input config
-        end
-
         def process_message(input_port, input_port_key, connection, message)
           return unless message.data_type_name == 'RFlow::Message::Data::Raw'
-
-          @output_file_entropy = 1
-          begin
-            final_output_file_name = output_file_name
-
-            temp_output_file_path = ::File.join(directory_path, ".#{final_output_file_name}")
-            final_output_file_path = ::File.join(directory_path, "#{final_output_file_name}")
-
-            RFlow.logger.debug { "#{name}: Outputting raw message to #{final_output_file_path} (via #{temp_output_file_path}) with #{message.data.raw.bytesize} bytes and md5 #{Digest::MD5.hexdigest message.data.raw}" }
-
-            ::File.open(temp_output_file_path, ::File::CREAT|::File::EXCL|::File::RDWR, 0644, :external_encoding => 'BINARY') do |file|
-              file.flock(::File::LOCK_EX)
-              file.write(message.data.raw)
-            end
-            ::File.rename(temp_output_file_path, final_output_file_path)
-          rescue Errno::EEXIST => e
-            RFlow.logger.debug { "#{name}: File #{temp_output_file_path} exists, increasing entropy" }
-            @output_file_entropy += 1
-            retry
-          end
-
-          final_output_file_path
-        end
-
-        private
-        def output_file_name
-          "#{file_name_prefix}.#{current_timestamp}.#{output_file_entropy_string}#{file_name_suffix}"
-        end
-
-        def output_file_entropy_string
-          sprintf("%04d", @output_file_entropy || 1)
-        end
-
-        def current_timestamp
-          time = Time.now
-          time.utc.strftime("%Y%m%d_%H%M%S.") + "%06d" % time.utc.usec
+          write_to_file(message.properties) {|file| file.write(message.data.raw) }
         end
       end
     end

data/lib/rflow/components/file/output_to_disk.rb

@@ -0,0 +1,83 @@
+class RFlow
+  module Components
+    module File
+      module OutputToDisk
+        DEFAULT_CONFIG = {
+          'directory_path'  => '/tmp',
+          'file_name_prefix' => 'output.',
+          'file_name_suffix' => '.out',
+        }
+
+        attr_accessor :config, :directory_path, :file_name_prefix, :file_name_suffix
+
+        def configure!(config)
+          @config = DEFAULT_CONFIG.merge config
+          @directory_path  = ::File.expand_path(@config['directory_path'])
+          @file_name_prefix = @config['file_name_prefix']
+          @file_name_suffix = @config['file_name_suffix']
+
+          unless ::File.directory?(@directory_path)
+            raise ArgumentError, "Invalid directory '#{@directory_path}'"
+          end
+
+          unless ::File.writable?(@directory_path)
+            raise ArgumentError, "Unable to read from directory '#{@directory_path}'"
+          end
+
+          # TODO: more error checking of input config
+        end
+
+        # if passed properties, will look for data_uuid property and use as suffix preamble
+        def write_to_file(properties)
+          properties ||= {}
+          @output_file_entropy = 1
+          begin
+            final_output_file_name = output_file_name(properties)
+
+            temp_output_file_path = ::File.join(directory_path, ".#{final_output_file_name}")
+            final_output_file_path = ::File.join(directory_path, "#{final_output_file_name}")
+
+            RFlow.logger.debug { "#{self.class}: Outputting message to #{final_output_file_path} (via #{temp_output_file_path})" }
+
+            ::File.open(temp_output_file_path, ::File::CREAT|::File::EXCL|::File::RDWR, 0644, :external_encoding => 'BINARY') do |file|
+              file.flock(::File::LOCK_EX)
+              bytes_written = yield file
+
+              file.flush
+              raise IOError, "file size of '#{::File.size(temp_output_file_path)}' does not match expected size of '#{bytes_written}'" unless ::File.size(temp_output_file_path) == bytes_written
+            end
+            ::File.rename(temp_output_file_path, final_output_file_path)
+            final_output_file_path
+          rescue Errno::EEXIST => e
+            RFlow.logger.debug { "#{self.class}: File #{temp_output_file_path} exists, increasing entropy" }
+            @output_file_entropy += 1
+            retry
+          rescue StandardError => e
+            RFlow.logger.error { "#{self.class} encountered #{e.message} when creating #{temp_output_file_path}" }
+            begin
+              ::File.delete(temp_output_file_path)
+            rescue => f
+              RFlow.logger.debug {"#{self.class} encountered #{f.message} on cleanup of #{temp_output_file_path}" }
+            end
+            raise e
+          end
+        end
+
+        private
+        def output_file_name(properties)
+          uuid = properties['data_uuid']
+          "#{file_name_prefix}.#{current_timestamp}.#{output_file_entropy_string}#{uuid ? ".#{uuid}" : ''}#{file_name_suffix}"
+        end
+
+        def output_file_entropy_string
+          sprintf("%04d", @output_file_entropy || 1)
+        end
+
+        def current_timestamp
+          time = Time.now
+          time.utc.strftime("%Y%m%d_%H%M%S.") + "%06d" % time.utc.usec
+        end
+      end
+    end
+  end
+end

data/lib/rflow/components/file/version.rb

@@ -1,7 +1,7 @@
 class RFlow
   module Components
     module File
-      VERSION = "1.0.1"
+      VERSION = "1.1.0"
     end
   end
 end

data/rflow-components-file.gemspec

@@ -20,8 +20,9 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
+  s.bindir        = 'bin'
 
-  s.add_dependency 'rflow', '~> 1.0.0'
+  s.add_dependency 'rflow', '~> 1.0'
 
   s.add_development_dependency 'rspec', '~> 3.0'
   s.add_development_dependency 'rspec-collection_matchers', '~> 1.0'

data/spec/rflow/components/file/output_raw_to_files_spec.rb

@@ -12,8 +12,13 @@ class RFlow
 
         let(:component) { described_class.new.tap {|c| c.configure!(config) } }
 
-        it "should correctly process file name prefix/suffix" do
-          expect(component.send(:output_file_name)).to match(/boom.*0001.town/)
+        it 'should correctly process file name prefix/suffix when given message properties with no uuid' do
+          expect(component.send(:output_file_name, {})).to match(/boom.*0001.town/)
+        end
+
+        it 'should correctly process file name prefix/suffix when given message properties' do
+          props = { 'data_uuid' => 'uuid' }
+          expect(component.send(:output_file_name, props)).to match(/boom.*\.0001\.uuid\.town/)
         end
 
         it "should do stuff" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rflow-components-file
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Michael L. Artz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-17 00:00:00.000000000 Z
+date: 2016-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rflow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -88,6 +88,7 @@ files:
 - lib/rflow/components/file/directory_watcher.rb
 - lib/rflow/components/file/extensions.rb
 - lib/rflow/components/file/output_raw_to_files.rb
+- lib/rflow/components/file/output_to_disk.rb
 - lib/rflow/components/file/version.rb
 - rflow-components-file.gemspec
 - schema/file.avsc
@@ -115,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: rflow-components-file
-rubygems_version: 2.3.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Components that operate on files for the RFlow FBP framework