rflow-components-file 1.0.0a1 → 1.0.0a2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 49e0feafd2bf3538bb54fc26b5397eb80990d8d5
+  data.tar.gz: c0f1749b1ea6370addc49da2bb445fe35618c231
+SHA512:
+  metadata.gz: 694d39fb20d27a10ea09c5599916fb015f90c636987b8254ec6ca229716247b58823c5599a085ead9e1b37ab58618152e9a7e2658c012d856f3a2812d9aad1a4
+  data.tar.gz: 58287bf900d590961d93f8db64de94eafcec18faa336cc9312922e294fe381ee70b94ffb6bb481ba836fdbacaef4a89cf4750ed5c3c3ef5ff3661b1d10c8d6e1

data/.rspec

@@ -0,0 +1 @@
+--color -fd

data/.ruby-gemset

@@ -0,0 +1 @@
+rflow-dev

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.1.1

data/lib/rflow/components/file/directory_watcher.rb

@@ -1,5 +1,4 @@
 require 'rflow/component'
-
 require 'digest/md5'
 
 class RFlow
@@ -38,7 +37,6 @@ class RFlow
           # TODO: more error checking of input config
         end
 
-
         # TODO: optimize sending of messages based on what is connected
         def run!
           timer = EventMachine::PeriodicTimer.new(poll_interval) do
@@ -47,32 +45,30 @@ class RFlow
             # modified file first
             file_paths = Dir.glob(::File.join(@directory_path, @file_name_glob)).sort_by {|f| test(?M, f)}
 
-            file_paths.first(@files_per_poll).each do |file_path|
-              RFlow.logger.debug "Importing #{file_path}"
-              ::File.open(file_path, 'r:BINARY') do |file|
-                file_content = file.read
-
-                RFlow.logger.debug "read #{file_content.bytesize} bytes of #{file.size} in #{file.path}, md5 #{Digest::MD5.hexdigest(file_content)}"
-
-                file_message = RFlow::Message.new('RFlow::Message::Data::File')
-
-                file_message.data.path = ::File.expand_path(file.path)
-                file_message.data.size = file.size
-                file_message.data.content = file_content
-                file_message.data.creation_timestamp = file.ctime
-                file_message.data.modification_timestamp = file.mtime
-                file_message.data.access_timestamp = file.atime
-
-                file_port.send_message file_message
-
-                raw_message = RFlow::Message.new('RFlow::Message::Data::Raw')
-                raw_message.data.raw = file_content
-                raw_port.send_message raw_message
+            file_paths.first(@files_per_poll).each do |path|
+              RFlow.logger.debug "Importing #{path}"
+              ::File.open(path, 'r:BINARY') do |file|
+                content = file.read
+
+                RFlow.logger.debug "read #{content.bytesize} bytes of #{file.size} in #{file.path}, md5 #{Digest::MD5.hexdigest(content)}"
+
+                file_port.send_message(RFlow::Message.new('RFlow::Message::Data::File').tap do |m|
+                  m.data.path = ::File.expand_path(file.path)
+                  m.data.size = file.size
+                  m.data.content = content
+                  m.data.creation_timestamp = file.ctime
+                  m.data.modification_timestamp = file.mtime
+                  m.data.access_timestamp = file.atime
+                end)
+
+                raw_port.send_message(RFlow::Message.new('RFlow::Message::Data::Raw').tap do |m|
+                  m.data.raw = content
+                end)
               end
 
               if @remove_files
-                RFlow.logger.debug "Removing #{::File.join(@directory_path, file_path)}"
-                ::File.delete file_path
+                RFlow.logger.debug "Removing #{::File.join(@directory_path, path)}"
+                ::File.delete path
               end
             end
           end
@@ -80,15 +76,11 @@ class RFlow
 
         def to_boolean(string)
           case string
-          when /^true$/i, '1', true
-            true
-          when /^false/i, '0', false
-            false
-          else
-            raise ArgumentError, "'#{string}' cannot be coerced to a boolean value"
+          when /^true$/i, '1', true; true
+          when /^false/i, '0', false; false
+          else raise ArgumentError, "'#{string}' cannot be coerced to a boolean value"
           end
         end
-
       end
     end
   end

data/lib/rflow/components/file/extensions.rb

@@ -1,10 +1,7 @@
 class RFlow
   module Components
     module File
-
-      # The set of extensions to add capability to File data types
       module Extensions
-
         # Need to be careful when extending to not clobber data already in data_object
         module FileExtension
           def self.extended(base_data)
@@ -50,7 +47,6 @@ class RFlow
               end
             end
           end
-
         end
       end
     end

data/lib/rflow/components/file/output_raw_to_files.rb

@@ -1,6 +1,5 @@
 require 'eventmachine'
 require 'rflow/component'
-
 require 'digest/md5'
 
 class RFlow
@@ -34,7 +33,6 @@ class RFlow
           # TODO: more error checking of input config
         end
 
-
         def process_message(input_port, input_port_key, connection, message)
           return unless message.data_type_name == 'RFlow::Message::Data::Raw'
 
@@ -62,25 +60,19 @@ class RFlow
           final_output_file_path
         end
 
-
         private
-
-
         def output_file_name
           "#{file_name_prefix}.#{current_timestamp}.#{output_file_entropy_string}#{file_name_suffix}"
         end
 
-
         def output_file_entropy_string
           sprintf("%04d", @output_file_entropy || 1)
         end
 
-
         def current_timestamp
           time = Time.now
           time.utc.strftime("%Y%m%d_%H%M%S.") + "%06d" % time.utc.usec
         end
-
       end
     end
   end

data/lib/rflow/components/file/version.rb

@@ -1,7 +1,7 @@
 class RFlow
   module Components
     module File
-      VERSION = "1.0.0a1"
+      VERSION = "1.0.0a2"
     end
   end
 end

data/lib/rflow/components/file.rb

@@ -20,7 +20,6 @@ class RFlow
       # Load the data extensions
       RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::File',
                                                         RFlow::Components::File::Extensions::FileExtension)
-
     end
   end
 end

data/rflow-components-file.gemspec

@@ -21,8 +21,9 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency 'rflow', '~> 1.0.0a1'
+  s.add_dependency 'rflow', '~> 1.0.0a2'
 
-  s.add_development_dependency 'rspec', '~> 2.6'
+  s.add_development_dependency 'rspec', '~> 2.99'
+  s.add_development_dependency 'rspec-collection_matchers', '~> 0.0.4'
   s.add_development_dependency 'rake', '~> 0.8'
 end

data/spec/rflow/components/file/extensions/file_extension_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+require 'time'
+
+class RFlow
+  module Components
+    module File
+      module Extensions
+        describe FileExtension do
+          it "should add the extension to RFlow::Configuration" do
+            Configuration.available_data_extensions['RFlow::Message::Data::File'].should include(described_class)
+          end
+
+          it "should set the defaults" do
+            file = Message.new('RFlow::Message::Data::File').data.tap do |d|
+              d.path.should == '/'
+              d.size.should == 0
+              d.content.should == ''
+              d.creation_timestamp.should be_nil
+              d.modification_timestamp.should be_nil
+              d.access_timestamp.should be_nil
+            end
+          end
+
+          it "should correctly use integers or strings for size field" do
+            Message.new('RFlow::Message::Data::File').data.tap do |d|
+              d.size.should == 0
+              d.size = 10
+              d.size.should == 10
+              d.size = '20'
+              d.size == 20
+            end
+          end
+
+          it "should correctly use Time or xmlschema strings for timestamp fields" do
+            Message.new('RFlow::Message::Data::File').data.tap do |d|
+              now = Time.now
+              d.creation_timestamp.should == nil
+              d.creation_timestamp = now
+              d.creation_timestamp.should == Time.xmlschema(now.xmlschema(9))
+              d.creation_timestamp = now.xmlschema
+              d.creation_timestamp.should == Time.xmlschema(now.xmlschema)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/rflow/components/file/output_raw_to_files_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+class RFlow
+  module Components
+    module File
+      describe OutputRawToFiles do
+        let(:component_config) do
+          OpenStruct.new(:name         => 'port name',
+                         :uuid         => 0,
+                         :input_ports  => [],
+                         :output_ports => [],
+                         :options      => {
+                           'file_name_prefix' => 'boom',
+                           'file_name_suffix' => '.town',
+                           'directory_path'   => '/tmp'})
+        end
+
+        let(:component) { described_class.new(component_config).tap {|c| c.configure!(component_config.options) } }
+
+        it "should correctly process file name prefix/suffix" do
+          component.send(:output_file_name).should match(/boom.*0001.town/)
+        end
+
+        it "should do stuff" do
+          message = Message.new('RFlow::Message::Data::Raw').tap do |m|
+            m.data.raw = 'boomertown'
+          end
+
+          output_file_path = component.process_message nil, nil, nil, message
+
+          ::File.exist?(output_file_path).should be true
+        end
+      end
+    end
+  end
+end

data/spec/schema_spec.rb

@@ -1,9 +1,7 @@
-require 'spec_helper.rb'
+require 'spec_helper'
 
 describe 'RFlow::Message::Data::File Avro Schema' do
-  before(:each) do
-    @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::File']['avro']
-  end
+  let(:schema) { RFlow::Configuration.available_data_types['RFlow::Message::Data::File']['avro'] }
 
   it "should encode and decode an object" do
     file = {
@@ -15,14 +13,12 @@ describe 'RFlow::Message::Data::File Avro Schema' do
       'access_timestamp' => 'ACCESSEDTIMESTRING'
     }
 
-    expect {encode_avro(@schema_string, file)}.to_not raise_error
-    avro_encoded_file = encode_avro(@schema_string, file)
+    expect { encode_avro(schema, file) }.to_not raise_error
+    encoded_file = encode_avro(schema, file)
 
-    expect {decode_avro(@schema_string, avro_encoded_file)}.to_not raise_error
-    decoded_file = decode_avro(@schema_string, avro_encoded_file)
+    expect { decode_avro(schema, encoded_file) }.to_not raise_error
+    decoded_file = decode_avro(schema, encoded_file)
 
     decoded_file.should == file
-
   end
 end
-

data/spec/spec_helper.rb

@@ -3,11 +3,11 @@ require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'rflow-c
 require 'logger'
 
 RFlow.logger = Logger.new STDOUT
+RFlow.logger.level = 5
 
-def decode_avro(schema_string, serialized_object)
+def decode_avro(schema_string, data)
   schema = Avro::Schema.parse(schema_string)
-  sio = StringIO.new(serialized_object)
-  Avro::IO::DatumReader.new(schema, schema).read Avro::IO::BinaryDecoder.new(sio)
+  Avro::IO::DatumReader.new(schema, schema).read Avro::IO::BinaryDecoder.new(StringIO.new(data))
 end
 
 def encode_avro(schema_string, object)

metadata

@@ -1,62 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: rflow-components-file
 version: !ruby/object:Gem::Version
-  version: 1.0.0a1
-  prerelease: 5
+  version: 1.0.0a2
 platform: ruby
 authors:
 - Michael L. Artz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-02 00:00:00.000000000 Z
+date: 2014-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rflow
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0a1
+        version: 1.0.0a2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0a1
+        version: 1.0.0a2
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.99'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: rspec-collection_matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
 description: Components that operate on files for the RFlow FBP framework.  Also includes
@@ -67,8 +74,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
+- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -81,39 +91,36 @@ files:
 - lib/rflow/components/file/version.rb
 - rflow-components-file.gemspec
 - schema/file.avsc
-- spec/directory_watcher_spec.rb
-- spec/extensions_spec.rb
-- spec/output_raw_to_files_spec.rb
+- spec/rflow/components/file/extensions/file_extension_spec.rb
+- spec/rflow/components/file/output_raw_to_files_spec.rb
 - spec/schema_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/redjack/rflow-components-file
 licenses:
 - Apache-2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>'
+  - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
 rubyforge_project: rflow-components-file
-rubygems_version: 1.8.24
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Components that operate on files for the RFlow FBP framework
 test_files:
-- spec/directory_watcher_spec.rb
-- spec/extensions_spec.rb
-- spec/output_raw_to_files_spec.rb
+- spec/rflow/components/file/extensions/file_extension_spec.rb
+- spec/rflow/components/file/output_raw_to_files_spec.rb
 - spec/schema_spec.rb
 - spec/spec_helper.rb

data/spec/directory_watcher_spec.rb

@@ -1,4 +0,0 @@
-require 'spec_helper.rb'
-
-describe RFlow::Components::File::DirectoryWatcher do
-end

data/spec/extensions_spec.rb

@@ -1,49 +0,0 @@
-require 'spec_helper.rb'
-
-require 'time'
-
-describe RFlow::Components::File::Extensions::FileExtension do
-  before(:each) do
-    @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::File']['avro']
-  end
-
-  it "should add the extension to RFlow::Configuration" do
-    RFlow::Configuration.available_data_extensions['RFlow::Message::Data::File'].should include(described_class)
-  end
-
-  it "should set the defaults" do
-    file = RFlow::Message.new('RFlow::Message::Data::File')
-
-    file.data.path.should == '/'
-    file.data.size.should == 0
-    file.data.content.should == ''
-    file.data.creation_timestamp.should == nil
-    file.data.modification_timestamp.should == nil
-    file.data.access_timestamp.should == nil
-  end
-
-  it "should correctly use integers or strings for size field" do
-    file = RFlow::Message.new('RFlow::Message::Data::File')
-
-    file.data.size.should == 0
-    file.data.size = 10
-    file.data.size.should == 10
-    file.data.size = '20'
-    file.data.size == 20
-  end
-
-  it "should correctly use Time or xmlschema strings for timestamp fields" do
-    file = RFlow::Message.new('RFlow::Message::Data::File')
-
-    file.data.creation_timestamp.should == nil
-    now = Time.now
-
-    file.data.creation_timestamp = now
-    file.data.creation_timestamp.should == Time.xmlschema(now.xmlschema(9))
-
-    file.data.creation_timestamp = now.xmlschema
-    file.data.creation_timestamp.should == Time.xmlschema(now.xmlschema)
-  end
-
-
-end

data/spec/output_raw_to_files_spec.rb

@@ -1,36 +0,0 @@
-require 'spec_helper.rb'
-
-describe RFlow::Components::File::OutputRawToFiles do
-
-  before(:each) do
-  end
-
-  let :component_config do
-    OpenStruct.new(:name         => 'port name',
-                   :uuid         => 0,
-                   :input_ports  => [],
-                   :output_ports => [],
-                   :options      => {
-                     'file_name_prefix' => 'boom',
-                     'file_name_suffix' => '.town',
-                     'directory_path'   => '/tmp'})
-  end
-
-  let :component do
-    described_class.new(component_config)
-  end
-
-  it "should correctly process file name prefix/suffix" do
-    component.send(:output_file_name).should match(/boom.*0001.town/)
-  end
-
-  it "should do stuff" do
-    message = RFlow::Message.new('RFlow::Message::Data::Raw')
-    message.data.raw = 'boomertown'
-
-    output_file_path = component.process_message nil, nil, nil, message
-
-    File.exist?(output_file_path).should be_true
-  end
-
-end