rflow-components-file 0.0.5

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in rflow-components-file.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,16 @@
+require 'bundler'
+require 'rspec/core/rake_task'
+require 'rake/rdoctask'
+Bundler::GemHelper.install_tasks
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.verbose = true
+  t.rspec_opts = '--tty --color'
+end
+
+Rake::RDocTask.new do |rd|
+  rd.main = "README"
+  rd.rdoc_files.include("README", "lib/**/*.rb")
+  rd.rdoc_dir = File.join('doc', 'html')
+end
+

data/lib/rflow/components/file/directory_watcher.rb

@@ -0,0 +1,95 @@
+require 'rflow/component'
+
+require 'digest/md5'
+
+class RFlow
+  module Components
+    module File
+      class DirectoryWatcher < RFlow::Component
+        output_port :file_port
+        output_port :raw_port
+        
+        DEFAULT_CONFIG = {
+          'directory_path'  => '/tmp/import',
+          'file_name_glob'  => '*',
+          'poll_interval'   => 1,
+          'files_per_poll'  => 1,
+          'remove_files'    => true,
+        }
+
+        attr_accessor :config, :poll_interval, :directory_path, :file_name_glob, :remove_files
+        
+        def configure!(config)
+          @config = DEFAULT_CONFIG.merge config
+          @directory_path  = ::File.expand_path(@config['directory_path'])
+          @file_name_glob  = @config['file_name_glob']
+          @poll_interval   = @config['poll_interval'].to_i
+          @files_per_poll  = @config['files_per_poll'].to_i
+          @remove_files    = to_boolean(@config['remove_files'])
+          
+          unless ::File.directory?(@directory_path)
+            raise ArgumentError, "Invalid directory '#{@directory_path}'"
+          end
+          
+          unless ::File.readable?(@directory_path)
+            raise ArgumentError, "Unable to read from directory '#{@directory_path}'"
+          end
+
+          # TODO: more error checking of input config
+        end
+
+
+        # TODO: optimize sending of messages based on what is connected
+        def run!
+          timer = EventMachine::PeriodicTimer.new(poll_interval) do
+            RFlow.logger.debug "Polling for files in #{::File.join(@directory_path, @file_name_glob)}"
+            # Sort by last modified, which will process the earliest
+            # modified file first
+            file_paths = Dir.glob(::File.join(@directory_path, @file_name_glob)).sort_by {|f| test(?M, f)}
+
+            file_paths.first(@files_per_poll).each do |file_path|
+              RFlow.logger.debug "Importing #{file_path}"
+              ::File.open(file_path, 'r:BINARY') do |file|
+                file_content = file.read
+                
+                RFlow.logger.debug "read #{file_content.bytesize} bytes of #{file.size} in #{file.path}, md5 #{Digest::MD5.hexdigest(file_content)}"
+
+                file_message = RFlow::Message.new('RFlow::Message::Data::File')
+
+                file_message.data.path = ::File.expand_path(file.path)
+                file_message.data.size = file.size
+                file_message.data.content = file_content
+                file_message.data.creation_timestamp = file.ctime
+                file_message.data.modification_timestamp = file.mtime
+                file_message.data.access_timestamp = file.atime
+                
+                file_port.send_message file_message
+
+                raw_message = RFlow::Message.new('RFlow::Message::Data::Raw')
+                raw_message.data.raw = file_content
+                raw_port.send_message raw_message
+              end
+
+              if @remove_files
+                RFlow.logger.debug "Removing #{::File.join(@directory_path, file_path)}"
+                ::File.delete file_path
+              end
+            end
+          end
+        end
+
+        def to_boolean(string)
+          case string
+          when /^true$/i, '1', true
+            true
+          when /^false/i, '0', false
+            false
+          else
+            raise ArgumentError, "'#{string}' cannot be coerced to a boolean value"
+          end
+        end
+        
+      end
+    end
+  end
+end

data/lib/rflow/components/file/extensions.rb

@@ -0,0 +1,58 @@
+class RFlow
+  module Components
+    module File
+      
+      # The set of extensions to add capability to File data types
+      module Extensions
+        
+        # Need to be careful when extending to not clobber data already in data_object
+        module FileExtension
+          def self.extended(base_data)
+            base_data.data_object ||= {
+              'path' => '/', 'size' => 0, 'content' => '',
+              'creation_timestamp' => nil, 'modification_timestamp' => nil, 'access_timestamp' => nil
+            }
+          end
+          
+          # Default/string accessors
+          ['path', 'content'].each do |name|
+            define_method name do |*args|
+              data_object[name]
+            end
+            define_method :"#{name}=" do |*args|
+              data_object[name] = args.first
+            end
+          end
+          
+          # Integer Accessors
+          ['size'].each do |name|
+            define_method name do |*args|
+              data_object[name]
+            end
+            define_method :"#{name}=" do |*args|
+              data_object[name] = args.first.to_i
+            end
+          end
+          
+          # Timestamp Accessors.  Note, the precision of the
+          # XMLTimestamp is set to 9 digits, meaning that the time you
+          # put in might be slightly different from the time you read
+          # out.
+          ['creation_timestamp', 'modification_timestamp', 'access_timestamp'].each do |name|
+            define_method name do |*args|
+              data_object[name] ? Time.xmlschema(data_object[name]) : nil
+            end
+            define_method :"#{name}=" do |*args|
+              if args.first.is_a? Time
+                data_object[name] = args.first.xmlschema(9)
+              else
+                data_object[name] = args.first
+              end
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/rflow/components/file/output_raw_to_files.rb

@@ -0,0 +1,90 @@
+require 'eventmachine'
+require 'rflow/component'
+
+require 'digest/md5'
+
+class RFlow
+  module Components
+    module File
+      class OutputRawToFiles < RFlow::Component
+        input_port :raw_port
+
+        DEFAULT_CONFIG = {
+          'directory_path'  => '/tmp',
+          'file_name_prefix' => 'output.', 
+          'file_name_suffix' => '.out', 
+        }
+        
+        attr_accessor :config, :directory_path, :file_name_prefix, :file_name_suffix
+
+        
+        def configure!(config)
+          @config = DEFAULT_CONFIG.merge config
+          @directory_path  = ::File.expand_path(@config['directory_path'])
+          @file_name_prefix = @config['file_name_prefix']
+          @file_name_suffix = @config['file_name_suffix']
+          
+          unless ::File.directory?(@directory_path)
+            raise ArgumentError, "Invalid directory '#{@directory_path}'"
+          end
+          
+          unless ::File.writable?(@directory_path)
+            raise ArgumentError, "Unable to read from directory '#{@directory_path}'"
+          end
+
+          @output_file_entropy = 0
+          
+          # TODO: more error checking of input config
+        end
+
+        
+        def process_message(input_port, input_port_key, connection, message)
+          return unless message.data_type_name == 'RFlow::Message::Data::Raw'
+          
+          @output_file_entropy = 0
+          begin
+            final_output_file_name = output_file_name
+
+            temp_output_file_path = ::File.join(directory_path, ".#{final_output_file_name}")
+            final_output_file_path = ::File.join(directory_path, "#{final_output_file_name}")
+
+            RFlow.logger.debug "#{self.class.name}##{__method__}: Outputting raw message to #{final_output_file_path} (via #{temp_output_file_path}) with #{message.data.raw.bytesize} bytes and md5 #{Digest::MD5.hexdigest message.data.raw}"
+            
+            ::File.open(temp_output_file_path, ::File::CREAT|::File::EXCL|::File::RDWR, 0644) do |file|
+              file.flock(::File::LOCK_EX)
+              file.write(message.data.raw)
+            end
+            ::File.rename(temp_output_file_path, final_output_file_path)
+            RFlow.logger.debug "#{self.class.name}##{__method__}: Succesfully output raw message to #{final_output_file_path}"
+          rescue Errno::EEXIST => e
+            RFlow.logger.debug("#{self.class.name}##{__method__}: File #{temp_output_file_path} exists, increasing entropy")
+            retry
+          end
+          
+          final_output_file_path
+        end
+
+        
+        private
+
+        
+        def output_file_name 
+          "#{file_name_prefix}#{current_timestamp}-#{output_file_entropy}#{file_name_suffix}"
+        end
+
+        
+        def output_file_entropy
+          @output_file_entropy += 1
+          sprintf("%04d", @output_file_entropy)
+        end
+
+        
+        def current_timestamp
+          time = Time.now
+          time.utc.strftime("%Y%m%d_%H%M%S.") + "%06d" % time.utc.usec
+        end
+        
+      end
+    end
+  end
+end

data/lib/rflow/components/file/version.rb

@@ -0,0 +1,7 @@
+class RFlow
+  module Components
+    module File
+      VERSION = "0.0.5"
+    end
+  end
+end

data/lib/rflow/components/file.rb

@@ -0,0 +1,26 @@
+require 'rflow/components/file/extensions'
+require 'rflow/components/file/directory_watcher'
+require 'rflow/components/file/output_raw_to_files'
+
+class RFlow
+  module Components
+    module File
+      # Load the schemas
+      SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
+      
+      SCHEMA_FILES = {
+        'file.avsc' => 'RFlow::Message::Data::File',
+      }
+      
+      SCHEMA_FILES.each do |file_name, data_type_name|
+        schema_string = ::File.read(::File.join(SCHEMA_DIRECTORY, file_name))
+        RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
+      end
+
+      # Load the data extensions
+      RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::File',
+                                                        RFlow::Components::File::Extensions::FileExtension)
+
+    end
+  end
+end

data/lib/rflow-components-file.rb

@@ -0,0 +1,2 @@
+require 'rflow'
+require 'rflow/components/file'

data/rflow-components-file.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "rflow/components/file/version"
+
+Gem::Specification.new do |s|
+  s.name        = "rflow-components-file"
+  s.version     = RFlow::Components::File::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.required_ruby_version = '~> 1.9'
+  s.authors     = ["Michael L. Artz"]
+  s.email       = ["michael.artz@redjack.com"]
+  s.homepage    = ""
+  s.summary     = %q{Components that operate on files for the RFlow FBP framework}
+  s.description = %q{Components that operate on files for the RFlow FBP framework.  Also includes the File schema}
+
+  s.rubyforge_project = "rflow-components-file"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'rflow', '~> 0.0'
+  s.add_dependency 'eventmachine_httpserver', '~> 0.2'
+
+  s.add_development_dependency 'rspec', '~> 2.6'
+  s.add_development_dependency 'rake', '~> 0.8'
+end

data/schema/file.avsc

@@ -0,0 +1,14 @@
+{
+    "type": "record",
+    "name": "File",
+    "namespace": "org.rflow.message.data",
+    "aliases": [],
+    "fields": [
+        {"name": "path",    "type": ["string", "null"]},
+        {"name": "size",    "type": "long"},
+        {"name": "content", "type": "bytes"},
+        {"name": "creation_timestamp",     "type": ["string", "null"]},
+        {"name": "modification_timestamp", "type": ["string", "null"]},
+        {"name": "access_timestamp",       "type": ["string", "null"]}
+    ]
+}

data/spec/directory_watcher_spec.rb

@@ -0,0 +1,4 @@
+require 'spec_helper.rb'
+
+describe RFlow::Components::File::DirectoryWatcher do
+end

data/spec/extensions_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper.rb'
+
+require 'time'
+
+describe RFlow::Components::File::Extensions::FileExtension do
+  before(:each) do
+    @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::File']['avro']
+  end
+
+  it "should add the extension to RFlow::Configuration" do
+    RFlow::Configuration.available_data_extensions['RFlow::Message::Data::File'].should include(described_class)
+  end
+  
+  it "should set the defaults" do
+    file = RFlow::Message.new('RFlow::Message::Data::File')
+
+    file.data.path.should == '/'
+    file.data.size.should == 0
+    file.data.content.should == ''
+    file.data.creation_timestamp.should == nil
+    file.data.modification_timestamp.should == nil
+    file.data.access_timestamp.should == nil
+  end
+
+  it "should correctly use integers or strings for size field" do
+    file = RFlow::Message.new('RFlow::Message::Data::File')
+
+    file.data.size.should == 0
+    file.data.size = 10
+    file.data.size.should == 10
+    file.data.size = '20'
+    file.data.size == 20
+  end
+
+  it "should correctly use Time or xmlschema strings for timestamp fields" do
+    file = RFlow::Message.new('RFlow::Message::Data::File')
+
+    file.data.creation_timestamp.should == nil
+    now = Time.now
+
+    file.data.creation_timestamp = now
+    file.data.creation_timestamp.should == Time.xmlschema(now.xmlschema(9))
+
+    file.data.creation_timestamp = now.xmlschema
+    file.data.creation_timestamp.should == Time.xmlschema(now.xmlschema)
+  end
+
+  
+end

data/spec/output_raw_to_files_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper.rb'
+
+describe RFlow::Components::File::OutputRawToFiles do
+
+  it "should correctly process file name prefix/suffix" do
+    component = described_class.new(1)
+    component.configure!('file_name_prefix' => 'boom', 'file_name_suffix' => 'town', 'directory_path' => '/tmp')
+    component.send(:output_file_name).should match(/boom.*0001town/)
+  end
+
+  it "should do stuff" do
+    component = described_class.new(1)
+    component.configure!('file_name_prefix' => 'boom.', 'file_name_suffix' => '.town', 'directory_path' => '/tmp')
+
+    message = RFlow::Message.new('RFlow::Message::Data::Raw')
+    message.data.raw = 'boomertown'
+    
+    output_file_path = component.process_message nil, nil, nil, message
+
+    File.exist?(output_file_path).should be_true
+  end
+  
+end

data/spec/schema_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper.rb'
+
+describe 'RFlow::Message::Data::File Avro Schema' do
+  before(:each) do
+    @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::File']['avro']
+  end
+  
+  it "should encode and decode an object" do
+    file = {
+      'path' => '/full/file/path/filename',
+      'size' => 1,
+      'content' => 'CONTENT',
+      'creation_timestamp' => 'CREATEDTIMESTRING',
+      'modification_timestamp' => 'MODIFIEDTIMESTRING',
+      'access_timestamp' => 'ACCESSEDTIMESTRING'
+    }
+
+    expect {encode_avro(@schema_string, file)}.to_not raise_error
+    avro_encoded_file = encode_avro(@schema_string, file)
+
+    expect {decode_avro(@schema_string, avro_encoded_file)}.to_not raise_error
+    decoded_file = decode_avro(@schema_string, avro_encoded_file)
+
+    decoded_file.should == file
+
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'rflow-components-file'))
+
+require 'logger'
+
+RFlow.logger = Logger.new STDOUT
+  
+def decode_avro(schema_string, serialized_object)
+  schema = Avro::Schema.parse(schema_string)
+  sio = StringIO.new(serialized_object)
+  Avro::IO::DatumReader.new(schema, schema).read Avro::IO::BinaryDecoder.new(sio)
+end
+
+def encode_avro(schema_string, object)
+  schema = Avro::Schema.parse(schema_string)
+  sio = StringIO.new
+  Avro::IO::DatumWriter.new(schema).write object, Avro::IO::BinaryEncoder.new(sio)
+  sio.string
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: rflow-components-file
+version: !ruby/object:Gem::Version
+  version: 0.0.5
+  prerelease: 
+platform: ruby
+authors:
+- Michael L. Artz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rflow
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine_httpserver
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: Components that operate on files for the RFlow FBP framework.  Also includes
+  the File schema
+email:
+- michael.artz@redjack.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- lib/rflow-components-file.rb
+- lib/rflow/components/file.rb
+- lib/rflow/components/file/directory_watcher.rb
+- lib/rflow/components/file/extensions.rb
+- lib/rflow/components/file/output_raw_to_files.rb
+- lib/rflow/components/file/version.rb
+- rflow-components-file.gemspec
+- schema/file.avsc
+- spec/directory_watcher_spec.rb
+- spec/extensions_spec.rb
+- spec/output_raw_to_files_spec.rb
+- spec/schema_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ~>
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: rflow-components-file
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Components that operate on files for the RFlow FBP framework
+test_files:
+- spec/directory_watcher_spec.rb
+- spec/extensions_spec.rb
+- spec/output_raw_to_files_spec.rb
+- spec/schema_spec.rb
+- spec/spec_helper.rb