rflow-components-file 1.2.0 → 1.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.yardopts +1 -0
- data/Rakefile +2 -6
- data/lib/rflow/components/file.rb +5 -0
- data/lib/rflow/components/file/directory_watcher.rb +28 -1
- data/lib/rflow/components/file/extensions.rb +42 -0
- data/lib/rflow/components/file/output_raw_to_files.rb +8 -0
- data/lib/rflow/components/file/output_to_disk.rb +22 -1
- data/lib/rflow/components/file/version.rb +2 -1
- data/rflow-components-file.gemspec +1 -0
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 445c9e66f4d8cd0990878daee9349741a9a2c957
|
4
|
+
data.tar.gz: 2ea84f5a1c971fbf5c5b2f803d5527a553cfc703
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 85642e1fae29212da950b3607c2adabc08dc479353e1cdbdc60e01823260547fa377a0a733f21bbe60c9ab40689f29f7e4f7cc16b016a39be3d096c06a040746
|
7
|
+
data.tar.gz: fbeb8ca403c8a707a3e2cd683ad84c54b3d029c3d035071a78b532e3eb3b781d1eca13fdfea2988f9142a57f58eb3f0e0c78f250c547e03e3c40a744f49bb299
|
data/.gitignore
CHANGED
data/.yardopts
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--output ./doc --main README.md --files schema/*.avsc lib/**/*.rb - README.md LICENSE
|
data/Rakefile
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'bundler'
|
2
2
|
require 'rspec/core/rake_task'
|
3
|
-
require '
|
3
|
+
require 'yard'
|
4
4
|
Bundler::GemHelper.install_tasks
|
5
5
|
|
6
6
|
RSpec::Core::RakeTask.new(:spec) do |t|
|
@@ -8,8 +8,4 @@ RSpec::Core::RakeTask.new(:spec) do |t|
|
|
8
8
|
t.rspec_opts = '--tty --color'
|
9
9
|
end
|
10
10
|
|
11
|
-
Rake::
|
12
|
-
rd.main = "README.md"
|
13
|
-
rd.rdoc_files.include("README.md", "lib/**/*.rb")
|
14
|
-
rd.rdoc_dir = File.join('doc', 'html')
|
15
|
-
end
|
11
|
+
YARD::Rake::YardocTask.new
|
@@ -2,12 +2,17 @@ require 'rflow/components/file/extensions'
|
|
2
2
|
require 'rflow/components/file/directory_watcher'
|
3
3
|
require 'rflow/components/file/output_raw_to_files'
|
4
4
|
|
5
|
+
# RFlow classes.
|
5
6
|
class RFlow
|
7
|
+
# RFlow component classes.
|
6
8
|
module Components
|
9
|
+
# File RFlow component classes.
|
7
10
|
module File
|
8
11
|
# Load the schemas
|
12
|
+
# @!visibility private
|
9
13
|
SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
|
10
14
|
|
15
|
+
# @!visibility private
|
11
16
|
SCHEMA_FILES = {
|
12
17
|
'file.avsc' => 'RFlow::Message::Data::File',
|
13
18
|
}
|
@@ -4,10 +4,31 @@ require 'digest/md5'
|
|
4
4
|
class RFlow
|
5
5
|
module Components
|
6
6
|
module File
|
7
|
+
# Component that watches a directory for new files. When it does, it (optionally) deletes them
|
8
|
+
# and sends along +RFlow::Message+s of type {RFlow::Message::Data::File} and +RFlow::Message::Data::Raw+.
|
9
|
+
#
|
10
|
+
# Accepts config parameters:
|
11
|
+
# - +directory_path+ - the directory path to monitor
|
12
|
+
# - +file_name_glob+ - glob of filenames to monitor within +directory_path+
|
13
|
+
# - +poll_interval+ - how often, in seconds, to poll
|
14
|
+
# - +files_per_poll+ - maximum number of files to be brought in per poll
|
15
|
+
# - +remove_files+ - true to remove the files after they've been brought in
|
7
16
|
class DirectoryWatcher < RFlow::Component
|
17
|
+
# @!attribute [r] file_port
|
18
|
+
# Outputs +RFlow::Message+s of type {RFlow::Message::Data::File}
|
19
|
+
# when detecting a new file in the directory.
|
20
|
+
#
|
21
|
+
# @return [RFlow::Component::OutputPort]
|
8
22
|
output_port :file_port
|
23
|
+
# @!attribute [r] raw_port
|
24
|
+
# Outputs +RFlow::Message+s of type +RFlow::Message::Data::Raw+
|
25
|
+
# when detecting a new file in the directory. The raw bytes are just
|
26
|
+
# the contents of the file.
|
27
|
+
#
|
28
|
+
# @return [RFlow::Component::OutputPort]
|
9
29
|
output_port :raw_port
|
10
30
|
|
31
|
+
# Default config.
|
11
32
|
DEFAULT_CONFIG = {
|
12
33
|
'directory_path' => '/tmp/import',
|
13
34
|
'file_name_glob' => '*',
|
@@ -16,8 +37,11 @@ class RFlow
|
|
16
37
|
'remove_files' => true,
|
17
38
|
}
|
18
39
|
|
40
|
+
# @!visibility private
|
19
41
|
attr_accessor :config, :poll_interval, :directory_path, :file_name_glob, :remove_files
|
20
42
|
|
43
|
+
# RFlow-called method at startup.
|
44
|
+
# @return [void]
|
21
45
|
def configure!(config)
|
22
46
|
@config = DEFAULT_CONFIG.merge config
|
23
47
|
@directory_path = ::File.expand_path(@config['directory_path'])
|
@@ -37,8 +61,10 @@ class RFlow
|
|
37
61
|
# TODO: more error checking of input config
|
38
62
|
end
|
39
63
|
|
40
|
-
#
|
64
|
+
# RFlow-called method at startup.
|
65
|
+
# @return [void]
|
41
66
|
def run!
|
67
|
+
# TODO: optimize sending of messages based on what is connected
|
42
68
|
timer = EventMachine::PeriodicTimer.new(poll_interval) do
|
43
69
|
RFlow.logger.debug { "#{name}: Polling for files in #{::File.join(@directory_path, @file_name_glob)}" }
|
44
70
|
file_paths = Dir.glob(::File.join(@directory_path, @file_name_glob)).
|
@@ -83,6 +109,7 @@ class RFlow
|
|
83
109
|
end
|
84
110
|
end
|
85
111
|
|
112
|
+
# @!visibility private
|
86
113
|
def to_boolean(string)
|
87
114
|
case string
|
88
115
|
when /^true$/i, '1', true; true
|
@@ -1,9 +1,51 @@
|
|
1
1
|
class RFlow
|
2
|
+
# @!parse
|
3
|
+
# # Fake classes in this tree to document the actual message types.
|
4
|
+
# class Message
|
5
|
+
# # Fake classes in this tree to document the actual message types.
|
6
|
+
# class Data
|
7
|
+
# # A message representing a file on disk, including its contents. The file might no longer exist.
|
8
|
+
# class File
|
9
|
+
# # @!attribute path
|
10
|
+
# # The file's pathname.
|
11
|
+
# # @return [String]
|
12
|
+
#
|
13
|
+
# # @!attribute size
|
14
|
+
# # The file's size.
|
15
|
+
# # @return [Integer]
|
16
|
+
#
|
17
|
+
# # @!attribute content
|
18
|
+
# # The file's binary content.
|
19
|
+
# # @return [String]
|
20
|
+
#
|
21
|
+
# # @!attribute creation_timestamp
|
22
|
+
# # The file's creation timestamp as an XML schema-compatible dateTime string.
|
23
|
+
# # @return [String]
|
24
|
+
#
|
25
|
+
# # @!attribute modification_timestamp
|
26
|
+
# # The file's modification timestamp as an XML schema-compatible dateTime string.
|
27
|
+
# # @return [String]
|
28
|
+
#
|
29
|
+
# # @!attribute access_timestamp
|
30
|
+
# # The file's access timestamp as an XML schema-compatible dateTime string.
|
31
|
+
# # @return [String]
|
32
|
+
#
|
33
|
+
# # Just here to force Yard to create documentation.
|
34
|
+
# # @!visibility private
|
35
|
+
# def initialize; end
|
36
|
+
# end
|
37
|
+
# end
|
38
|
+
# end
|
39
|
+
|
40
|
+
# RFlow component classes.
|
2
41
|
module Components
|
3
42
|
module File
|
43
|
+
# @!visibility private
|
4
44
|
module Extensions
|
5
45
|
# Need to be careful when extending to not clobber data already in data_object
|
46
|
+
# @!visibility private
|
6
47
|
module FileExtension
|
48
|
+
# @!visibility private
|
7
49
|
def self.extended(base_data)
|
8
50
|
base_data.data_object ||= {
|
9
51
|
'path' => '/', 'size' => 0, 'content' => '',
|
@@ -6,10 +6,18 @@ require 'rflow/components/file/output_to_disk'
|
|
6
6
|
class RFlow
|
7
7
|
module Components
|
8
8
|
module File
|
9
|
+
# Component that receives +RFlow::Message+s of type +RFlow::Message::Data::Raw+
|
10
|
+
# and writes new files to disk whose contents are the raw bytes of the message.
|
9
11
|
class OutputRawToFiles < RFlow::Component
|
10
12
|
include RFlow::Components::File::OutputToDisk
|
13
|
+
|
14
|
+
# Input port where +RFlow::Message+s of type +RFlow::Message::Data::Raw+ are
|
15
|
+
# received. When one is, a new file is written based on the message's properties.
|
16
|
+
# @return [RFlow::Component::InputPort]
|
11
17
|
input_port :raw_port
|
12
18
|
|
19
|
+
# RFlow-called method when a message is received.
|
20
|
+
# @return [void]
|
13
21
|
def process_message(input_port, input_port_key, connection, message)
|
14
22
|
return unless message.data_type_name == 'RFlow::Message::Data::Raw'
|
15
23
|
write_to_file(message.properties) {|file| file.write(message.data.raw) }
|
@@ -3,15 +3,29 @@ require 'uuidtools'
|
|
3
3
|
class RFlow
|
4
4
|
module Components
|
5
5
|
module File
|
6
|
+
# To be included into any component that plans to output files to disk.
|
7
|
+
# Provides default configuration and a {write_to_file} method.
|
8
|
+
#
|
9
|
+
# Best not to depend on the filename other than the prefix and suffix
|
10
|
+
# portions as it is implementation-dependent.
|
11
|
+
#
|
12
|
+
# Mixed-in component will support the following config parameters:
|
13
|
+
# - +directory_path+ - directory to write files into
|
14
|
+
# - +file_name_prefix+ - written files will always begin with this prefix
|
15
|
+
# - +file_name_suffix+ - written files will always end with this suffix
|
6
16
|
module OutputToDisk
|
17
|
+
# Default config.
|
7
18
|
DEFAULT_CONFIG = {
|
8
19
|
'directory_path' => '/tmp',
|
9
20
|
'file_name_prefix' => 'output.',
|
10
21
|
'file_name_suffix' => '.out',
|
11
22
|
}
|
12
23
|
|
24
|
+
# @!visibility private
|
13
25
|
attr_accessor :config, :directory_path, :file_name_prefix, :file_name_suffix
|
14
26
|
|
27
|
+
# RFlow-called method at startup.
|
28
|
+
# @return [void]
|
15
29
|
def configure!(config)
|
16
30
|
@config = DEFAULT_CONFIG.merge config
|
17
31
|
@directory_path = ::File.expand_path(@config['directory_path'])
|
@@ -29,7 +43,14 @@ class RFlow
|
|
29
43
|
# TODO: more error checking of input config
|
30
44
|
end
|
31
45
|
|
32
|
-
#
|
46
|
+
# Write out a file to disk based on message properties. Filename is implementation-dependent
|
47
|
+
# but will certainly contain the data UUID, priority, filename prefix, suffix, and a timestamp
|
48
|
+
# (properties +data_uuid+, +priority+ plus config variables).
|
49
|
+
#
|
50
|
+
# Opens the file and +yield+s it back to the caller for actual writing. Caller should return
|
51
|
+
# the number of bytes written.
|
52
|
+
#
|
53
|
+
# @return [String] the final output file path
|
33
54
|
def write_to_file(properties)
|
34
55
|
properties ||= {}
|
35
56
|
begin
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rflow-components-file
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael L. Artz
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-03
|
11
|
+
date: 2017-10-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rflow
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '10.3'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: yard
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.9'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0.9'
|
69
83
|
description: Components that operate on files for the RFlow FBP framework. Also includes
|
70
84
|
the File schema
|
71
85
|
email:
|
@@ -79,6 +93,7 @@ files:
|
|
79
93
|
- ".ruby-gemset"
|
80
94
|
- ".ruby-version"
|
81
95
|
- ".travis.yml"
|
96
|
+
- ".yardopts"
|
82
97
|
- Gemfile
|
83
98
|
- LICENSE
|
84
99
|
- README.md
|