iostreams 0.19.0 → 0.20.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/io_streams/base_path.rb +72 -0
- data/lib/io_streams/file/path.rb +58 -0
- data/lib/io_streams/file/writer.rb +1 -1
- data/lib/io_streams/gzip/writer.rb +1 -1
- data/lib/io_streams/http/reader.rb +1 -1
- data/lib/io_streams/io_streams.rb +64 -27
- data/lib/io_streams/line/reader.rb +2 -1
- data/lib/io_streams/pgp/reader.rb +29 -23
- data/lib/io_streams/pgp/writer.rb +13 -4
- data/lib/io_streams/s3/path.rb +40 -0
- data/lib/io_streams/s3/reader.rb +2 -1
- data/lib/io_streams/s3.rb +1 -0
- data/lib/io_streams/tabular/header.rb +2 -2
- data/lib/io_streams/tabular/parser/csv.rb +1 -1
- data/lib/io_streams/tabular/utility/csv_row.rb +1 -1
- data/lib/io_streams/tabular.rb +5 -5
- data/lib/io_streams/utils.rb +14 -0
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/xlsx/reader.rb +2 -2
- data/lib/io_streams/zip/reader.rb +1 -1
- data/lib/io_streams/zip/writer.rb +2 -2
- data/lib/iostreams.rb +19 -19
- data/test/base_path_test.rb +35 -0
- data/test/file_path_test.rb +97 -0
- data/test/path_test.rb +16 -37
- data/test/pgp_reader_test.rb +8 -4
- data/test/pgp_writer_test.rb +12 -9
- data/test/test_helper.rb +2 -2
- data/test/xlsx_reader_test.rb +0 -1
- metadata +10 -3
- data/lib/io_streams/path.rb +0 -85
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b16ce2bb98466540950ee6a40c9b3066630e99e7987c93621b3a175b5b027731
|
4
|
+
data.tar.gz: 6fb4dd208cc04a71d78ac1355f8035159a97b67f2a749dc922df3f6edf7208ac
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e58da1ef631c0eae5574194e6586dbf7b52becc934a01a00ede0ef0f665d14f08391a5a2fee645bbe8ffe8fe8452c6d4b79eb84396f1e94e9b98ad9dea229041
|
7
|
+
data.tar.gz: d821299bb5742832125ff2b7a6826f68b734967ea7a5f5d94cf1b619ef3ad82dfd2c457455a5d62523ca52832f5d17b7c2f343f5f387f77abdbeaca6bff42dd7
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module IOStreams
|
3
|
+
class BasePath
|
4
|
+
attr_reader :path
|
5
|
+
|
6
|
+
def initialize(path)
|
7
|
+
@path = path.frozen? ? path : path.dup.freeze
|
8
|
+
end
|
9
|
+
|
10
|
+
# If elements already contains the current path then it is used as is without
|
11
|
+
# adding the current path for a second time
|
12
|
+
def join(*elements)
|
13
|
+
return self if elements.empty?
|
14
|
+
|
15
|
+
relative = ::File.join(*elements)
|
16
|
+
if relative.start_with?(path)
|
17
|
+
self.class.new(relative)
|
18
|
+
else
|
19
|
+
self.class.new(::File.join(path, relative))
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_s
|
24
|
+
path
|
25
|
+
end
|
26
|
+
|
27
|
+
# Removes the last element of the path, the file name, before creating the entire path.
|
28
|
+
# Returns self
|
29
|
+
def mkpath
|
30
|
+
raise NotImplementedError
|
31
|
+
end
|
32
|
+
|
33
|
+
# Assumes the current path does not include a file name, and creates all elements in the path.
|
34
|
+
# Returns self
|
35
|
+
#
|
36
|
+
# Note: Do not call this method if the path contains a file name, see `#mkpath`
|
37
|
+
def mkdir
|
38
|
+
raise NotImplementedError
|
39
|
+
end
|
40
|
+
|
41
|
+
# Returns [true|false] whether the file exists
|
42
|
+
def exist?
|
43
|
+
raise NotImplementedError
|
44
|
+
end
|
45
|
+
|
46
|
+
# Returns [Integer] size of the file
|
47
|
+
def size
|
48
|
+
raise NotImplementedError
|
49
|
+
end
|
50
|
+
|
51
|
+
# Delete the file.
|
52
|
+
# Returns self
|
53
|
+
#
|
54
|
+
# Notes:
|
55
|
+
# * No error is raised if the file is not present.
|
56
|
+
# * Only the file is removed, not any of the parent paths.
|
57
|
+
def delete
|
58
|
+
raise NotImplementedError
|
59
|
+
end
|
60
|
+
|
61
|
+
# Return a reader for this path
|
62
|
+
def reader(**args, &block)
|
63
|
+
IOStreams.reader(path, **args, &block)
|
64
|
+
end
|
65
|
+
|
66
|
+
# Return a writer for this path
|
67
|
+
def writer(**args, &block)
|
68
|
+
IOStreams.writer(path, **args, &block)
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module IOStreams
|
5
|
+
module File
|
6
|
+
class Path < IOStreams::BasePath
|
7
|
+
# Yields the path to a temporary file_name.
|
8
|
+
#
|
9
|
+
# File is deleted upon completion if present.
|
10
|
+
def self.temp_file_name(basename, extension = '')
|
11
|
+
result = nil
|
12
|
+
::Dir::Tmpname.create([basename, extension]) do |tmpname|
|
13
|
+
begin
|
14
|
+
result = yield(tmpname)
|
15
|
+
ensure
|
16
|
+
::File.unlink(tmpname) if ::File.exist?(tmpname)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
result
|
20
|
+
end
|
21
|
+
|
22
|
+
# Used by writers that can write directly to file to create the target path
|
23
|
+
def self.mkpath(path)
|
24
|
+
dir = ::File.dirname(path)
|
25
|
+
FileUtils.mkdir_p(dir) unless ::File.exist?(dir)
|
26
|
+
end
|
27
|
+
|
28
|
+
def mkpath
|
29
|
+
self.class.mkpath(path)
|
30
|
+
self
|
31
|
+
end
|
32
|
+
|
33
|
+
def mkdir
|
34
|
+
FileUtils.mkdir_p(path) unless ::File.exist?(path)
|
35
|
+
self
|
36
|
+
end
|
37
|
+
|
38
|
+
def exist?
|
39
|
+
::File.exist?(path)
|
40
|
+
end
|
41
|
+
|
42
|
+
def size
|
43
|
+
::File.size(path)
|
44
|
+
end
|
45
|
+
|
46
|
+
def delete(recursively: false)
|
47
|
+
return self unless ::File.exist?(path)
|
48
|
+
|
49
|
+
if ::File.directory?(path)
|
50
|
+
recursively ? FileUtils.remove_dir(path) : Dir.delete(path)
|
51
|
+
else
|
52
|
+
::File.unlink(path)
|
53
|
+
end
|
54
|
+
self
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -9,7 +9,7 @@ module IOStreams
|
|
9
9
|
def self.open(file_name, **args, &block)
|
10
10
|
raise(ArgumentError, 'File name must be a string') unless file_name.is_a?(String)
|
11
11
|
|
12
|
-
IOStreams.mkpath(file_name)
|
12
|
+
IOStreams::File::Path.mkpath(file_name)
|
13
13
|
begin
|
14
14
|
::File.open(file_name, 'wb', &block)
|
15
15
|
rescue StandardError => e
|
@@ -4,7 +4,7 @@ module IOStreams
|
|
4
4
|
# Write to a file / stream, compressing with GZip
|
5
5
|
def self.open(file_name_or_io, **args, &block)
|
6
6
|
unless IOStreams.writer_stream?(file_name_or_io)
|
7
|
-
IOStreams.mkpath(file_name_or_io)
|
7
|
+
IOStreams::File::Path.mkpath(file_name_or_io)
|
8
8
|
Zlib::GzipWriter.open(file_name_or_io, &block)
|
9
9
|
else
|
10
10
|
begin
|
@@ -55,7 +55,7 @@ module IOStreams
|
|
55
55
|
raise(IOStreams::Errors::CommunicationsFailure, "Invalid response code: #{response.code}") unless response.is_a?(Net::HTTPSuccess)
|
56
56
|
|
57
57
|
# Since Net::HTTP download only supports a push stream, write it to a tempfile first.
|
58
|
-
IOStreams::Path.temp_file_name('iostreams_http') do |file_name|
|
58
|
+
IOStreams::File::Path.temp_file_name('iostreams_http') do |file_name|
|
59
59
|
IOStreams::File::Writer.open(file_name) do |io|
|
60
60
|
response.read_body { |chunk| io.write(chunk) }
|
61
61
|
end
|
@@ -425,7 +425,52 @@ module IOStreams
|
|
425
425
|
end
|
426
426
|
end
|
427
427
|
|
428
|
-
|
428
|
+
# Return the complete path including supplied elements.
|
429
|
+
#
|
430
|
+
# Example:
|
431
|
+
# IOStreams.path('sample')
|
432
|
+
# # => "/default_root/sample"
|
433
|
+
#
|
434
|
+
# IOStreams.path('file.xls')
|
435
|
+
# # => "/default_root/file.xls"
|
436
|
+
#
|
437
|
+
# IOStreams.path('sample', root: :ftp)
|
438
|
+
# # => "/ftp_root/sample"
|
439
|
+
#
|
440
|
+
# IOStreams.path('sample', 'file.xls', root: :ftp)
|
441
|
+
# # => "/ftp_root/sample/file.xls"
|
442
|
+
#
|
443
|
+
# Notes:
|
444
|
+
# * Add the root path first against which this path is permitted to operate.
|
445
|
+
# `IOStreams.add_root_path(:default, "/usr/local/var/files")`
|
446
|
+
def self.path(*elements, root: :default)
|
447
|
+
root_path(root).join(*elements)
|
448
|
+
end
|
449
|
+
|
450
|
+
# When using a full path, without needing the root prefixed.
|
451
|
+
def self.full_path(*elements)
|
452
|
+
path = ::File.join(*elements)
|
453
|
+
uri = URI.parse(path)
|
454
|
+
IOStreams.scheme(uri.scheme).path_class.new(path)
|
455
|
+
end
|
456
|
+
|
457
|
+
# Return named root path
|
458
|
+
def self.root_path(root = :default)
|
459
|
+
@roots_paths[root.to_sym] || raise(ArgumentError, "Unknown root: #{root.inspect}")
|
460
|
+
end
|
461
|
+
|
462
|
+
# Add a named root path
|
463
|
+
def self.add_root_path(root, path)
|
464
|
+
raise(ArgumentError, "Invalid root name #{root.inspect}") unless root.to_s =~ /\A\w+\Z/
|
465
|
+
|
466
|
+
uri = URI.parse(path.to_s)
|
467
|
+
path_class = IOStreams.scheme(uri.scheme).path_class
|
468
|
+
@roots_paths[root.to_sym] = path_class.new(path)
|
469
|
+
end
|
470
|
+
|
471
|
+
def self.root_paths
|
472
|
+
@roots_paths.dup
|
473
|
+
end
|
429
474
|
|
430
475
|
# Register a file extension and the reader and writer streaming classes
|
431
476
|
#
|
@@ -453,34 +498,23 @@ module IOStreams
|
|
453
498
|
# Example:
|
454
499
|
# # MyXls::Reader and MyXls::Writer must implement .open
|
455
500
|
# register_extension(:xls, MyXls::Reader, MyXls::Writer)
|
456
|
-
def self.register_scheme(scheme, reader_class, writer_class)
|
501
|
+
def self.register_scheme(scheme, reader_class, writer_class, path_class = nil)
|
457
502
|
raise(ArgumentError, "Invalid scheme #{scheme.inspect}") unless scheme.nil? || scheme.to_s =~ /\A\w+\Z/
|
458
|
-
@schemes[scheme.nil? ? nil : scheme.to_sym] =
|
459
|
-
end
|
460
|
-
|
461
|
-
# Helper method: Returns [true|false] if a value is blank?
|
462
|
-
def self.blank?(value)
|
463
|
-
if value.nil?
|
464
|
-
true
|
465
|
-
elsif value.is_a?(String)
|
466
|
-
value !~ /\S/
|
467
|
-
else
|
468
|
-
value.respond_to?(:empty?) ? value.empty? : !value
|
469
|
-
end
|
470
|
-
end
|
471
|
-
|
472
|
-
# Used by writers that can write directly to file to create the target path
|
473
|
-
def self.mkpath(file_name)
|
474
|
-
path = ::File.dirname(file_name)
|
475
|
-
FileUtils.mkdir_p(path) unless ::File.exist?(path)
|
503
|
+
@schemes[scheme.nil? ? nil : scheme.to_sym] = Scheme.new(reader_class, writer_class, path_class)
|
476
504
|
end
|
477
505
|
|
478
506
|
private
|
479
507
|
|
508
|
+
# Hold root paths
|
509
|
+
@roots_paths = {}
|
510
|
+
|
480
511
|
# A registry to hold formats for processing files during upload or download
|
481
512
|
@extensions = {}
|
482
513
|
@schemes = {}
|
483
514
|
|
515
|
+
Extension = Struct.new(:reader_class, :writer_class)
|
516
|
+
Scheme = Struct.new(:reader_class, :writer_class, :path_class)
|
517
|
+
|
484
518
|
# Struct to hold the Stream and options if any
|
485
519
|
StreamStruct = Struct.new(:klass, :options)
|
486
520
|
|
@@ -556,9 +590,12 @@ module IOStreams
|
|
556
590
|
StreamStruct.new(klass, options)
|
557
591
|
end
|
558
592
|
|
559
|
-
def self.
|
560
|
-
|
561
|
-
|
593
|
+
def self.scheme(scheme_name)
|
594
|
+
@schemes[scheme_name.nil? ? nil : scheme_name.to_sym] || raise(ArgumentError, "Unknown Scheme type: #{scheme_name.inspect}")
|
595
|
+
end
|
596
|
+
|
597
|
+
def self.stream_struct_for_scheme(type, scheme_name, options = {})
|
598
|
+
klass = scheme(scheme_name).send("#{type}_class")
|
562
599
|
StreamStruct.new(klass, options)
|
563
600
|
end
|
564
601
|
|
@@ -584,9 +621,9 @@ module IOStreams
|
|
584
621
|
# https://hostname/path/file_name
|
585
622
|
# sftp://hostname/path/file_name
|
586
623
|
# s3://bucket/key
|
587
|
-
register_scheme(nil, IOStreams::File::Reader, IOStreams::File::Writer)
|
588
|
-
register_scheme(:http,
|
624
|
+
register_scheme(nil, IOStreams::File::Reader, IOStreams::File::Writer, IOStreams::File::Path)
|
625
|
+
register_scheme(:http, IOStreams::HTTP::Reader, nil)
|
589
626
|
register_scheme(:https, IOStreams::HTTP::Reader, nil)
|
590
|
-
|
591
|
-
register_scheme(:s3, IOStreams::S3::Reader, IOStreams::S3::Writer)
|
627
|
+
register_scheme(:sftp, IOStreams::SFTP::Reader, IOStreams::SFTP::Writer)
|
628
|
+
register_scheme(:s3, IOStreams::S3::Reader, IOStreams::S3::Writer, IOStreams::S3::Path)
|
592
629
|
end
|
@@ -57,10 +57,11 @@ module IOStreams
|
|
57
57
|
@eof = false
|
58
58
|
@read_cache_buffer = nil
|
59
59
|
@buffer = nil
|
60
|
+
@delimiter = delimiter
|
60
61
|
|
61
62
|
read_block
|
62
63
|
# Auto-detect windows/linux line endings if not supplied. \n or \r\n
|
63
|
-
@delimiter
|
64
|
+
@delimiter ||= auto_detect_line_endings
|
64
65
|
|
65
66
|
if @buffer
|
66
67
|
# Change the delimiters encoding to match that of the input stream
|
@@ -14,31 +14,37 @@ module IOStreams
|
|
14
14
|
# Or, the IO stream to receive the decrypted contents
|
15
15
|
# passphrase: [String]
|
16
16
|
# Pass phrase for private key to decrypt the file with
|
17
|
-
def self.open(file_name_or_io, passphrase: self.default_passphrase, binary: true)
|
17
|
+
def self.open(file_name_or_io, passphrase: self.default_passphrase, binary: true, &block)
|
18
18
|
raise(ArgumentError, 'Missing both passphrase and IOStreams::Pgp::Reader.default_passphrase') unless passphrase
|
19
19
|
|
20
|
-
if
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
20
|
+
return read_file(file_name_or_io, passphrase: passphrase, binary: binary, &block) if file_name_or_io.is_a?(String)
|
21
|
+
|
22
|
+
# PGP can only work against a file, not a stream, so create temp file.
|
23
|
+
IOStreams::File::Path.temp_file_name('iostreams_pgp') do |temp_file_name|
|
24
|
+
IOStreams.copy(file_name_or_io, temp_file_name, target_options: {streams: []})
|
25
|
+
read_file(temp_file_name, passphrase: passphrase, binary: binary, &block)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.read_file(file_name, passphrase: self.default_passphrase, binary: true)
|
30
|
+
loopback = IOStreams::Pgp.pgp_version.to_f >= 2.1 ? '--pinentry-mode loopback' : ''
|
31
|
+
command = "#{IOStreams::Pgp.executable} #{loopback} --batch --no-tty --yes --decrypt --passphrase-fd 0 #{file_name}"
|
32
|
+
IOStreams::Pgp.logger.debug { "IOStreams::Pgp::Reader.open: #{command}" } if IOStreams::Pgp.logger
|
33
|
+
|
34
|
+
# Read decrypted contents from stdout
|
35
|
+
Open3.popen3(command) do |stdin, stdout, stderr, waith_thr|
|
36
|
+
stdin.puts(passphrase) if passphrase
|
37
|
+
stdin.close
|
38
|
+
result =
|
39
|
+
begin
|
40
|
+
stdout.binmode if binary
|
41
|
+
yield(stdout)
|
42
|
+
rescue Errno::EPIPE
|
43
|
+
# Ignore broken pipe because gpg terminates early due to an error
|
44
|
+
raise(Pgp::Failure, "GPG Failed reading from encrypted file: #{file_name}: #{stderr.read.chomp}")
|
45
|
+
end
|
46
|
+
raise(Pgp::Failure, "GPG Failed to decrypt file: #{file_name}: #{stderr.read.chomp}") unless waith_thr.value.success?
|
47
|
+
result
|
42
48
|
end
|
43
49
|
end
|
44
50
|
|
@@ -45,13 +45,22 @@ module IOStreams
|
|
45
45
|
# compress_level: [Integer]
|
46
46
|
# Compression level
|
47
47
|
# Default: 6
|
48
|
-
def self.open(
|
48
|
+
def self.open(file_name_or_io, recipient:, signer: default_signer, signer_passphrase: default_signer_passphrase, binary: true, compression: :zip, compress_level: 6, &block)
|
49
49
|
compress_level = 0 if compression == :none
|
50
|
-
|
51
|
-
|
50
|
+
|
51
|
+
if file_name_or_io.is_a?(String)
|
52
|
+
IOStreams::File::Path.mkpath(file_name_or_io)
|
53
|
+
return write_file(file_name_or_io, recipient: recipient, signer: signer, signer_passphrase: signer_passphrase, binary: binary, compression: compression, compress_level: compress_level, &block)
|
54
|
+
end
|
55
|
+
|
56
|
+
# PGP can only work against a file, not a stream, so create temp file.
|
57
|
+
IOStreams::File::Path.temp_file_name('iostreams_pgp') do |temp_file_name|
|
58
|
+
write_file(temp_file_name, recipient: recipient, signer: signer, signer_passphrase: signer_passphrase, binary: binary, compression: compression, compress_level: compress_level, &block)
|
59
|
+
IOStreams.copy(temp_file_name, file_name_or_io, source_options: {streams: []})
|
52
60
|
end
|
53
|
-
|
61
|
+
end
|
54
62
|
|
63
|
+
def self.write_file(file_name, recipient:, signer: default_signer, signer_passphrase: default_signer_passphrase, binary: true, compression: :zip, compress_level: 6)
|
55
64
|
# Write to stdin, with encrypted contents being written to the file
|
56
65
|
command = "#{IOStreams::Pgp.executable} --batch --no-tty --yes --encrypt"
|
57
66
|
command << " --sign --local-user \"#{signer}\"" if signer
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module IOStreams
|
3
|
+
module S3
|
4
|
+
class Path < IOStreams::BasePath
|
5
|
+
def initialize(path)
|
6
|
+
IOStreams::S3.load_dependencies
|
7
|
+
@s3 = Aws::S3::Resource.new
|
8
|
+
@options = IOStreams::S3.parse_uri(path)
|
9
|
+
@object = s3.bucket(options[:bucket]).object(options[:key])
|
10
|
+
super(path)
|
11
|
+
end
|
12
|
+
|
13
|
+
# S3 logically creates paths when a key is set.
|
14
|
+
def mkpath
|
15
|
+
self
|
16
|
+
end
|
17
|
+
|
18
|
+
def mkdir
|
19
|
+
self
|
20
|
+
end
|
21
|
+
|
22
|
+
def exist?
|
23
|
+
object.exists?
|
24
|
+
end
|
25
|
+
|
26
|
+
def size
|
27
|
+
object.size
|
28
|
+
end
|
29
|
+
|
30
|
+
def delete
|
31
|
+
object.delete
|
32
|
+
self
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
attr_reader :s3, :options, :object
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/lib/io_streams/s3/reader.rb
CHANGED
@@ -7,13 +7,14 @@ module IOStreams
|
|
7
7
|
|
8
8
|
IOStreams::S3.load_dependencies
|
9
9
|
|
10
|
+
# https://aws.amazon.com/blogs/developer/using-resources/
|
10
11
|
s3 = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
|
11
12
|
options = IOStreams::S3.parse_uri(uri)
|
12
13
|
object = s3.bucket(options[:bucket]).object(options[:key])
|
13
14
|
|
14
15
|
begin
|
15
16
|
# Since S3 download only supports a push stream, write it to a tempfile first.
|
16
|
-
IOStreams::Path.temp_file_name('iostreams_s3') do |file_name|
|
17
|
+
IOStreams::File::Path.temp_file_name('iostreams_s3') do |file_name|
|
17
18
|
args[:response_target] = file_name
|
18
19
|
object.get(args)
|
19
20
|
|
data/lib/io_streams/s3.rb
CHANGED
@@ -86,7 +86,7 @@ module IOStreams
|
|
86
86
|
# Only Applies to when the hash is already a Hash.
|
87
87
|
# Useful to turn off narrowing when the input data is already trusted.
|
88
88
|
def to_hash(row, cleanse = true)
|
89
|
-
return if IOStreams.blank?(row)
|
89
|
+
return if IOStreams::Utils.blank?(row)
|
90
90
|
|
91
91
|
case row
|
92
92
|
when Array
|
@@ -112,7 +112,7 @@ module IOStreams
|
|
112
112
|
|
113
113
|
def array_to_hash(row)
|
114
114
|
h = {}
|
115
|
-
columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams.blank?(col) }
|
115
|
+
columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams::Utils.blank?(col) }
|
116
116
|
h
|
117
117
|
end
|
118
118
|
|
@@ -40,7 +40,7 @@ module IOStreams
|
|
40
40
|
# About 10 times slower than the approach used in Ruby 2.5 and earlier,
|
41
41
|
# but at least it works on Ruby 2.6 and above.
|
42
42
|
def parse_line(line)
|
43
|
-
return if IOStreams.blank?(line)
|
43
|
+
return if IOStreams::Utils.blank?(line)
|
44
44
|
|
45
45
|
CSV.parse_line(line)
|
46
46
|
end
|
@@ -23,7 +23,7 @@ module IOStreams
|
|
23
23
|
# line [String]
|
24
24
|
# A single line of CSV data without any line terminators
|
25
25
|
def parse(line)
|
26
|
-
return if IOStreams.blank?(line)
|
26
|
+
return if IOStreams::Utils.blank?(line)
|
27
27
|
return if @skip_lines and @skip_lines.match line
|
28
28
|
|
29
29
|
in_extended_col = false
|
data/lib/io_streams/tabular.rb
CHANGED
@@ -66,7 +66,7 @@ module IOStreams
|
|
66
66
|
|
67
67
|
# Returns [true|false] whether a header is still required in order to parse or render the current format.
|
68
68
|
def header?
|
69
|
-
parser.requires_header? && IOStreams.blank?(header.columns)
|
69
|
+
parser.requires_header? && IOStreams::Utils.blank?(header.columns)
|
70
70
|
end
|
71
71
|
|
72
72
|
# Returns [true|false] whether a header row show be rendered on output.
|
@@ -81,7 +81,7 @@ module IOStreams
|
|
81
81
|
# * Call `header?` first to determine if the header should be parsed first.
|
82
82
|
# * The header columns are set after parsing the row, but the header is not cleansed.
|
83
83
|
def parse_header(line)
|
84
|
-
return if IOStreams.blank?(line) || !parser.requires_header?
|
84
|
+
return if IOStreams::Utils.blank?(line) || !parser.requires_header?
|
85
85
|
|
86
86
|
header.columns = parser.parse(line)
|
87
87
|
end
|
@@ -96,14 +96,14 @@ module IOStreams
|
|
96
96
|
# Returns [Array] the row/line as a parsed Array of values.
|
97
97
|
# Returns nil if the row/line is blank.
|
98
98
|
def row_parse(line)
|
99
|
-
return if IOStreams.blank?(line)
|
99
|
+
return if IOStreams::Utils.blank?(line)
|
100
100
|
|
101
101
|
parser.parse(line)
|
102
102
|
end
|
103
103
|
|
104
104
|
# Renders the output row
|
105
105
|
def render(row)
|
106
|
-
return if IOStreams.blank?(row)
|
106
|
+
return if IOStreams::Utils.blank?(row)
|
107
107
|
|
108
108
|
parser.render(row, header)
|
109
109
|
end
|
@@ -113,7 +113,7 @@ module IOStreams
|
|
113
113
|
def render_header
|
114
114
|
return unless requires_header?
|
115
115
|
|
116
|
-
if IOStreams.blank?(header.columns)
|
116
|
+
if IOStreams::Utils.blank?(header.columns)
|
117
117
|
raise(Errors::MissingHeader, "Header columns must be set before attempting to render a header for format: #{format.inspect}")
|
118
118
|
end
|
119
119
|
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module IOStreams
|
2
|
+
module Utils
|
3
|
+
# Helper method: Returns [true|false] if a value is blank?
|
4
|
+
def self.blank?(value)
|
5
|
+
if value.nil?
|
6
|
+
true
|
7
|
+
elsif value.is_a?(String)
|
8
|
+
value !~ /\S/
|
9
|
+
else
|
10
|
+
value.respond_to?(:empty?) ? value.empty? : !value
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/io_streams/version.rb
CHANGED
@@ -8,7 +8,7 @@ module IOStreams
|
|
8
8
|
return extract_csv(file_name_or_io, &block) if file_name_or_io.is_a?(String)
|
9
9
|
|
10
10
|
# Creek gem can only work against a file, not a stream, so create temp file.
|
11
|
-
IOStreams::Path.temp_file_name('iostreams_xlsx') do |temp_file_name|
|
11
|
+
IOStreams::File::Path.temp_file_name('iostreams_xlsx') do |temp_file_name|
|
12
12
|
IOStreams.copy(file_name_or_io, temp_file_name, target_options: {streams: []})
|
13
13
|
extract_csv(temp_file_name, &block)
|
14
14
|
end
|
@@ -16,7 +16,7 @@ module IOStreams
|
|
16
16
|
|
17
17
|
# Convert the spreadsheet to csv in a tempfile
|
18
18
|
def self.extract_csv(file_name, &block)
|
19
|
-
IOStreams::Path.temp_file_name('iostreams_csv') do |temp_file_name|
|
19
|
+
IOStreams::File::Path.temp_file_name('iostreams_csv') do |temp_file_name|
|
20
20
|
IOStreams::File::Writer.open(temp_file_name) do |io|
|
21
21
|
new(file_name).each { |lines| io << lines.to_csv }
|
22
22
|
end
|
@@ -26,7 +26,7 @@ module IOStreams
|
|
26
26
|
return read_file(file_name_or_io, &block) unless IOStreams.reader_stream?(file_name_or_io)
|
27
27
|
|
28
28
|
# ZIP can only work against a file, not a stream, so create temp file.
|
29
|
-
IOStreams::Path.temp_file_name('iostreams_zip') do |temp_file_name|
|
29
|
+
IOStreams::File::Path.temp_file_name('iostreams_zip') do |temp_file_name|
|
30
30
|
IOStreams.copy(file_name_or_io, temp_file_name, target_options: {streams: []})
|
31
31
|
read_file(temp_file_name, &block)
|
32
32
|
end
|
@@ -39,7 +39,7 @@ module IOStreams
|
|
39
39
|
return write_file(file_name_or_io, zip_file_name, &block) unless IOStreams.writer_stream?(file_name_or_io)
|
40
40
|
|
41
41
|
# ZIP can only work against a file, not a stream, so create temp file.
|
42
|
-
IOStreams::Path.temp_file_name('iostreams_zip') do |temp_file_name|
|
42
|
+
IOStreams::File::Path.temp_file_name('iostreams_zip') do |temp_file_name|
|
43
43
|
write_file(temp_file_name, zip_file_name, &block)
|
44
44
|
IOStreams.copy(temp_file_name, file_name_or_io, source_options: {streams: []})
|
45
45
|
end
|
@@ -62,7 +62,7 @@ module IOStreams
|
|
62
62
|
|
63
63
|
else
|
64
64
|
def self.write_file(file_name, zip_file_name, &block)
|
65
|
-
IOStreams.mkpath(file_name)
|
65
|
+
IOStreams::File::Path.mkpath(file_name)
|
66
66
|
zos = ::Zip::OutputStream.new(file_name)
|
67
67
|
zos.put_next_entry(zip_file_name)
|
68
68
|
block.call(zos)
|
data/lib/iostreams.rb
CHANGED
@@ -1,13 +1,23 @@
|
|
1
1
|
require 'io_streams/version'
|
2
2
|
#@formatter:off
|
3
3
|
module IOStreams
|
4
|
-
autoload :
|
4
|
+
autoload :BasePath, 'io_streams/base_path'
|
5
|
+
autoload :Errors, 'io_streams/errors'
|
6
|
+
autoload :Pgp, 'io_streams/pgp'
|
7
|
+
autoload :S3, 'io_streams/s3'
|
8
|
+
autoload :Tabular, 'io_streams/tabular'
|
9
|
+
autoload :Utils, 'io_streams/utils'
|
5
10
|
|
6
11
|
module Bzip2
|
7
12
|
autoload :Reader, 'io_streams/bzip2/reader'
|
8
13
|
autoload :Writer, 'io_streams/bzip2/writer'
|
9
14
|
end
|
15
|
+
module Encode
|
16
|
+
autoload :Reader, 'io_streams/encode/reader'
|
17
|
+
autoload :Writer, 'io_streams/encode/writer'
|
18
|
+
end
|
10
19
|
module File
|
20
|
+
autoload :Path, 'io_streams/file/path'
|
11
21
|
autoload :Reader, 'io_streams/file/reader'
|
12
22
|
autoload :Writer, 'io_streams/file/writer'
|
13
23
|
end
|
@@ -18,22 +28,6 @@ module IOStreams
|
|
18
28
|
module HTTP
|
19
29
|
autoload :Reader, 'io_streams/http/reader'
|
20
30
|
end
|
21
|
-
autoload :Path, 'io_streams/path'
|
22
|
-
autoload :Pgp, 'io_streams/pgp'
|
23
|
-
autoload :S3, 'io_streams/s3'
|
24
|
-
module SFTP
|
25
|
-
autoload :Reader, 'io_streams/sftp/reader'
|
26
|
-
autoload :Writer, 'io_streams/sftp/writer'
|
27
|
-
end
|
28
|
-
module Zip
|
29
|
-
autoload :Reader, 'io_streams/zip/reader'
|
30
|
-
autoload :Writer, 'io_streams/zip/writer'
|
31
|
-
end
|
32
|
-
|
33
|
-
module Encode
|
34
|
-
autoload :Reader, 'io_streams/encode/reader'
|
35
|
-
autoload :Writer, 'io_streams/encode/writer'
|
36
|
-
end
|
37
31
|
module Line
|
38
32
|
autoload :Reader, 'io_streams/line/reader'
|
39
33
|
autoload :Writer, 'io_streams/line/writer'
|
@@ -46,6 +40,10 @@ module IOStreams
|
|
46
40
|
autoload :Reader, 'io_streams/row/reader'
|
47
41
|
autoload :Writer, 'io_streams/row/writer'
|
48
42
|
end
|
43
|
+
module SFTP
|
44
|
+
autoload :Reader, 'io_streams/sftp/reader'
|
45
|
+
autoload :Writer, 'io_streams/sftp/writer'
|
46
|
+
end
|
49
47
|
module SymmetricEncryption
|
50
48
|
autoload :Reader, 'io_streams/symmetric_encryption/reader'
|
51
49
|
autoload :Writer, 'io_streams/symmetric_encryption/writer'
|
@@ -53,7 +51,9 @@ module IOStreams
|
|
53
51
|
module Xlsx
|
54
52
|
autoload :Reader, 'io_streams/xlsx/reader'
|
55
53
|
end
|
56
|
-
|
57
|
-
|
54
|
+
module Zip
|
55
|
+
autoload :Reader, 'io_streams/zip/reader'
|
56
|
+
autoload :Writer, 'io_streams/zip/writer'
|
57
|
+
end
|
58
58
|
end
|
59
59
|
require 'io_streams/io_streams'
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
|
3
|
+
module IOStreams
|
4
|
+
class BasePathTest < Minitest::Test
|
5
|
+
describe IOStreams::BasePath do
|
6
|
+
describe '.join' do
|
7
|
+
let(:path) { IOStreams::BasePath.new('some_path') }
|
8
|
+
|
9
|
+
it 'returns self when no elements' do
|
10
|
+
assert_equal path.object_id, path.join.object_id
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'adds element to path' do
|
14
|
+
assert_equal ::File.join('some_path', 'test'), path.join('test').to_s
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'adds paths to root' do
|
18
|
+
assert_equal ::File.join('some_path', 'test', 'second', 'third'), path.join('test', 'second', 'third').to_s
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'returns path and filename' do
|
22
|
+
assert_equal ::File.join('some_path', 'file.xls'), path.join('file.xls').to_s
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'adds elements to path' do
|
26
|
+
assert_equal ::File.join('some_path', 'test', 'second', 'third', 'file.xls'), path.join('test', 'second', 'third', 'file.xls').to_s
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'return path as sent in when full path' do
|
30
|
+
assert_equal ::File.join('some_path', 'test', 'second', 'third', 'file.xls'), path.join('some_path', 'test', 'second', 'third', 'file.xls').to_s
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
|
3
|
+
module IOStreams
|
4
|
+
class FilePathTest < Minitest::Test
|
5
|
+
describe IOStreams::File::Path do
|
6
|
+
let(:path) { IOStreams::File::Path.new('/tmp/iostreams/some_test_path').delete(recursively: true) }
|
7
|
+
let(:path_with_file_name) { IOStreams::File::Path.new('/tmp/iostreams/some_test_path/test_file.txt').delete }
|
8
|
+
|
9
|
+
describe '.temp_file_name' do
|
10
|
+
it 'returns value from block' do
|
11
|
+
result = IOStreams::File::Path.temp_file_name('base', '.ext') { |name| 257 }
|
12
|
+
assert_equal 257, result
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'supplies new temp file_name' do
|
16
|
+
file_name = nil
|
17
|
+
file_name2 = nil
|
18
|
+
IOStreams::File::Path.temp_file_name('base', '.ext') { |name| file_name = name }
|
19
|
+
IOStreams::File::Path.temp_file_name('base', '.ext') { |name| file_name2 = name }
|
20
|
+
refute_equal file_name, file_name2
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
describe '.mkpath' do
|
25
|
+
it 'makes path skipping file_name' do
|
26
|
+
new_path = path.join('test_mkpath.xls')
|
27
|
+
IOStreams::File::Path.mkpath(new_path.to_s)
|
28
|
+
|
29
|
+
assert ::File.exist?(path.to_s)
|
30
|
+
refute ::File.exist?(new_path.to_s)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
describe '#mkpath' do
|
35
|
+
it 'makes path skipping file_name' do
|
36
|
+
#path.writer { |io| io << "Hello World" }
|
37
|
+
new_path = path.join('test_mkpath.xls').mkpath
|
38
|
+
assert ::File.exist?(path.to_s)
|
39
|
+
refute ::File.exist?(new_path.to_s)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
describe '#mkdir' do
|
44
|
+
it 'makes entire path that does not have a file name' do
|
45
|
+
new_path = path.join('more_path').mkdir
|
46
|
+
assert ::File.exist?(path.to_s)
|
47
|
+
assert ::File.exist?(new_path.to_s)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe '#exist?' do
|
52
|
+
it 'true on existing file' do
|
53
|
+
new_path = path.join('test_exist.txt')
|
54
|
+
new_path.writer { |io| io << "Hello World" }
|
55
|
+
assert ::File.exist?(path.to_s)
|
56
|
+
assert ::File.exist?(new_path.to_s)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
describe '#size' do
|
61
|
+
it 'of file' do
|
62
|
+
data = "Hello World"
|
63
|
+
path_with_file_name.writer { |io| io << data }
|
64
|
+
assert_equal data.size, path_with_file_name.size
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
describe '#delete' do
|
69
|
+
it 'deletes existing file' do
|
70
|
+
path_with_file_name.writer { |io| io << "Hello World" }
|
71
|
+
assert ::File.exist?(path_with_file_name.to_s)
|
72
|
+
path_with_file_name.delete
|
73
|
+
refute ::File.exist?(path_with_file_name.to_s)
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'ignores missing file' do
|
77
|
+
path_with_file_name.delete
|
78
|
+
path_with_file_name.delete
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
describe 'reader' do
|
83
|
+
it 'reads file' do
|
84
|
+
path_with_file_name.writer { |io| io << "Hello World" }
|
85
|
+
assert_equal "Hello World", path_with_file_name.reader(&:read)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
describe 'writer' do
|
90
|
+
it 'creates file' do
|
91
|
+
path_with_file_name.writer { |io| io << "Hello World" }
|
92
|
+
assert ::File.exist?(path_with_file_name.to_s)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
data/test/path_test.rb
CHANGED
@@ -2,73 +2,52 @@ require_relative 'test_helper'
|
|
2
2
|
|
3
3
|
module IOStreams
|
4
4
|
class PathTest < Minitest::Test
|
5
|
-
describe IOStreams
|
6
|
-
describe '.
|
5
|
+
describe IOStreams do
|
6
|
+
describe '.root_path' do
|
7
7
|
it 'return default path' do
|
8
8
|
path = ::File.expand_path(::File.join(__dir__, '../tmp/default'))
|
9
|
-
assert_equal path, IOStreams
|
9
|
+
assert_equal path, IOStreams.root_path.to_s
|
10
10
|
end
|
11
11
|
|
12
12
|
it 'return downloads path' do
|
13
13
|
path = ::File.expand_path(::File.join(__dir__, '../tmp/downloads'))
|
14
|
-
assert_equal path, IOStreams
|
14
|
+
assert_equal path, IOStreams.root_path(:downloads).to_s
|
15
15
|
end
|
16
16
|
end
|
17
17
|
|
18
|
-
describe '.
|
18
|
+
describe '.path' do
|
19
19
|
it 'returns path' do
|
20
|
-
assert_equal IOStreams
|
20
|
+
assert_equal IOStreams.root_path.to_s, IOStreams.path.to_s
|
21
21
|
end
|
22
22
|
|
23
23
|
it 'adds path to root' do
|
24
|
-
assert_equal ::File.join(IOStreams
|
24
|
+
assert_equal ::File.join(IOStreams.root_path.to_s, 'test'), IOStreams.path('test').to_s
|
25
25
|
end
|
26
26
|
|
27
27
|
it 'adds paths to root' do
|
28
|
-
assert_equal ::File.join(IOStreams
|
28
|
+
assert_equal ::File.join(IOStreams.root_path.to_s, 'test', 'second', 'third'), IOStreams.path('test', 'second', 'third').to_s
|
29
29
|
end
|
30
30
|
|
31
31
|
it 'returns path and filename' do
|
32
|
-
path = ::File.join(IOStreams
|
33
|
-
assert_equal path, IOStreams
|
32
|
+
path = ::File.join(IOStreams.root_path.to_s, 'file.xls')
|
33
|
+
assert_equal path, IOStreams.path('file.xls').to_s
|
34
34
|
end
|
35
35
|
|
36
36
|
it 'adds path to root and filename' do
|
37
|
-
path = ::File.join(IOStreams
|
38
|
-
assert_equal path, IOStreams
|
37
|
+
path = ::File.join(IOStreams.root_path.to_s, 'test', 'file.xls')
|
38
|
+
assert_equal path, IOStreams.path('test', 'file.xls').to_s
|
39
39
|
end
|
40
40
|
|
41
41
|
it 'adds paths to root' do
|
42
|
-
path = ::File.join(IOStreams
|
43
|
-
assert_equal path, IOStreams
|
42
|
+
path = ::File.join(IOStreams.root_path.to_s, 'test', 'second', 'third', 'file.xls')
|
43
|
+
assert_equal path, IOStreams.path('test', 'second', 'third', 'file.xls').to_s
|
44
44
|
end
|
45
45
|
|
46
46
|
it 'return path as sent in when full path' do
|
47
|
-
path = ::File.join(IOStreams
|
48
|
-
assert_equal path, IOStreams
|
47
|
+
path = ::File.join(IOStreams.root_path.to_s, 'file.xls')
|
48
|
+
assert_equal path, IOStreams.path(path).to_s
|
49
49
|
end
|
50
50
|
end
|
51
|
-
|
52
|
-
describe '.mkpath' do
|
53
|
-
it 'makes root' do
|
54
|
-
path = IOStreams::Path.new('test.xls')
|
55
|
-
assert_equal path, path.mkpath
|
56
|
-
assert ::File.exist?(IOStreams::Path.new.to_s)
|
57
|
-
end
|
58
|
-
|
59
|
-
it 'makes root with path' do
|
60
|
-
path = IOStreams::Path.new('test', 'test.xls')
|
61
|
-
assert_equal path, path.mkpath
|
62
|
-
assert ::File.exist?(IOStreams::Path.new('test').to_s)
|
63
|
-
end
|
64
|
-
|
65
|
-
it 'makes root with paths' do
|
66
|
-
path = IOStreams::Path.new('test', 'second', 'third', 'test.xls')
|
67
|
-
assert_equal path, path.mkpath
|
68
|
-
assert ::File.exist?(IOStreams::Path.new('test', 'second', 'third').to_s)
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
51
|
end
|
73
52
|
end
|
74
53
|
end
|
data/test/pgp_reader_test.rb
CHANGED
@@ -31,11 +31,15 @@ class PgpReaderTest < Minitest::Test
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
34
|
-
it '
|
35
|
-
|
36
|
-
|
37
|
-
IOStreams::Pgp::Reader.open(io, passphrase: 'BAD') { |file| file.read }
|
34
|
+
it 'streams input' do
|
35
|
+
IOStreams::Pgp::Writer.open(temp_file.path, recipient: 'receiver@example.org') do |io|
|
36
|
+
io.write(decrypted)
|
38
37
|
end
|
38
|
+
|
39
|
+
encrypted = IOStreams::File::Reader.open(temp_file.path, &:read)
|
40
|
+
io = StringIO.new(encrypted)
|
41
|
+
result = IOStreams::Pgp::Reader.open(io, passphrase: 'receiver_passphrase', &:read)
|
42
|
+
assert_equal decrypted, result
|
39
43
|
end
|
40
44
|
|
41
45
|
end
|
data/test/pgp_writer_test.rb
CHANGED
@@ -25,7 +25,7 @@ class PgpWriterTest < Minitest::Test
|
|
25
25
|
io.write(decrypted)
|
26
26
|
end
|
27
27
|
|
28
|
-
result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase', binary: false
|
28
|
+
result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase', binary: false, &:read)
|
29
29
|
assert_equal decrypted, result
|
30
30
|
end
|
31
31
|
|
@@ -39,7 +39,7 @@ class PgpWriterTest < Minitest::Test
|
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
-
result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase'
|
42
|
+
result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase', &:read)
|
43
43
|
assert_equal binary_data, result
|
44
44
|
end
|
45
45
|
|
@@ -48,7 +48,7 @@ class PgpWriterTest < Minitest::Test
|
|
48
48
|
io.write(decrypted)
|
49
49
|
end
|
50
50
|
|
51
|
-
result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase'
|
51
|
+
result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase', &:read)
|
52
52
|
assert_equal decrypted, result
|
53
53
|
end
|
54
54
|
|
@@ -80,14 +80,17 @@ class PgpWriterTest < Minitest::Test
|
|
80
80
|
end
|
81
81
|
end
|
82
82
|
|
83
|
-
it '
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
io.write(decrypted)
|
88
|
-
end
|
83
|
+
it 'writes to a stream' do
|
84
|
+
io_string = StringIO.new(''.b)
|
85
|
+
IOStreams::Pgp::Writer.open(io_string, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
|
86
|
+
io.write(decrypted)
|
89
87
|
end
|
88
|
+
|
89
|
+
io = StringIO.new(io_string.string)
|
90
|
+
result = IOStreams::Pgp::Reader.open(io, passphrase: 'receiver_passphrase', &:read)
|
91
|
+
assert_equal decrypted, result
|
90
92
|
end
|
93
|
+
|
91
94
|
end
|
92
95
|
end
|
93
96
|
end
|
data/test/test_helper.rb
CHANGED
@@ -35,5 +35,5 @@ end
|
|
35
35
|
|
36
36
|
# Test paths
|
37
37
|
root = File.expand_path(File.join(__dir__, '../tmp'))
|
38
|
-
IOStreams
|
39
|
-
IOStreams
|
38
|
+
IOStreams.add_root_path(:default, File.join(root, 'default'))
|
39
|
+
IOStreams.add_root_path(:downloads, File.join(root, 'downloads'))
|
data/test/xlsx_reader_test.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: iostreams
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.20.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-08-
|
11
|
+
date: 2019-08-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -34,11 +34,13 @@ files:
|
|
34
34
|
- LICENSE
|
35
35
|
- README.md
|
36
36
|
- Rakefile
|
37
|
+
- lib/io_streams/base_path.rb
|
37
38
|
- lib/io_streams/bzip2/reader.rb
|
38
39
|
- lib/io_streams/bzip2/writer.rb
|
39
40
|
- lib/io_streams/encode/reader.rb
|
40
41
|
- lib/io_streams/encode/writer.rb
|
41
42
|
- lib/io_streams/errors.rb
|
43
|
+
- lib/io_streams/file/path.rb
|
42
44
|
- lib/io_streams/file/reader.rb
|
43
45
|
- lib/io_streams/file/writer.rb
|
44
46
|
- lib/io_streams/gzip/reader.rb
|
@@ -47,7 +49,6 @@ files:
|
|
47
49
|
- lib/io_streams/io_streams.rb
|
48
50
|
- lib/io_streams/line/reader.rb
|
49
51
|
- lib/io_streams/line/writer.rb
|
50
|
-
- lib/io_streams/path.rb
|
51
52
|
- lib/io_streams/pgp.rb
|
52
53
|
- lib/io_streams/pgp/reader.rb
|
53
54
|
- lib/io_streams/pgp/writer.rb
|
@@ -56,6 +57,7 @@ files:
|
|
56
57
|
- lib/io_streams/row/reader.rb
|
57
58
|
- lib/io_streams/row/writer.rb
|
58
59
|
- lib/io_streams/s3.rb
|
60
|
+
- lib/io_streams/s3/path.rb
|
59
61
|
- lib/io_streams/s3/reader.rb
|
60
62
|
- lib/io_streams/s3/writer.rb
|
61
63
|
- lib/io_streams/sftp/reader.rb
|
@@ -72,15 +74,18 @@ files:
|
|
72
74
|
- lib/io_streams/tabular/parser/json.rb
|
73
75
|
- lib/io_streams/tabular/parser/psv.rb
|
74
76
|
- lib/io_streams/tabular/utility/csv_row.rb
|
77
|
+
- lib/io_streams/utils.rb
|
75
78
|
- lib/io_streams/version.rb
|
76
79
|
- lib/io_streams/xlsx/reader.rb
|
77
80
|
- lib/io_streams/zip/reader.rb
|
78
81
|
- lib/io_streams/zip/writer.rb
|
79
82
|
- lib/iostreams.rb
|
83
|
+
- test/base_path_test.rb
|
80
84
|
- test/bzip2_reader_test.rb
|
81
85
|
- test/bzip2_writer_test.rb
|
82
86
|
- test/encode_reader_test.rb
|
83
87
|
- test/encode_writer_test.rb
|
88
|
+
- test/file_path_test.rb
|
84
89
|
- test/file_reader_test.rb
|
85
90
|
- test/file_writer_test.rb
|
86
91
|
- test/files/embedded_lines_test.csv
|
@@ -139,6 +144,7 @@ specification_version: 4
|
|
139
144
|
summary: Input and Output streaming for Ruby.
|
140
145
|
test_files:
|
141
146
|
- test/pgp_reader_test.rb
|
147
|
+
- test/base_path_test.rb
|
142
148
|
- test/line_reader_test.rb
|
143
149
|
- test/xlsx_reader_test.rb
|
144
150
|
- test/row_writer_test.rb
|
@@ -175,3 +181,4 @@ test_files:
|
|
175
181
|
- test/io_streams_test.rb
|
176
182
|
- test/record_writer_test.rb
|
177
183
|
- test/s3_reader_test.rb
|
184
|
+
- test/file_path_test.rb
|
data/lib/io_streams/path.rb
DELETED
@@ -1,85 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
require 'fileutils'
|
3
|
-
|
4
|
-
module IOStreams
|
5
|
-
#
|
6
|
-
# NOTE: This is a proof of concept class and will change significantly.
|
7
|
-
# I.e. Dont use it yet.
|
8
|
-
#
|
9
|
-
class Path
|
10
|
-
attr_reader :root, :relative
|
11
|
-
|
12
|
-
# Return named root path
|
13
|
-
def self.[](root)
|
14
|
-
@roots[root.to_sym] || raise(ArgumentError, "Unknown root: #{root.inspect}")
|
15
|
-
end
|
16
|
-
|
17
|
-
# Add a named root path
|
18
|
-
def self.add_root(root, path)
|
19
|
-
@roots[root.to_sym] = path.dup.freeze
|
20
|
-
end
|
21
|
-
|
22
|
-
def self.roots
|
23
|
-
@roots.dup
|
24
|
-
end
|
25
|
-
|
26
|
-
# Yields the path to a temporary file_name.
|
27
|
-
#
|
28
|
-
# File is deleted upon completion if present.
|
29
|
-
def self.temp_file_name(basename, extension = '')
|
30
|
-
result = nil
|
31
|
-
::Dir::Tmpname.create([basename, extension]) do |tmpname|
|
32
|
-
begin
|
33
|
-
result = yield(tmpname)
|
34
|
-
ensure
|
35
|
-
::File.unlink(tmpname) if ::File.exist?(tmpname)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
result
|
39
|
-
end
|
40
|
-
|
41
|
-
def initialize(*elements, root: :default)
|
42
|
-
@root = root.to_sym
|
43
|
-
root_path = self.class[@root]
|
44
|
-
if elements.empty?
|
45
|
-
@relative = ''
|
46
|
-
@path = root_path
|
47
|
-
else
|
48
|
-
@relative = ::File.join(*elements).freeze
|
49
|
-
if @relative.start_with?(root_path)
|
50
|
-
@path = @relative
|
51
|
-
@relative = @path[root_path.size + 1..-1].freeze
|
52
|
-
else
|
53
|
-
@path = ::File.join(root_path, @relative).freeze
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def to_s
|
59
|
-
@path
|
60
|
-
end
|
61
|
-
|
62
|
-
# Creates the entire path excluding the file_name.
|
63
|
-
def mkpath
|
64
|
-
path = ::File.dirname(@path)
|
65
|
-
FileUtils.mkdir_p(path) unless ::File.exist?(path)
|
66
|
-
self
|
67
|
-
end
|
68
|
-
|
69
|
-
def exist?
|
70
|
-
::File.exist?(@path)
|
71
|
-
end
|
72
|
-
|
73
|
-
# Delete the file.
|
74
|
-
#
|
75
|
-
# Note: Only the file is removed, not any of the parent paths.
|
76
|
-
def delete
|
77
|
-
::File.unlink(@path)
|
78
|
-
self
|
79
|
-
end
|
80
|
-
|
81
|
-
private
|
82
|
-
|
83
|
-
@roots = {}
|
84
|
-
end
|
85
|
-
end
|