iostreams 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/README.md +1 -1
 - data/Rakefile +7 -7
 - data/lib/io_streams/builder.rb +4 -3
 - data/lib/io_streams/bzip2/reader.rb +1 -1
 - data/lib/io_streams/bzip2/writer.rb +1 -1
 - data/lib/io_streams/deprecated.rb +2 -3
 - data/lib/io_streams/encode/reader.rb +5 -8
 - data/lib/io_streams/encode/writer.rb +1 -1
 - data/lib/io_streams/io_streams.rb +5 -2
 - data/lib/io_streams/line/reader.rb +2 -1
 - data/lib/io_streams/path.rb +4 -4
 - data/lib/io_streams/paths/http.rb +8 -10
 - data/lib/io_streams/paths/matcher.rb +10 -11
 - data/lib/io_streams/paths/s3.rb +6 -6
 - data/lib/io_streams/paths/sftp.rb +38 -23
 - data/lib/io_streams/pgp.rb +45 -35
 - data/lib/io_streams/pgp/reader.rb +4 -6
 - data/lib/io_streams/pgp/writer.rb +1 -2
 - data/lib/io_streams/reader.rb +2 -2
 - data/lib/io_streams/record/writer.rb +2 -4
 - data/lib/io_streams/row/writer.rb +3 -5
 - data/lib/io_streams/stream.rb +6 -6
 - data/lib/io_streams/symmetric_encryption/reader.rb +1 -3
 - data/lib/io_streams/symmetric_encryption/writer.rb +2 -6
 - data/lib/io_streams/tabular.rb +12 -10
 - data/lib/io_streams/tabular/header.rb +4 -4
 - data/lib/io_streams/tabular/parser/array.rb +2 -4
 - data/lib/io_streams/tabular/parser/csv.rb +3 -5
 - data/lib/io_streams/tabular/parser/fixed.rb +4 -3
 - data/lib/io_streams/tabular/parser/hash.rb +2 -4
 - data/lib/io_streams/tabular/parser/json.rb +2 -4
 - data/lib/io_streams/tabular/parser/psv.rb +5 -7
 - data/lib/io_streams/tabular/utility/csv_row.rb +9 -17
 - data/lib/io_streams/utils.rb +3 -3
 - data/lib/io_streams/utils/reliable_http.rb +98 -0
 - data/lib/io_streams/version.rb +1 -1
 - data/lib/io_streams/writer.rb +1 -1
 - data/lib/io_streams/xlsx/reader.rb +5 -5
 - data/lib/io_streams/zip/reader.rb +1 -1
 - data/lib/io_streams/zip/writer.rb +2 -2
 - data/lib/iostreams.rb +34 -34
 - data/test/builder_test.rb +74 -74
 - data/test/bzip2_reader_test.rb +8 -13
 - data/test/bzip2_writer_test.rb +8 -9
 - data/test/deprecated_test.rb +25 -29
 - data/test/encode_reader_test.rb +14 -18
 - data/test/encode_writer_test.rb +29 -30
 - data/test/gzip_reader_test.rb +8 -13
 - data/test/gzip_writer_test.rb +10 -11
 - data/test/io_streams_test.rb +35 -35
 - data/test/line_reader_test.rb +35 -39
 - data/test/line_writer_test.rb +8 -9
 - data/test/minimal_file_reader.rb +1 -1
 - data/test/path_test.rb +24 -24
 - data/test/paths/file_test.rb +42 -42
 - data/test/paths/http_test.rb +5 -5
 - data/test/paths/matcher_test.rb +11 -12
 - data/test/paths/s3_test.rb +44 -46
 - data/test/paths/sftp_test.rb +18 -18
 - data/test/pgp_reader_test.rb +13 -15
 - data/test/pgp_test.rb +43 -44
 - data/test/pgp_writer_test.rb +27 -28
 - data/test/record_reader_test.rb +9 -10
 - data/test/record_writer_test.rb +10 -11
 - data/test/row_reader_test.rb +5 -6
 - data/test/row_writer_test.rb +7 -8
 - data/test/stream_test.rb +60 -62
 - data/test/tabular_test.rb +111 -111
 - data/test/test_helper.rb +22 -22
 - data/test/utils_test.rb +7 -7
 - data/test/xlsx_reader_test.rb +12 -12
 - data/test/zip_reader_test.rb +14 -21
 - data/test/zip_writer_test.rb +10 -10
 - metadata +4 -3
 
    
        data/lib/io_streams/pgp.rb
    CHANGED
    
    | 
         @@ -1,4 +1,4 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require  
     | 
| 
      
 1 
     | 
    
         
            +
            require "open3"
         
     | 
| 
       2 
2 
     | 
    
         
             
            module IOStreams
         
     | 
| 
       3 
3 
     | 
    
         
             
              # Read/Write PGP/GPG file or stream.
         
     | 
| 
       4 
4 
     | 
    
         
             
              #
         
     | 
| 
         @@ -82,8 +82,8 @@ module IOStreams 
     | 
|
| 
       82 
82 
     | 
    
         
             
              # - Tested against gnupg v1.4.21 and v2.0.30
         
     | 
| 
       83 
83 
     | 
    
         
             
              # - Does not work yet with gnupg v2.1. Pull Requests welcome.
         
     | 
| 
       84 
84 
     | 
    
         
             
              module Pgp
         
     | 
| 
       85 
     | 
    
         
            -
                autoload :Reader,  
     | 
| 
       86 
     | 
    
         
            -
                autoload :Writer,  
     | 
| 
      
 85 
     | 
    
         
            +
                autoload :Reader, "io_streams/pgp/reader"
         
     | 
| 
      
 86 
     | 
    
         
            +
                autoload :Writer, "io_streams/pgp/writer"
         
     | 
| 
       87 
87 
     | 
    
         | 
| 
       88 
88 
     | 
    
         
             
                class Failure < StandardError
         
     | 
| 
       89 
89 
     | 
    
         
             
                end
         
     | 
| 
         @@ -99,7 +99,7 @@ module IOStreams 
     | 
|
| 
       99 
99 
     | 
    
         
             
                  @executable = executable
         
     | 
| 
       100 
100 
     | 
    
         
             
                end
         
     | 
| 
       101 
101 
     | 
    
         | 
| 
       102 
     | 
    
         
            -
                @executable =  
     | 
| 
      
 102 
     | 
    
         
            +
                @executable = "gpg"
         
     | 
| 
       103 
103 
     | 
    
         | 
| 
       104 
104 
     | 
    
         
             
                # Generate a new ultimate trusted local public and private key.
         
     | 
| 
       105 
105 
     | 
    
         
             
                #
         
     | 
| 
         @@ -122,9 +122,9 @@ module IOStreams 
     | 
|
| 
       122 
122 
     | 
    
         
             
                #     `SecureRandom.urlsafe_base64(128)`
         
     | 
| 
       123 
123 
     | 
    
         
             
                #
         
     | 
| 
       124 
124 
     | 
    
         
             
                # See `man gpg` for the remaining options
         
     | 
| 
       125 
     | 
    
         
            -
                def self.generate_key(name:, email:, comment: nil, passphrase:, key_type:  
     | 
| 
      
 125 
     | 
    
         
            +
                def self.generate_key(name:, email:, comment: nil, passphrase:, key_type: "RSA", key_length: 4096, subkey_type: "RSA", subkey_length: key_length, expire_date: nil)
         
     | 
| 
       126 
126 
     | 
    
         
             
                  version_check
         
     | 
| 
       127 
     | 
    
         
            -
                  params =  
     | 
| 
      
 127 
     | 
    
         
            +
                  params = ""
         
     | 
| 
       128 
128 
     | 
    
         
             
                  params << "Key-Type: #{key_type}\n" if key_type
         
     | 
| 
       129 
129 
     | 
    
         
             
                  params << "Key-Length: #{key_length}\n" if key_length
         
     | 
| 
       130 
130 
     | 
    
         
             
                  params << "Subkey-Type: #{subkey_type}\n" if subkey_type
         
     | 
| 
         @@ -134,14 +134,14 @@ module IOStreams 
     | 
|
| 
       134 
134 
     | 
    
         
             
                  params << "Name-Email: #{email}\n" if email
         
     | 
| 
       135 
135 
     | 
    
         
             
                  params << "Expire-Date: #{expire_date}\n" if expire_date
         
     | 
| 
       136 
136 
     | 
    
         
             
                  params << "Passphrase: #{passphrase}\n" if passphrase
         
     | 
| 
       137 
     | 
    
         
            -
                  params <<  
     | 
| 
       138 
     | 
    
         
            -
                  command = "#{executable} --batch --gen-key --no-tty 
     | 
| 
      
 137 
     | 
    
         
            +
                  params << "%commit"
         
     | 
| 
      
 138 
     | 
    
         
            +
                  command = "#{executable} --batch --gen-key --no-tty"
         
     | 
| 
       139 
139 
     | 
    
         | 
| 
       140 
140 
     | 
    
         
             
                  out, err, status = Open3.capture3(command, binmode: true, stdin_data: params)
         
     | 
| 
       141 
     | 
    
         
            -
                  logger 
     | 
| 
      
 141 
     | 
    
         
            +
                  logger&.debug { "IOStreams::Pgp.generate_key: #{command}\n#{params}\n#{err}#{out}" }
         
     | 
| 
       142 
142 
     | 
    
         
             
                  if status.success?
         
     | 
| 
       143 
143 
     | 
    
         
             
                    if match = err.match(/gpg: key ([0-9A-F]+)\s+/)
         
     | 
| 
       144 
     | 
    
         
            -
                       
     | 
| 
      
 144 
     | 
    
         
            +
                      match[1]
         
     | 
| 
       145 
145 
     | 
    
         
             
                    end
         
     | 
| 
       146 
146 
     | 
    
         
             
                  else
         
     | 
| 
       147 
147 
     | 
    
         
             
                    raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}")
         
     | 
| 
         @@ -172,12 +172,17 @@ module IOStreams 
     | 
|
| 
       172 
172 
     | 
    
         
             
                end
         
     | 
| 
       173 
173 
     | 
    
         | 
| 
       174 
174 
     | 
    
         
             
                # Returns [true|false] whether their is a key for the supplied email or key_id
         
     | 
| 
       175 
     | 
    
         
            -
                def self. 
     | 
| 
       176 
     | 
    
         
            -
                  raise(ArgumentError,  
     | 
| 
      
 175 
     | 
    
         
            +
                def self.key?(email: nil, key_id: nil, private: false)
         
     | 
| 
      
 176 
     | 
    
         
            +
                  raise(ArgumentError, "Either :email, or :key_id must be supplied") if email.nil? && key_id.nil?
         
     | 
| 
       177 
177 
     | 
    
         | 
| 
       178 
178 
     | 
    
         
             
                  !list_keys(email: email, key_id: key_id, private: private).empty?
         
     | 
| 
       179 
179 
     | 
    
         
             
                end
         
     | 
| 
       180 
180 
     | 
    
         | 
| 
      
 181 
     | 
    
         
            +
                # Deprecated
         
     | 
| 
      
 182 
     | 
    
         
            +
                def self.has_key?(**args)
         
     | 
| 
      
 183 
     | 
    
         
            +
                  key(**args)
         
     | 
| 
      
 184 
     | 
    
         
            +
                end
         
     | 
| 
      
 185 
     | 
    
         
            +
             
     | 
| 
       181 
186 
     | 
    
         
             
                # Returns [Array<Hash>] the list of keys.
         
     | 
| 
       182 
187 
     | 
    
         
             
                #   Each Hash consists of:
         
     | 
| 
       183 
188 
     | 
    
         
             
                #     key_length: [Integer]
         
     | 
| 
         @@ -189,15 +194,16 @@ module IOStreams 
     | 
|
| 
       189 
194 
     | 
    
         
             
                # Returns [] if no keys were found.
         
     | 
| 
       190 
195 
     | 
    
         
             
                def self.list_keys(email: nil, key_id: nil, private: false)
         
     | 
| 
       191 
196 
     | 
    
         
             
                  version_check
         
     | 
| 
       192 
     | 
    
         
            -
                  cmd     = private ?  
     | 
| 
      
 197 
     | 
    
         
            +
                  cmd     = private ? "--list-secret-keys" : "--list-keys"
         
     | 
| 
       193 
198 
     | 
    
         
             
                  command = "#{executable} #{cmd} #{email || key_id}"
         
     | 
| 
       194 
199 
     | 
    
         | 
| 
       195 
200 
     | 
    
         
             
                  out, err, status = Open3.capture3(command, binmode: true)
         
     | 
| 
       196 
     | 
    
         
            -
                  logger 
     | 
| 
       197 
     | 
    
         
            -
                  if status.success? && out.length 
     | 
| 
      
 201 
     | 
    
         
            +
                  logger&.debug { "IOStreams::Pgp.list_keys: #{command}\n#{err}#{out}" }
         
     | 
| 
      
 202 
     | 
    
         
            +
                  if status.success? && out.length.positive?
         
     | 
| 
       198 
203 
     | 
    
         
             
                    parse_list_output(out)
         
     | 
| 
       199 
204 
     | 
    
         
             
                  else
         
     | 
| 
       200 
205 
     | 
    
         
             
                    return [] if err =~ /(not found|No (public|secret) key|key not available)/i
         
     | 
| 
      
 206 
     | 
    
         
            +
             
     | 
| 
       201 
207 
     | 
    
         
             
                    raise(Pgp::Failure, "GPG Failed calling '#{executable}' to list keys for #{email || key_id}: #{err}#{out}")
         
     | 
| 
       202 
208 
     | 
    
         
             
                  end
         
     | 
| 
       203 
209 
     | 
    
         
             
                end
         
     | 
| 
         @@ -219,8 +225,8 @@ module IOStreams 
     | 
|
| 
       219 
225 
     | 
    
         
             
                  command = executable.to_s
         
     | 
| 
       220 
226 
     | 
    
         | 
| 
       221 
227 
     | 
    
         
             
                  out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
         
     | 
| 
       222 
     | 
    
         
            -
                  logger 
     | 
| 
       223 
     | 
    
         
            -
                  if status.success? && out.length 
     | 
| 
      
 228 
     | 
    
         
            +
                  logger&.debug { "IOStreams::Pgp.key_info: #{command}\n#{err}#{out}" }
         
     | 
| 
      
 229 
     | 
    
         
            +
                  if status.success? && out.length.positive?
         
     | 
| 
       224 
230 
     | 
    
         
             
                    # Sample Output:
         
     | 
| 
       225 
231 
     | 
    
         
             
                    #
         
     | 
| 
       226 
232 
     | 
    
         
             
                    #   pub  4096R/3A5456F5 2017-06-07
         
     | 
| 
         @@ -243,15 +249,15 @@ module IOStreams 
     | 
|
| 
       243 
249 
     | 
    
         
             
                  version_check
         
     | 
| 
       244 
250 
     | 
    
         | 
| 
       245 
251 
     | 
    
         
             
                  command = "#{executable} "
         
     | 
| 
       246 
     | 
    
         
            -
                  command <<  
     | 
| 
       247 
     | 
    
         
            -
                  command <<  
     | 
| 
      
 252 
     | 
    
         
            +
                  command << "--pinentry-mode loopback " if pgp_version.to_f >= 2.1
         
     | 
| 
      
 253 
     | 
    
         
            +
                  command << "--armor " if ascii
         
     | 
| 
       248 
254 
     | 
    
         
             
                  command << "--no-tty  --batch --passphrase"
         
     | 
| 
       249 
255 
     | 
    
         
             
                  command << (passphrase ? " #{passphrase} " : "-fd 0 ")
         
     | 
| 
       250 
256 
     | 
    
         
             
                  command << (private ? "--export-secret-keys #{email}" : "--export #{email}")
         
     | 
| 
       251 
257 
     | 
    
         | 
| 
       252 
258 
     | 
    
         
             
                  out, err, status = Open3.capture3(command, binmode: true)
         
     | 
| 
       253 
     | 
    
         
            -
                  logger 
     | 
| 
       254 
     | 
    
         
            -
                  if status.success? && out.length 
     | 
| 
      
 259 
     | 
    
         
            +
                  logger&.debug { "IOStreams::Pgp.export: #{command}\n#{err}" }
         
     | 
| 
      
 260 
     | 
    
         
            +
                  if status.success? && out.length.positive?
         
     | 
| 
       255 
261 
     | 
    
         
             
                    out
         
     | 
| 
       256 
262 
     | 
    
         
             
                  else
         
     | 
| 
       257 
263 
     | 
    
         
             
                    raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err}")
         
     | 
| 
         @@ -319,7 +325,7 @@ module IOStreams 
     | 
|
| 
       319 
325 
     | 
    
         
             
                # Notes:
         
     | 
| 
       320 
326 
     | 
    
         
             
                # - If the same email address has multiple keys then only the first is currently trusted.
         
     | 
| 
       321 
327 
     | 
    
         
             
                def self.import_and_trust(key:)
         
     | 
| 
       322 
     | 
    
         
            -
                  raise(ArgumentError, "Key cannot be empty") if key.nil? || (key ==  
     | 
| 
      
 328 
     | 
    
         
            +
                  raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == "")
         
     | 
| 
       323 
329 
     | 
    
         | 
| 
       324 
330 
     | 
    
         
             
                  email = key_info(key: key).first.fetch(:email)
         
     | 
| 
       325 
331 
     | 
    
         
             
                  raise(ArgumentError, "Recipient email cannot be extracted from supplied key") unless email
         
     | 
| 
         @@ -443,20 +449,28 @@ module IOStreams 
     | 
|
| 
       443 
449 
     | 
    
         
             
                    if match = line.match(/(pub|sec)\s+(\D+)(\d+)\s+(\d+-\d+-\d+)\s+(.*)/)
         
     | 
| 
       444 
450 
     | 
    
         
             
                      # v2.2:    pub   rsa1024 2017-10-24 [SCEA]
         
     | 
| 
       445 
451 
     | 
    
         
             
                      hash = {
         
     | 
| 
       446 
     | 
    
         
            -
                        private:    match[1] ==  
     | 
| 
      
 452 
     | 
    
         
            +
                        private:    match[1] == "sec",
         
     | 
| 
       447 
453 
     | 
    
         
             
                        key_length: match[3].to_s.to_i,
         
     | 
| 
       448 
454 
     | 
    
         
             
                        key_type:   match[2],
         
     | 
| 
       449 
     | 
    
         
            -
                        date:       ( 
     | 
| 
      
 455 
     | 
    
         
            +
                        date:       (begin
         
     | 
| 
      
 456 
     | 
    
         
            +
                                       Date.parse(match[4].to_s)
         
     | 
| 
      
 457 
     | 
    
         
            +
                                     rescue StandardError
         
     | 
| 
      
 458 
     | 
    
         
            +
                                       match[4]
         
     | 
| 
      
 459 
     | 
    
         
            +
                                     end)
         
     | 
| 
       450 
460 
     | 
    
         
             
                      }
         
     | 
| 
       451 
461 
     | 
    
         
             
                    elsif match = line.match(%r{(pub|sec)\s+(\d+)(.*)/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?})
         
     | 
| 
       452 
462 
     | 
    
         
             
                      # Matches: pub  2048R/C7F9D9CB 2016-10-26
         
     | 
| 
       453 
463 
     | 
    
         
             
                      # Or:      pub  2048R/C7F9D9CB 2016-10-26 Receiver <receiver@example.org>
         
     | 
| 
       454 
464 
     | 
    
         
             
                      hash = {
         
     | 
| 
       455 
     | 
    
         
            -
                        private:    match[1] ==  
     | 
| 
      
 465 
     | 
    
         
            +
                        private:    match[1] == "sec",
         
     | 
| 
       456 
466 
     | 
    
         
             
                        key_length: match[2].to_s.to_i,
         
     | 
| 
       457 
467 
     | 
    
         
             
                        key_type:   match[3],
         
     | 
| 
       458 
468 
     | 
    
         
             
                        key_id:     match[4],
         
     | 
| 
       459 
     | 
    
         
            -
                        date:       ( 
     | 
| 
      
 469 
     | 
    
         
            +
                        date:       (begin
         
     | 
| 
      
 470 
     | 
    
         
            +
                                       Date.parse(match[5].to_s)
         
     | 
| 
      
 471 
     | 
    
         
            +
                                     rescue StandardError
         
     | 
| 
      
 472 
     | 
    
         
            +
                                       match[5]
         
     | 
| 
      
 473 
     | 
    
         
            +
                                     end)
         
     | 
| 
       460 
474 
     | 
    
         
             
                      }
         
     | 
| 
       461 
475 
     | 
    
         
             
                      # Prior to gpg v2.0.30
         
     | 
| 
       462 
476 
     | 
    
         
             
                      if match[7]
         
     | 
| 
         @@ -483,7 +497,7 @@ module IOStreams 
     | 
|
| 
       483 
497 
     | 
    
         
             
                end
         
     | 
| 
       484 
498 
     | 
    
         | 
| 
       485 
499 
     | 
    
         
             
                def self.delete_public_or_private_keys(email:, private: false)
         
     | 
| 
       486 
     | 
    
         
            -
                  keys = private ?  
     | 
| 
      
 500 
     | 
    
         
            +
                  keys = private ? "secret-keys" : "keys"
         
     | 
| 
       487 
501 
     | 
    
         | 
| 
       488 
502 
     | 
    
         
             
                  list = list_keys(email: email, private: private)
         
     | 
| 
       489 
503 
     | 
    
         
             
                  return false if list.empty?
         
     | 
| 
         @@ -499,19 +513,17 @@ module IOStreams 
     | 
|
| 
       499 
513 
     | 
    
         
             
                    unless status.success?
         
     | 
| 
       500 
514 
     | 
    
         
             
                      raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
         
     | 
| 
       501 
515 
     | 
    
         
             
                    end
         
     | 
| 
       502 
     | 
    
         
            -
                    if out.include?( 
     | 
| 
       503 
     | 
    
         
            -
                      raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}:#{out}")
         
     | 
| 
       504 
     | 
    
         
            -
                    end
         
     | 
| 
      
 516 
     | 
    
         
            +
                    raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}:#{out}") if out.include?("error")
         
     | 
| 
       505 
517 
     | 
    
         
             
                  end
         
     | 
| 
       506 
518 
     | 
    
         
             
                  true
         
     | 
| 
       507 
519 
     | 
    
         
             
                end
         
     | 
| 
       508 
520 
     | 
    
         | 
| 
       509 
521 
     | 
    
         
             
                def self.delete_public_or_private_keys_v1(email:, private: false)
         
     | 
| 
       510 
     | 
    
         
            -
                  keys = private ?  
     | 
| 
      
 522 
     | 
    
         
            +
                  keys = private ? "secret-keys" : "keys"
         
     | 
| 
       511 
523 
     | 
    
         | 
| 
       512 
524 
     | 
    
         
             
                  command = "for i in `#{executable} --list-#{keys} --with-colons --fingerprint #{email} | grep \"^fpr\" | cut -d: -f10`; do\n"
         
     | 
| 
       513 
525 
     | 
    
         
             
                  command << "#{executable} --batch --no-tty --yes --delete-#{keys} \"$i\" ;\n"
         
     | 
| 
       514 
     | 
    
         
            -
                  command <<  
     | 
| 
      
 526 
     | 
    
         
            +
                  command << "done"
         
     | 
| 
       515 
527 
     | 
    
         | 
| 
       516 
528 
     | 
    
         
             
                  out, err, status = Open3.capture3(command, binmode: true)
         
     | 
| 
       517 
529 
     | 
    
         
             
                  logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}: #{out}" }
         
     | 
| 
         @@ -520,9 +532,7 @@ module IOStreams 
     | 
|
| 
       520 
532 
     | 
    
         
             
                  unless status.success?
         
     | 
| 
       521 
533 
     | 
    
         
             
                    raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
         
     | 
| 
       522 
534 
     | 
    
         
             
                  end
         
     | 
| 
       523 
     | 
    
         
            -
                  if out.include?( 
     | 
| 
       524 
     | 
    
         
            -
                    raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}: #{out}")
         
     | 
| 
       525 
     | 
    
         
            -
                  end
         
     | 
| 
      
 535 
     | 
    
         
            +
                  raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}: #{out}") if out.include?("error")
         
     | 
| 
       526 
536 
     | 
    
         | 
| 
       527 
537 
     | 
    
         
             
                  true
         
     | 
| 
       528 
538 
     | 
    
         
             
                end
         
     | 
| 
         @@ -1,4 +1,4 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require  
     | 
| 
      
 1 
     | 
    
         
            +
            require "open3"
         
     | 
| 
       2 
2 
     | 
    
         | 
| 
       3 
3 
     | 
    
         
             
            module IOStreams
         
     | 
| 
       4 
4 
     | 
    
         
             
              module Pgp
         
     | 
| 
         @@ -24,9 +24,9 @@ module IOStreams 
     | 
|
| 
       24 
24 
     | 
    
         
             
                  def self.file(file_name, passphrase: nil)
         
     | 
| 
       25 
25 
     | 
    
         
             
                    # Cannot use `passphrase: self.default_passphrase` since it is considered private
         
     | 
| 
       26 
26 
     | 
    
         
             
                    passphrase ||= default_passphrase
         
     | 
| 
       27 
     | 
    
         
            -
                    raise(ArgumentError,  
     | 
| 
      
 27 
     | 
    
         
            +
                    raise(ArgumentError, "Missing both passphrase and IOStreams::Pgp::Reader.default_passphrase") unless passphrase
         
     | 
| 
       28 
28 
     | 
    
         | 
| 
       29 
     | 
    
         
            -
                    loopback = IOStreams::Pgp.pgp_version.to_f >= 2.1 ?  
     | 
| 
      
 29 
     | 
    
         
            +
                    loopback = IOStreams::Pgp.pgp_version.to_f >= 2.1 ? "--pinentry-mode loopback" : ""
         
     | 
| 
       30 
30 
     | 
    
         
             
                    command  = "#{IOStreams::Pgp.executable} #{loopback} --batch --no-tty --yes --decrypt --passphrase-fd 0 #{file_name}"
         
     | 
| 
       31 
31 
     | 
    
         
             
                    IOStreams::Pgp.logger&.debug { "IOStreams::Pgp::Reader.open: #{command}" }
         
     | 
| 
       32 
32 
     | 
    
         | 
| 
         @@ -42,9 +42,7 @@ module IOStreams 
     | 
|
| 
       42 
42 
     | 
    
         
             
                          # Ignore broken pipe because gpg terminates early due to an error
         
     | 
| 
       43 
43 
     | 
    
         
             
                          raise(Pgp::Failure, "GPG Failed reading from encrypted file: #{file_name}: #{stderr.read.chomp}")
         
     | 
| 
       44 
44 
     | 
    
         
             
                        end
         
     | 
| 
       45 
     | 
    
         
            -
                      unless waith_thr.value.success?
         
     | 
| 
       46 
     | 
    
         
            -
                        raise(Pgp::Failure, "GPG Failed to decrypt file: #{file_name}: #{stderr.read.chomp}")
         
     | 
| 
       47 
     | 
    
         
            -
                      end
         
     | 
| 
      
 45 
     | 
    
         
            +
                      raise(Pgp::Failure, "GPG Failed to decrypt file: #{file_name}: #{stderr.read.chomp}") unless waith_thr.value.success?
         
     | 
| 
       48 
46 
     | 
    
         | 
| 
       49 
47 
     | 
    
         
             
                      result
         
     | 
| 
       50 
48 
     | 
    
         
             
                    end
         
     | 
    
        data/lib/io_streams/reader.rb
    CHANGED
    
    | 
         @@ -4,14 +4,14 @@ module IOStreams 
     | 
|
| 
       4 
4 
     | 
    
         
             
                # and then pass that filename in for this reader.
         
     | 
| 
       5 
5 
     | 
    
         
             
                def self.stream(input_stream, **args, &block)
         
     | 
| 
       6 
6 
     | 
    
         
             
                  Utils.temp_file_name("iostreams_reader") do |file_name|
         
     | 
| 
       7 
     | 
    
         
            -
                    ::File.open(file_name,  
     | 
| 
      
 7 
     | 
    
         
            +
                    ::File.open(file_name, "wb") { |target| ::IO.copy_stream(input_stream, target) }
         
     | 
| 
       8 
8 
     | 
    
         
             
                    file(file_name, **args, &block)
         
     | 
| 
       9 
9 
     | 
    
         
             
                  end
         
     | 
| 
       10 
10 
     | 
    
         
             
                end
         
     | 
| 
       11 
11 
     | 
    
         | 
| 
       12 
12 
     | 
    
         
             
                # When a Writer supports streams, also allow it to simply support a file
         
     | 
| 
       13 
13 
     | 
    
         
             
                def self.file(file_name, original_file_name: file_name, **args, &block)
         
     | 
| 
       14 
     | 
    
         
            -
                  ::File.open(file_name,  
     | 
| 
      
 14 
     | 
    
         
            +
                  ::File.open(file_name, "rb") { |file| stream(file, original_file_name: original_file_name, **args, &block) }
         
     | 
| 
       15 
15 
     | 
    
         
             
                end
         
     | 
| 
       16 
16 
     | 
    
         | 
| 
       17 
17 
     | 
    
         
             
                # For processing by either a file name or an open IO stream.
         
     | 
| 
         @@ -35,9 +35,7 @@ module IOStreams 
     | 
|
| 
       35 
35 
     | 
    
         
             
                  #
         
     | 
| 
       36 
36 
     | 
    
         
             
                  #   For all other parameters, see Tabular::Header.new
         
     | 
| 
       37 
37 
     | 
    
         
             
                  def initialize(line_writer, columns: nil, **args)
         
     | 
| 
       38 
     | 
    
         
            -
                    unless line_writer.respond_to?(:<<)
         
     | 
| 
       39 
     | 
    
         
            -
                      raise(ArgumentError, 'Stream must be a IOStreams::Line::Writer or implement #<<')
         
     | 
| 
       40 
     | 
    
         
            -
                    end
         
     | 
| 
      
 38 
     | 
    
         
            +
                    raise(ArgumentError, "Stream must be a IOStreams::Line::Writer or implement #<<") unless line_writer.respond_to?(:<<)
         
     | 
| 
       41 
39 
     | 
    
         | 
| 
       42 
40 
     | 
    
         
             
                    @tabular     = IOStreams::Tabular.new(columns: columns, **args)
         
     | 
| 
       43 
41 
     | 
    
         
             
                    @line_writer = line_writer
         
     | 
| 
         @@ -47,7 +45,7 @@ module IOStreams 
     | 
|
| 
       47 
45 
     | 
    
         
             
                  end
         
     | 
| 
       48 
46 
     | 
    
         | 
| 
       49 
47 
     | 
    
         
             
                  def <<(hash)
         
     | 
| 
       50 
     | 
    
         
            -
                    raise(ArgumentError,  
     | 
| 
      
 48 
     | 
    
         
            +
                    raise(ArgumentError, "#<< only accepts a Hash argument") unless hash.is_a?(Hash)
         
     | 
| 
       51 
49 
     | 
    
         | 
| 
       52 
50 
     | 
    
         
             
                    if @tabular.header?
         
     | 
| 
       53 
51 
     | 
    
         
             
                      # Extract header from the keys from the first row when not supplied above.
         
     | 
| 
         @@ -1,4 +1,4 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require  
     | 
| 
      
 1 
     | 
    
         
            +
            require "csv"
         
     | 
| 
       2 
2 
     | 
    
         
             
            module IOStreams
         
     | 
| 
       3 
3 
     | 
    
         
             
              module Row
         
     | 
| 
       4 
4 
     | 
    
         
             
                # Example:
         
     | 
| 
         @@ -37,9 +37,7 @@ module IOStreams 
     | 
|
| 
       37 
37 
     | 
    
         
             
                  #
         
     | 
| 
       38 
38 
     | 
    
         
             
                  #   For all other parameters, see Tabular::Header.new
         
     | 
| 
       39 
39 
     | 
    
         
             
                  def initialize(line_writer, columns: nil, **args)
         
     | 
| 
       40 
     | 
    
         
            -
                    unless line_writer.respond_to?(:<<)
         
     | 
| 
       41 
     | 
    
         
            -
                      raise(ArgumentError, 'Stream must be a IOStreams::Line::Writer or implement #<<')
         
     | 
| 
       42 
     | 
    
         
            -
                    end
         
     | 
| 
      
 40 
     | 
    
         
            +
                    raise(ArgumentError, "Stream must be a IOStreams::Line::Writer or implement #<<") unless line_writer.respond_to?(:<<)
         
     | 
| 
       43 
41 
     | 
    
         | 
| 
       44 
42 
     | 
    
         
             
                    @tabular     = IOStreams::Tabular.new(columns: columns, **args)
         
     | 
| 
       45 
43 
     | 
    
         
             
                    @line_writer = line_writer
         
     | 
| 
         @@ -50,7 +48,7 @@ module IOStreams 
     | 
|
| 
       50 
48 
     | 
    
         | 
| 
       51 
49 
     | 
    
         
             
                  # Supply a hash or an array to render
         
     | 
| 
       52 
50 
     | 
    
         
             
                  def <<(array)
         
     | 
| 
       53 
     | 
    
         
            -
                    raise(ArgumentError,  
     | 
| 
      
 51 
     | 
    
         
            +
                    raise(ArgumentError, "Must supply an Array") unless array.is_a?(Array)
         
     | 
| 
       54 
52 
     | 
    
         | 
| 
       55 
53 
     | 
    
         
             
                    if @tabular.header?
         
     | 
| 
       56 
54 
     | 
    
         
             
                      # If header (columns) was not supplied as an argument, assume first line is the header.
         
     | 
    
        data/lib/io_streams/stream.rb
    CHANGED
    
    | 
         @@ -4,7 +4,7 @@ module IOStreams 
     | 
|
| 
       4 
4 
     | 
    
         
             
                attr_writer :builder
         
     | 
| 
       5 
5 
     | 
    
         | 
| 
       6 
6 
     | 
    
         
             
                def initialize(io_stream)
         
     | 
| 
       7 
     | 
    
         
            -
                  raise(ArgumentError,  
     | 
| 
      
 7 
     | 
    
         
            +
                  raise(ArgumentError, "io_stream cannot be nil") if io_stream.nil?
         
     | 
| 
       8 
8 
     | 
    
         
             
                  raise(ArgumentError, "io_stream must not be a string: #{io_stream.inspect}") if io_stream.is_a?(String)
         
     | 
| 
       9 
9 
     | 
    
         | 
| 
       10 
10 
     | 
    
         
             
                  @io_stream = io_stream
         
     | 
| 
         @@ -266,7 +266,7 @@ module IOStreams 
     | 
|
| 
       266 
266 
     | 
    
         
             
                #   IOStreams.path(".profile").extension        #=> ""
         
     | 
| 
       267 
267 
     | 
    
         
             
                #   IOStreams.path(".profile.sh").extension     #=> "sh"
         
     | 
| 
       268 
268 
     | 
    
         
             
                def extension
         
     | 
| 
       269 
     | 
    
         
            -
                  extname&.sub(/^\./,  
     | 
| 
      
 269 
     | 
    
         
            +
                  extname&.sub(/^\./, "")
         
     | 
| 
       270 
270 
     | 
    
         
             
                end
         
     | 
| 
       271 
271 
     | 
    
         | 
| 
       272 
272 
     | 
    
         
             
                private
         
     | 
| 
         @@ -280,7 +280,7 @@ module IOStreams 
     | 
|
| 
       280 
280 
     | 
    
         
             
                end
         
     | 
| 
       281 
281 
     | 
    
         | 
| 
       282 
282 
     | 
    
         
             
                def line_reader(embedded_within: nil, **args)
         
     | 
| 
       283 
     | 
    
         
            -
                  embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?( 
     | 
| 
      
 283 
     | 
    
         
            +
                  embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?(".csv")
         
     | 
| 
       284 
284 
     | 
    
         | 
| 
       285 
285 
     | 
    
         
             
                  stream_reader { |io| yield IOStreams::Line::Reader.new(io, embedded_within: embedded_within, **args) }
         
     | 
| 
       286 
286 
     | 
    
         
             
                end
         
     | 
| 
         @@ -304,19 +304,19 @@ module IOStreams 
     | 
|
| 
       304 
304 
     | 
    
         
             
                end
         
     | 
| 
       305 
305 
     | 
    
         | 
| 
       306 
306 
     | 
    
         
             
                def line_writer(**args, &block)
         
     | 
| 
       307 
     | 
    
         
            -
                  return block.call(io_stream) if io_stream 
     | 
| 
      
 307 
     | 
    
         
            +
                  return block.call(io_stream) if io_stream&.is_a?(IOStreams::Line::Writer)
         
     | 
| 
       308 
308 
     | 
    
         | 
| 
       309 
309 
     | 
    
         
             
                  writer { |io| IOStreams::Line::Writer.stream(io, **args, &block) }
         
     | 
| 
       310 
310 
     | 
    
         
             
                end
         
     | 
| 
       311 
311 
     | 
    
         | 
| 
       312 
312 
     | 
    
         
             
                def row_writer(delimiter: $/, **args, &block)
         
     | 
| 
       313 
     | 
    
         
            -
                  return block.call(io_stream) if io_stream 
     | 
| 
      
 313 
     | 
    
         
            +
                  return block.call(io_stream) if io_stream&.is_a?(IOStreams::Row::Writer)
         
     | 
| 
       314 
314 
     | 
    
         | 
| 
       315 
315 
     | 
    
         
             
                  line_writer(delimiter: delimiter) { |io| IOStreams::Row::Writer.stream(io, **args, &block) }
         
     | 
| 
       316 
316 
     | 
    
         
             
                end
         
     | 
| 
       317 
317 
     | 
    
         | 
| 
       318 
318 
     | 
    
         
             
                def record_writer(delimiter: $/, **args, &block)
         
     | 
| 
       319 
     | 
    
         
            -
                  return block.call(io_stream) if io_stream 
     | 
| 
      
 319 
     | 
    
         
            +
                  return block.call(io_stream) if io_stream&.is_a?(IOStreams::Record::Writer)
         
     | 
| 
       320 
320 
     | 
    
         | 
| 
       321 
321 
     | 
    
         
             
                  line_writer(delimiter: delimiter) { |io| IOStreams::Record::Writer.stream(io, **args, &block) }
         
     | 
| 
       322 
322 
     | 
    
         
             
                end
         
     | 
| 
         @@ -3,9 +3,7 @@ module IOStreams 
     | 
|
| 
       3 
3 
     | 
    
         
             
                class Reader < IOStreams::Reader
         
     | 
| 
       4 
4 
     | 
    
         
             
                  # read from a file/stream using Symmetric Encryption
         
     | 
| 
       5 
5 
     | 
    
         
             
                  def self.stream(input_stream, **args, &block)
         
     | 
| 
       6 
     | 
    
         
            -
                    unless defined?(SymmetricEncryption)
         
     | 
| 
       7 
     | 
    
         
            -
                      Utils.load_soft_dependency('symmetric-encryption', '.enc streaming')
         
     | 
| 
       8 
     | 
    
         
            -
                    end
         
     | 
| 
      
 6 
     | 
    
         
            +
                    Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
         
     | 
| 
       9 
7 
     | 
    
         | 
| 
       10 
8 
     | 
    
         
             
                    ::SymmetricEncryption::Reader.open(input_stream, **args, &block)
         
     | 
| 
       11 
9 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -5,9 +5,7 @@ module IOStreams 
     | 
|
| 
       5 
5 
     | 
    
         
             
                  # By default the output stream is compressed.
         
     | 
| 
       6 
6 
     | 
    
         
             
                  # If the input_stream is already compressed consider setting compress: false.
         
     | 
| 
       7 
7 
     | 
    
         
             
                  def self.stream(input_stream, compress: true, **args, &block)
         
     | 
| 
       8 
     | 
    
         
            -
                    unless defined?(SymmetricEncryption)
         
     | 
| 
       9 
     | 
    
         
            -
                      Utils.load_soft_dependency('symmetric-encryption', '.enc streaming')
         
     | 
| 
       10 
     | 
    
         
            -
                    end
         
     | 
| 
      
 8 
     | 
    
         
            +
                    Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
         
     | 
| 
       11 
9 
     | 
    
         | 
| 
       12 
10 
     | 
    
         
             
                    ::SymmetricEncryption::Writer.open(input_stream, compress: compress, **args, &block)
         
     | 
| 
       13 
11 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -15,9 +13,7 @@ module IOStreams 
     | 
|
| 
       15 
13 
     | 
    
         
             
                  # Write to stream using Symmetric Encryption
         
     | 
| 
       16 
14 
     | 
    
         
             
                  # By default the output stream is compressed unless the file_name extension indicates the file is already compressed.
         
     | 
| 
       17 
15 
     | 
    
         
             
                  def self.file(file_name, compress: nil, **args, &block)
         
     | 
| 
       18 
     | 
    
         
            -
                    unless defined?(SymmetricEncryption)
         
     | 
| 
       19 
     | 
    
         
            -
                      Utils.load_soft_dependency('symmetric-encryption', '.enc streaming')
         
     | 
| 
       20 
     | 
    
         
            -
                    end
         
     | 
| 
      
 16 
     | 
    
         
            +
                    Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
         
     | 
| 
       21 
17 
     | 
    
         | 
| 
       22 
18 
     | 
    
         
             
                    ::SymmetricEncryption::Writer.open(file_name, compress: compress, **args, &block)
         
     | 
| 
       23 
19 
     | 
    
         
             
                  end
         
     | 
    
        data/lib/io_streams/tabular.rb
    CHANGED
    
    | 
         @@ -28,20 +28,20 @@ module IOStreams 
     | 
|
| 
       28 
28 
     | 
    
         
             
              #   tabular.render({"third"=>"3", "first_field"=>"1" })
         
     | 
| 
       29 
29 
     | 
    
         
             
              #   # => "1,,3"
         
     | 
| 
       30 
30 
     | 
    
         
             
              class Tabular
         
     | 
| 
       31 
     | 
    
         
            -
                autoload :Header,  
     | 
| 
      
 31 
     | 
    
         
            +
                autoload :Header, "io_streams/tabular/header"
         
     | 
| 
       32 
32 
     | 
    
         | 
| 
       33 
33 
     | 
    
         
             
                module Parser
         
     | 
| 
       34 
     | 
    
         
            -
                  autoload :Array,  
     | 
| 
       35 
     | 
    
         
            -
                  autoload :Base,  
     | 
| 
       36 
     | 
    
         
            -
                  autoload :Csv,  
     | 
| 
       37 
     | 
    
         
            -
                  autoload :Fixed,  
     | 
| 
       38 
     | 
    
         
            -
                  autoload :Hash,  
     | 
| 
       39 
     | 
    
         
            -
                  autoload :Json,  
     | 
| 
       40 
     | 
    
         
            -
                  autoload :Psv,  
     | 
| 
      
 34 
     | 
    
         
            +
                  autoload :Array, "io_streams/tabular/parser/array"
         
     | 
| 
      
 35 
     | 
    
         
            +
                  autoload :Base, "io_streams/tabular/parser/base"
         
     | 
| 
      
 36 
     | 
    
         
            +
                  autoload :Csv, "io_streams/tabular/parser/csv"
         
     | 
| 
      
 37 
     | 
    
         
            +
                  autoload :Fixed, "io_streams/tabular/parser/fixed"
         
     | 
| 
      
 38 
     | 
    
         
            +
                  autoload :Hash, "io_streams/tabular/parser/hash"
         
     | 
| 
      
 39 
     | 
    
         
            +
                  autoload :Json, "io_streams/tabular/parser/json"
         
     | 
| 
      
 40 
     | 
    
         
            +
                  autoload :Psv, "io_streams/tabular/parser/psv"
         
     | 
| 
       41 
41 
     | 
    
         
             
                end
         
     | 
| 
       42 
42 
     | 
    
         | 
| 
       43 
43 
     | 
    
         
             
                module Utility
         
     | 
| 
       44 
     | 
    
         
            -
                  autoload :CSVRow,  
     | 
| 
      
 44 
     | 
    
         
            +
                  autoload :CSVRow, "io_streams/tabular/utility/csv_row"
         
     | 
| 
       45 
45 
     | 
    
         
             
                end
         
     | 
| 
       46 
46 
     | 
    
         | 
| 
       47 
47 
     | 
    
         
             
                attr_reader :format, :header, :parser
         
     | 
| 
         @@ -132,6 +132,7 @@ module IOStreams 
     | 
|
| 
       132 
132 
     | 
    
         
             
                #   register_format(:csv, IOStreams::Tabular::Parser::Csv)
         
     | 
| 
       133 
133 
     | 
    
         
             
                def self.register_format(format, parser)
         
     | 
| 
       134 
134 
     | 
    
         
             
                  raise(ArgumentError, "Invalid format #{format.inspect}") unless format.nil? || format.to_s =~ /\A\w+\Z/
         
     | 
| 
      
 135 
     | 
    
         
            +
             
     | 
| 
       135 
136 
     | 
    
         
             
                  @formats[format.nil? ? nil : format.to_sym] = parser
         
     | 
| 
       136 
137 
     | 
    
         
             
                end
         
     | 
| 
       137 
138 
     | 
    
         | 
| 
         @@ -143,6 +144,7 @@ module IOStreams 
     | 
|
| 
       143 
144 
     | 
    
         
             
                #   register_extension(:xls)
         
     | 
| 
       144 
145 
     | 
    
         
             
                def self.deregister_format(format)
         
     | 
| 
       145 
146 
     | 
    
         
             
                  raise(ArgumentError, "Invalid format #{format.inspect}") unless format.to_s =~ /\A\w+\Z/
         
     | 
| 
      
 147 
     | 
    
         
            +
             
     | 
| 
       146 
148 
     | 
    
         
             
                  @formats.delete(format.to_sym)
         
     | 
| 
       147 
149 
     | 
    
         
             
                end
         
     | 
| 
       148 
150 
     | 
    
         | 
| 
         @@ -163,7 +165,7 @@ module IOStreams 
     | 
|
| 
       163 
165 
     | 
    
         
             
                # Returns the parser to use with tabular for the supplied file_name
         
     | 
| 
       164 
166 
     | 
    
         
             
                def self.parser_class_for_file_name(file_name)
         
     | 
| 
       165 
167 
     | 
    
         
             
                  format = nil
         
     | 
| 
       166 
     | 
    
         
            -
                  file_name.to_s.split( 
     | 
| 
      
 168 
     | 
    
         
            +
                  file_name.to_s.split(".").reverse_each do |ext|
         
     | 
| 
       167 
169 
     | 
    
         
             
                    if @formats.include?(ext.to_sym)
         
     | 
| 
       168 
170 
     | 
    
         
             
                      format = ext.to_sym
         
     | 
| 
       169 
171 
     | 
    
         
             
                      break
         
     |