iostreams 1.10.3 → 2.0.0

data/lib/io_streams/paths/matcher.rb

@@ -3,7 +3,7 @@ module IOStreams
     # Implement fnmatch logic for any path iterator
     class Matcher
       # Characters indicating that pattern matching is required
-      MATCH_START_CHARS = /[*?\[{]/.freeze
+      MATCH_START_CHARS = /[*?\[{]/
 
       attr_reader :path, :pattern, :flags
 
@@ -50,9 +50,9 @@ module IOStreams
           @path    = path || IOStreams.path
           @pattern = pattern
         else
-          new_path = elements[0..index - 1].join("/")
+          new_path = elements[0..(index - 1)].join("/")
           @path    = path.nil? ? IOStreams.path(new_path) : path.join(new_path)
-          @pattern = elements[index..-1].join("/")
+          @pattern = elements[index..].join("/")
         end
       end
     end

data/lib/io_streams/paths/s3.rb

@@ -8,6 +8,9 @@ module IOStreams
       # Largest file size supported by the S3 copy object api.
       S3_COPY_OBJECT_SIZE_LIMIT = 5 * 1024 * 1024 * 1024
 
+      # When an upload file exceeds this size, use a multipart file upload.
+      MULTIPART_UPLOAD_SIZE = 5 * 1024 * 1024
+
       # Arguments:
       #
       # url: [String]
@@ -24,6 +27,21 @@ module IOStreams
       # secret_access_key: [String]
       #   AWS Secret Access Key Id to use to access this bucket.
       #
+      # region: [String]
+      #   The AWS region to connect to.
+      #   Defaults to region set in environment variable, or credential files.
+      #
+      # client: [Aws::S3::Client | Hash]
+      #   Supply the AWS S3 Client instance to use for this path.
+      #   Or, when a Hash, build a new client using the hash parameters.
+      #
+      #   Example:
+      #     client = Aws::S3::Client.new(endpoint: "https://s3.test.com")
+      #     IOStreams::Paths::S3.new("s3://bucket/path/file_name.txt", client: client)
+      #
+      #   Example:
+      #     IOStreams::Paths::S3.new("s3://bucket/path/file_name.txt", client: { endpoint: "https://s3.test.com" })
+      #
       # Writer specific options:
       #
       # @option params [String] :acl
@@ -133,7 +151,7 @@ module IOStreams
       #
       # @option params [String] :object_lock_legal_hold_status
       #   The Legal Hold status that you want to apply to the specified object.
-      def initialize(url, client: nil, access_key_id: nil, secret_access_key: nil, **args)
+      def initialize(url, client: nil, access_key_id: nil, secret_access_key: nil, region: nil, **args)
         Utils.load_soft_dependency("aws-sdk-s3", "AWS S3") unless defined?(::Aws::S3::Client)
 
         uri = Utils::URI.new(url)
@@ -148,6 +166,7 @@ module IOStreams
           @client_options                     = client.is_a?(Hash) ? client.dup : {}
           @client_options[:access_key_id]     = access_key_id if access_key_id
           @client_options[:secret_access_key] = secret_access_key if secret_access_key
+          @client_options[:region]            = region if region
         end
 
         @options = args
@@ -192,7 +211,7 @@ module IOStreams
 
       # Make S3 perform direct copies within S3 itself.
       def copy_to(target_path, convert: true, **args)
-        return super(target_path, convert: convert, **args) if convert || (size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
+        return super if convert || (size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
 
         target = IOStreams.new(target_path)
         return super(target, convert: convert, **args) unless target.is_a?(self.class)
@@ -269,7 +288,7 @@ module IOStreams
 
       # Shortcut method if caller has a filename already with no other streams applied:
       def write_file(file_name)
-        if ::File.size(file_name) > 5 * 1024 * 1024
+        if ::File.size(file_name) > MULTIPART_UPLOAD_SIZE
           # Use multipart file upload
           s3  = Aws::S3::Resource.new(client: client)
           obj = s3.bucket(bucket_name).object(path)

data/lib/io_streams/paths/sftp.rb

@@ -6,7 +6,7 @@ module IOStreams
     #
     # Example:
     #   IOStreams.
-    #     path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
+    #     path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret").
     #     reader do |input|
     #       puts input.read
     #     end
@@ -18,13 +18,11 @@ module IOStreams
     #
     # Example:
     #   IOStreams.
-    #     path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
+    #     path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret").
     #     writer do |output|
     #       output.write('Hello World')
     #     end
     class SFTP < IOStreams::Path
-      include SemanticLogger::Loggable if defined?(SemanticLogger)
-
       class << self
         attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds, :before_password_wait_seconds
       end
@@ -195,7 +193,7 @@ module IOStreams
             # Give time for password to be processed and stdin to be passed to sftp process.
             sleep self.class.sshpass_wait_seconds
 
-            writer.puts "get #{remote_file_name} #{local_file_name}"
+            writer.puts "get #{remote_file_name.inspect} #{local_file_name.inspect}"
             writer.puts "bye"
             writer.close
             out = reader.read.chomp
@@ -311,7 +309,7 @@ module IOStreams
 
       def build_ssh_options
         options = ssh_options.dup
-        options[:logger]       ||= logger if defined?(SemanticLogger)
+        options[:logger]       ||= IOStreams.logger if IOStreams.logger
         options[:port]         ||= port
         options[:max_pkt_size] ||= 65_536
         options[:password]     ||= @password
@@ -319,15 +317,16 @@ module IOStreams
       end
 
       def map_log_level
-        return "INFO" unless defined?(SemanticLogger)
-
-        case logger.level
+        level = IOStreams.logger&.level
+        case level
         when :trace
           "DEBUG3"
         when :warn
           "ERROR"
+        when Symbol
+          level.to_s
         else
-          logger.level.to_s
+          "INFO"
         end
       end
     end

data/lib/io_streams/pgp/reader.rb

@@ -20,18 +20,36 @@ module IOStreams
       #   Name of file to read from
       #
       # passphrase: [String]
-      #   Pass phrase for private key to decrypt the file with
-      def self.file(file_name, passphrase: nil)
+      #   Pass phrase for private key to decrypt the file with.
+      #   Not required when the file is signed but not encrypted.
+      #
+      # ignore_mdc_error: [true|false]
+      #   Decrypt files that lack MDC (Modification Detection Code) integrity protection.
+      #   Some legacy/enterprise systems (e.g. Workday) still produce such files, which
+      #   modern GnuPG refuses to decrypt with `gpg: decryption forced to fail!`.
+      #   Only enable this for files from a trusted source: without MDC the decrypted
+      #   contents are not protected against tampering.
+      #   Default: false
+      def self.file(file_name, passphrase: nil, ignore_mdc_error: false)
         # Cannot use `passphrase: self.default_passphrase` since it is considered private
         passphrase ||= default_passphrase
-        raise(ArgumentError, "Missing both passphrase and IOStreams::Pgp::Reader.default_passphrase") unless passphrase
 
-        loopback = IOStreams::Pgp.pgp_version.to_f >= 2.1 ? "--pinentry-mode loopback" : ""
-        command  = "#{IOStreams::Pgp.executable} #{loopback} --batch --no-tty --yes --decrypt --passphrase-fd 0 #{file_name}"
-        IOStreams::Pgp.logger&.debug { "IOStreams::Pgp::Reader.open: #{command}" }
+        args = []
+        # Use --pinentry-mode loopback for all GnuPG versions >= 2.1
+        args += ["--pinentry-mode", "loopback"] if IOStreams::Pgp.pgp_version.to_f >= 2.1
+        # Use --no-symkey-cache for GnuPG versions >= 2.4 to avoid caching session keys
+        args << "--no-symkey-cache" if IOStreams::Pgp.pgp_version.to_f >= 2.4
+        args << "--ignore-mdc-error" if ignore_mdc_error
+        args += ["--batch", "--no-tty", "--yes", "--decrypt"]
+        # Only feed a passphrase when one is supplied; sign-only files need none.
+        args += ["--passphrase-fd", "0"] if passphrase
+        args << file_name.to_s
+
+        command = IOStreams::Pgp.gpg_command(*args)
+        IOStreams.logger&.debug { "IOStreams::Pgp::Reader.open: #{command.shelljoin}" }
 
         # Read decrypted contents from stdout
-        Open3.popen3(command) do |stdin, stdout, stderr, waith_thr|
+        Open3.popen3(*command) do |stdin, stdout, stderr, waith_thr|
           stdin.puts(passphrase) if passphrase
           stdin.close
           result =

data/lib/io_streams/pgp/writer.rb

@@ -26,18 +26,42 @@ module IOStreams
         @audit_recipient           = nil
       end
 
-      # Write to a PGP / GPG file, encrypting the contents as it is written.
+      # Write to a PGP / GPG file, encrypting and/or signing the contents as it is written.
       #
       # file_name: [String]
       #   Name of file to write to.
       #
+      # encrypt: [true|false]
+      #   Whether to encrypt the file for the supplied recipient(s).
+      #   When set to false the file is signed but not encrypted, in which case a
+      #   :signer must be supplied and :recipient / :import_and_trust_key are ignored.
+      #   Default: true
+      #
       # recipient: [String|Array<String>]
       #   One or more emails of users for which to encrypt the file.
+      #   Ignored when encrypt is false.
       #
       # import_and_trust_key: [String|Array<String>]
       #   One or more pgp keys to import and then use to encrypt the file.
       #   Note: Ascii Keys can contain multiple keys, only the last one in the file is used.
       #
+      # import_and_trust_level: [Integer]
+      #   The owner-trust level to assign to keys supplied via :import_and_trust_key.
+      #     1 : Undefined  (no opinion)
+      #     2 : Never      (do not trust)
+      #     3 : Marginal
+      #     4 : Full
+      #     5 : Ultimate
+      #   Default: 5 : Ultimate
+      #
+      #   SECURITY WARNING:
+      #     Only import and trust keys received from a verified, trusted source.
+      #     The default trust level is `5` (Ultimate), which tells GPG to treat the imported key
+      #     as if it were one of your own keys. An ultimately trusted key is implicitly valid and
+      #     can in turn confer validity on other keys it has signed. Importing an attacker supplied
+      #     key at this level allows that attacker to impersonate other recipients.
+      #     When the key cannot be fully verified, supply a lower `import_and_trust_level`.
+      #
       # signer: [String]
       #   Name of user with which to sign the encypted file.
       #   Default: default_signer or do not sign.
@@ -46,7 +70,7 @@ module IOStreams
       #   Passphrase to use to open the private key when signing the file.
       #   Default: default_signer_passphrase
       #
-      # compression: [:none|:zip|:zlib|:bzip2]
+      # compress: [:none|:zip|:zlib|:bzip2]
       #   Note: Standard PGP only supports :zip.
       #   :zlib is better than zip.
       #   :bzip2 is best, but uses a lot of memory and is much slower.
@@ -55,58 +79,100 @@ module IOStreams
       # compress_level: [Integer]
       #   Compression level
       #   Default: 6
+      #
+      # Note: There is intentionally no option here to disable MDC (Modification Detection
+      # Code) integrity protection on the files we produce. The reader exposes
+      # `ignore_mdc_error:` so we can *consume* legacy files that lack MDC (see Reader),
+      # but we never want to *generate* them: MDC is what protects the encrypted contents
+      # against tampering, and modern GnuPG mandates it for current ciphers anyway
+      # (`--disable-mdc` is a no-op unless an obsolete cipher is forced). Omitting MDC on
+      # output would only weaken files we create, with no upside for this library.
       def self.file(file_name,
+                    encrypt: true,
                     recipient: nil,
                     import_and_trust_key: nil,
+                    import_and_trust_level: 5,
                     signer: default_signer,
                     signer_passphrase: default_signer_passphrase,
-                    compression: :zip,
-                    compress_level: 6,
-                    original_file_name: nil)
-
-        raise(ArgumentError, "Requires either :recipient or :import_and_trust_key") unless recipient || import_and_trust_key
+                    compress: :zip,
+                    compress_level: 6)
+        if encrypt
+          raise(ArgumentError, "Requires either :recipient or :import_and_trust_key") unless recipient || import_and_trust_key
+        elsif !signer
+          raise(ArgumentError, "Requires a :signer when encrypt is false")
+        end
 
-        compress_level = 0 if compression == :none
+        compress_level = 0 if compress == :none
 
-        recipients = Array(recipient)
-        recipients << audit_recipient if audit_recipient
+        recipients =
+          if encrypt
+            collect_recipients(recipient, import_and_trust_key, import_and_trust_level)
+          else
+            []
+          end
 
-        Array(import_and_trust_key).each do |key|
-          recipients << IOStreams::Pgp.import_and_trust(key: key)
-        end
+        # Write to stdin, with the encrypted and/or signed contents being written to the file
+        args = build_args(
+          file_name:         file_name,
+          encrypt:           encrypt,
+          signer:            signer,
+          signer_passphrase: signer_passphrase,
+          compress:          compress,
+          compress_level:    compress_level,
+          recipients:        recipients
+        )
+        command = IOStreams::Pgp.gpg_command(*args)
 
-        # Write to stdin, with encrypted contents being written to the file
-        command = "#{IOStreams::Pgp.executable} --batch --no-tty --yes --encrypt"
-        command << " --sign --local-user \"#{signer}\"" if signer
-        if signer_passphrase
-          command << " --pinentry-mode loopback" if IOStreams::Pgp.pgp_version.to_f >= 2.1
-          command << " --passphrase \"#{signer_passphrase}\""
-        end
-        command << " -z #{compress_level}" if compress_level != 6
-        command << " --compress-algo #{compression}" unless compression == :none
-        recipients.each { |address| command << " --recipient \"#{address}\"" }
-        command << " -o \"#{file_name}\""
-
-        IOStreams::Pgp.logger&.debug { "IOStreams::Pgp::Writer.open: #{command}" }
+        # Do not log the command, it may contain the signer passphrase.
+        action = encrypt ? "encrypt" : "sign"
+        IOStreams.logger&.debug { "IOStreams::Pgp::Writer.open: #{action} -o #{file_name}" }
 
         result = nil
-        Open3.popen2e(command) do |stdin, out, waith_thr|
+        Open3.popen2e(*command) do |stdin, out, waith_thr|
           begin
             stdin.binmode
             result = yield(stdin)
             stdin.close
           rescue Errno::EPIPE
             # Ignore broken pipe because gpg terminates early due to an error
-            ::File.delete(file_name) if ::File.exist?(file_name)
+            ::FileUtils.rm_f(file_name)
             raise(Pgp::Failure, "GPG Failed writing to encrypted file: #{file_name}: #{out.read.chomp}")
           end
           unless waith_thr.value.success?
-            ::File.delete(file_name) if ::File.exist?(file_name)
+            ::FileUtils.rm_f(file_name)
             raise(Pgp::Failure, "GPG Failed to create encrypted file: #{file_name}: #{out.read.chomp}")
           end
         end
         result
       end
+
+      def self.build_args(file_name:, encrypt:, signer:, signer_passphrase:, compress:, compress_level:, recipients:)
+        args = ["--batch", "--no-tty", "--yes"]
+        args << "--encrypt" if encrypt
+        args += ["--sign", "--local-user", signer.to_s] if signer
+        if signer_passphrase
+          args += ["--pinentry-mode", "loopback"] if IOStreams::Pgp.pgp_version.to_f >= 2.1
+          args << "--no-symkey-cache" if IOStreams::Pgp.pgp_version.to_f >= 2.4
+          args += ["--passphrase", signer_passphrase.to_s]
+        end
+        args += ["-z", compress_level.to_s] if compress_level != 6
+        args += ["--compress-algo", compress.to_s] unless compress == :none
+        recipients.each { |address| args += ["--recipient", address.to_s] }
+        args += ["-o", file_name.to_s]
+        args
+      end
+      private_class_method :build_args
+
+      def self.collect_recipients(recipient, import_and_trust_key, import_and_trust_level)
+        recipients = Array(recipient)
+        recipients << audit_recipient if audit_recipient
+
+        Array(import_and_trust_key).each do |key|
+          recipients << IOStreams::Pgp.import_and_trust(key: key, trust_level: import_and_trust_level)
+        end
+        recipients
+      end
+      private_class_method :collect_recipients
     end
   end
 end