logrotate 1.0.0 → 1.1.0

data/lib/logrotate/impl.rb

@@ -0,0 +1,416 @@
+#!/usr/bin/env ruby 
+
+require 'zlib'
+require 'date'
+require 'logrotate/rotateinfo'
+
+module LogRotate
+  
+  #
+  # This module contains implementation methods for the LogRotate class.
+  # In most cases, the client of this library will only use the
+  # LogRotate class directly to rotate files.  However, I have exposed a
+  # number of methods in this class, as they may be useful.
+  #
+  module Impl
+    
+    DEFAULT_COUNT = 5
+    DEFAULT_GZIP = false
+    DEFAULT_DATE_TIME_EXTENSION = false
+    DEFAULT_DATE_TIME_FORMAT = '%F'
+    DEFAULT_DATE_TIME_EXT = false
+
+    # This method validates the rotated file name with date extension.
+    # Specifically, it checks that the file name specified has a date
+    # that is formatted according to the date/time format specified.
+    #
+    # ====Parameters:
+    # [file_name]
+    #      The rotated file name to be validated.
+    # [rotation_base_name]
+    #      The base name of the rotated file (without the date/time extension).
+    # [date_time_format]
+    #      The format of the date/time extension of the rotated file.
+    #
+    def self.is_valid_date_time_file_name(file_name, rotation_base_name, date_time_format)
+      base_name = File.basename(file_name)
+      
+      if (match_data = base_name.match("^#{rotation_base_name}\.([^.]+)(\.gz)?$"))
+        # Since the caller is allowed to specify the date/time format,
+        # here we do not have a regular expression to match the
+        # date/time portion of the rotated file's extension.  As a
+        # result, some foreign files can sneak into our list of rotated
+        # files (eg. base_file.GARBAGE.gz).  We will attempt to weed
+        # them out below.
+        
+        # Validation #1: convert the date/time portion of the
+        # extension to a DateTime object and skip the file if an
+        # exception is thrown.
+        begin
+          date_time = DateTime.strptime(match_data[1], date_time_format)
+          gz = match_data[2] ? match_data[2] : ""
+          
+          # Validation #2: convert the date/time portion of the
+          # extension to a DateTime object and back to a string again.
+          # Then reconstruct the rotated file name with this string,
+          # and ensure the reconstructed rotated file matches the
+          # original.  This will take into account cases where
+          # strptime succeeded but had additional garbage characters
+          # present.  
+          # eg: date_time_format = '%F', file = 'mysql_backup.2008-08-04.BACK.gz'
+          reconstructed_file = "#{rotation_base_name}.#{date_time.strftime(date_time_format)}#{gz}"
+          if (base_name == reconstructed_file)
+            return true
+          end
+        rescue => e
+        end
+      end
+      
+      return false
+    end
+    
+    # 
+    # ====Description: 
+    # This method is passed a list of rotated file names.  This method
+    # validates that list, and returns a list of rotated file names and
+    # corresponding dates/times.  Specifically, an array is returned,
+    # where each element is a hash containing the following fields:
+    # +file+:: A rotated file name.
+    # +date_time+:: A DateTime object extracted from the rotated file name's extension.
+    #
+    # Validating the list refers to ensuring that each file name has a
+    # properly formatted date/time.  See is_valid_date_time_file_name
+    # for further information.
+    #
+    # ====Parameters:
+    # [file_names]
+    #      The rotated file names.
+    # [rotation_base_name]
+    #      The base name of the rotated file (without the date/time extension).
+    # [date_time_format]
+    #      The format of the date/time extension of the rotated file.
+    #
+    # ===Returns:
+    # A list of rotated file names and corresponding date/times.
+    #
+    def self.organize_rotated_files_date_extension(file_names, rotation_base_name, date_time_format)
+      rotated_files_and_dates = file_names.map do |file|
+        if (is_valid_date_time_file_name(file, rotation_base_name, date_time_format))
+          next get_rotated_file_and_date(file, date_time_format)
+        end
+        
+        nil
+      end
+      
+      # Remove the null entries.
+      rotated_files_and_dates = rotated_files_and_dates.select { |entry| entry }
+      
+      # Sort files by date (newer to older).
+      rotated_files_and_dates.sort! { |left, right| right[:date_time] <=> left[:date_time] }
+      
+      return rotated_files_and_dates
+    end
+    
+    # 
+    # ====Description: 
+    # This method returns a list of rotated files that are due to be deleted.
+    #
+    # The new_rotated_file field should only be set if the caller is
+    # currently in the process of rotating a file (eg. inside
+    # LogRotate::rotate_file), and the caller wants to retain 1 less
+    # file than count.  This would be because the caller will shortly
+    # create a new rotated file, and would rather remove the expired
+    # rotated file BEFORE creating the new rotated file to minimize
+    # storage space usage.  Thus, there would only be count rotated
+    # files at any time (as opposed to count + 1 if the caller first
+    # generated the new rotated file, and then deleted the expired
+    # file).
+    # ====Parameters:
+    # [rotated_files_and_dates]
+    #      The rotated file names and dates.
+    # [count]
+    #      The number of rotated files to keep.
+    # [new_rotated_file]
+    #      If the caller is in the midst of rotating a file, this value
+    #      is set to the new rotated file (yet to be created, not listed
+    #      in rotated_files_and_dates).
+    # ===Returns:
+    # A list of rotated file names and corresponding date/times to delete.
+    #
+    def self.get_expired_rotated_files_date_extension(rotated_files_and_dates, 
+                                                      count, 
+                                                      new_rotated_file = nil)
+      if (new_rotated_file)
+        # Add 1 to account for the additional rotated file that will be
+        # created after the original file passed in is rotated.
+        num_old_files = rotated_files_and_dates.length() - count + 1
+        
+        # There is a chance that the newly rotated file will have the same
+        # name as one of the already existing rotated files (eg. if the
+        # extension is simply the date, and the job is ran a second time
+        # in the same day).  In this case, there will be 1 less rotated
+        # file (and thus 1 less to delete).
+        if (rotated_files_and_dates.find {|entry| entry[:file] == new_rotated_file})
+          num_old_files -= 1
+        end
+      else
+        num_old_files = rotated_files_and_dates.length() - count
+      end
+      
+      files_to_delete = []
+      if (num_old_files > 0)
+        files_to_delete = rotated_files_and_dates.slice!(-num_old_files, num_old_files)
+      end
+      
+      return files_to_delete
+    end
+
+    # 
+    # ====Description: 
+    # This method returns a list of rotated files that are due to be deleted.
+    #
+    # The new_rotated_file field should only be set if the caller is
+    # currently in the process of rotating a file (eg. inside
+    # LogRotate::rotate_file), and the caller wants to retain 1 less
+    # file than count.  This would be because the caller will shortly
+    # create a new rotated file, and would rather remove the expired
+    # rotated file BEFORE creating the new rotated file to minimize
+    # storage space usage.  Thus, there would only be count rotated
+    # files at any time (as opposed to count + 1 if the caller first
+    # generated the new rotated file, and then deleted the expired
+    # file).
+    # ====Parameters:
+    # [rotated_files_and_counts]
+    #      The rotated file names and associated indexes.
+    # [count]
+    #      The number of rotated files to keep.
+    # [new_rotated_file]
+    #      If the caller is in the midst of rotating a file, this value
+    #      is set to the new rotated file (yet to be created, not listed
+    #      in rotated_files_and_counts).
+    # ===Returns:
+    # A list of rotated file names and corresponding indexes to delete.
+    #
+    def self.get_expired_rotated_files_integer_extension(rotated_files_and_counts, 
+                                                         count, 
+                                                         new_rotated_file = nil)
+      if (new_rotated_file)
+        count -= 1
+      end
+      deleted_files_and_counts = rotated_files_and_counts.select { |entry| entry[:index] > count }
+      
+      return deleted_files_and_counts
+    end
+    
+    # 
+    # ====Description: 
+    # This method is passed a list of rotated file names.  This method
+    # validates that list, and returns a list of rotated file names and
+    # corresponding indexes (derived from the file names' integer
+    # extensions).  Specifically, an array is returned, where each element
+    # is a hash containing the following fields: 
+    # +file+:: A rotated file name.  
+    # +index+:: The index extracted from the rotated file name's extension.
+    #
+    # Validating the list of file names specified consists of ensuring
+    # that each file name has a properly formatted integer extension.
+    #
+    # ====Parameters:
+    # [file_names]
+    #      The rotated file names.
+    # [rotation_base_name]
+    #      The base name of the rotated file (without the date/time extension).
+    #
+    # ===Returns:
+    # A list of rotated file names and corresponding indexes.
+    #
+    def self.organize_rotated_files_integer_extension(file_names, rotation_base_name)
+      rotated_files_and_counts = file_names.map do |file_name|
+
+        if (file_name.match("#{rotation_base_name}\.[0-9]+(\.gz)?$"))
+          index = file_name.match("\.([0-9]+)(\.gz)?$")[1].to_i()
+          next { :index => index, :file => file_name }
+        end
+        
+        nil
+      end
+      
+      rotated_files_and_counts = rotated_files_and_counts.select { |entry| entry }
+      
+      # Sort files by count (newer to older).
+      rotated_files_and_counts.sort! { |left, right| left[:index] <=> right[:index] }
+      
+      return rotated_files_and_counts
+    end
+    
+    def self.rotate_single_file(file, options = {})
+      gzip = options[:gzip] ? options[:gzip] : DEFAULT_GZIP
+      
+      # error checking
+      if (!File.exist?(file)) then raise "File does not exist: #{file}." end
+      if (!File.readable?(file)) then raise "File is not readable: #{file}." end
+      
+      if (options[:pre_rotate]) 
+        options[:pre_rotate].call() 
+      end
+      
+      is_date_time_ext = options[:date_time_ext] ? options[:date_time_ext] : DEFAULT_DATE_TIME_EXT
+      if (is_date_time_ext)
+        result = rotate_file_date_extension(file, options)
+      else
+        result = rotate_file_integer_extension(file, options)
+      end
+      
+      if (options[:post_rotate]) 
+        options[:post_rotate].call() 
+      end
+      
+      if (gzip) 
+        result.new_rotated_file = gzip_file(result.new_rotated_file) 
+        result.rotated_files[0] = result.new_rotated_file
+
+        if (options[:date_time_ext])
+          result.rotated_files_and_dates[0][:file] = result.new_rotated_file
+        else
+          result.rotated_files_and_counts[0][:file] = result.new_rotated_file
+        end
+      end
+      
+      return result
+    end
+    private_class_method(:rotate_single_file)
+    
+    def self.rotate_file_date_extension(file, options)
+      count = options[:count] ? options[:count] : DEFAULT_COUNT
+      
+      now = options[:date_time] ? options[:date_time] : DateTime.now()
+      
+      date_time_format = options[:date_time_format] ? options[:date_time_format] : DEFAULT_DATE_TIME_FORMAT
+      
+      # Get a list of the rotated files.
+      (source_directory, rotation_base_name) = File.split(file)
+      directory = options[:directory] ? options[:directory] : source_directory
+      
+      rotated_files = Dir.glob("#{directory}/*").select do |entry|
+        File.basename(entry).match("^#{rotation_base_name}\..+(\.gz)?$") 
+      end
+      rotated_files_and_dates = organize_rotated_files_date_extension(rotated_files,
+                                                                      rotation_base_name,
+                                                                      date_time_format)
+
+      #
+      # Remove the oldest files so that there are count rotated files remaining.
+      #
+      new_rotated_file = File.join(directory, rotation_base_name + "." + now.strftime(date_time_format))
+      deleted_files_and_dates = get_expired_rotated_files_date_extension(rotated_files_and_dates, 
+                                                                         count, 
+                                                                         new_rotated_file)
+      
+      deleted_files = deleted_files_and_dates.map { |entry| entry[:file] }
+      if (deleted_files_and_dates.length() > 0)
+        File.unlink(*deleted_files)
+      end
+      
+      # Rename the original file.
+      File.rename(file, new_rotated_file)
+      
+      # Push the newly rotated file name and associated date/time onto
+      # the front of the list of rotated files and date/times to be
+      # returned to the caller.  Note: Do not specify the newly rotated
+      # file's date/time as the original date/time used when formatting
+      # the extension.  Instead, specify the date/time with the
+      # granularity allowed by the date_time_format field.
+      rotated_files_and_dates.unshift(get_rotated_file_and_date(new_rotated_file, date_time_format))
+      
+      return RotateInfoDateExtension.new(rotated_files_and_dates,
+                                         rotated_files_and_dates.map { |entry| entry[:file] },
+                                         deleted_files,
+                                         new_rotated_file)
+    end
+    private_class_method(:rotate_file_date_extension)
+
+    def self.rotate_file_integer_extension(file, options)
+      count = options[:count] ? options[:count] : DEFAULT_COUNT
+      
+      # Get a list of the backed up files.
+      (source_directory, rotation_base_name) = File.split(file)
+      directory = options[:directory] ? options[:directory] : source_directory
+      
+      rotated_files = Dir.glob("#{directory}/*").select do |entry|
+        File.basename(entry).match("^#{rotation_base_name}\..+(\.gz)?$") 
+      end
+      rotated_files_and_counts = organize_rotated_files_integer_extension(rotated_files, rotation_base_name)
+
+      # Delete old files.
+      new_rotated_file = File.join(directory, rotation_base_name + ".1")
+      deleted_files_and_counts = get_expired_rotated_files_integer_extension(rotated_files_and_counts, 
+                                                                             count,
+                                                                             new_rotated_file)
+      deleted_files = deleted_files_and_counts.map {|entry| entry[:file] }
+      File.unlink(*deleted_files)
+      
+      rotated_files_and_counts = rotated_files_and_counts - deleted_files_and_counts
+      
+      # Sort files such that the index is descending (eg. [hello.txt.5, hello.txt.4, ..]).
+      rotated_files_and_counts.sort! { |left, right| right[:index] <=> left[:index] }
+      
+
+      # Rotate the files- Increment the index on each file.
+      rotated_files_and_counts = rotated_files_and_counts.map do |rotated_file|
+        
+        new_index = rotated_file[:index] + 1
+        new_file = rotated_file[:file].sub(/(^.*\.)(#{rotated_file[:index]})(\.gz)?$/, 
+                                           "\\1#{new_index}\\3")
+        File.rename(rotated_file[:file], new_file)
+        
+        { :index => new_index, :file => new_file }
+      end
+      
+      # Rename the original file.
+      File.rename(file, new_rotated_file)
+      
+      # Reverse the list of files so that the index is ascending (newer
+      # to older), and add the newly backed up file.
+      rotated_files_and_counts.reverse!()
+      rotated_files_and_counts.unshift({ :index => 1, :file => new_rotated_file })
+      
+      return RotateInfoIntegerExtension.new(rotated_files_and_counts,
+                                            rotated_files_and_counts.map { |entry| entry[:file] },
+                                            deleted_files,
+                                            new_rotated_file)
+    end
+    private_class_method(:rotate_file_integer_extension)
+    
+    def self.gzip_file(file)
+      # Zip the original file if necessary.
+      gzip_file = file + ".gz"
+      
+      begin
+        File.open(file) do |file_stream|
+          Zlib::GzipWriter.open(gzip_file) do |gz|
+            while (buffer = file_stream.read(1024))
+              gz.write(buffer)
+            end
+          end
+        end
+      rescue => error
+        raise "Unable to zip file: #{file}. Error: #{error}\n"
+      end
+      File.unlink(file)
+      
+      return gzip_file
+    end
+    private_class_method(:gzip_file)
+    
+    def self.get_rotated_file_and_date(file_name, date_time_format)
+      base_name = File.basename(file_name)
+      
+      match_data = base_name.match("\.([^.]+)(\.gz)?$")
+      date_time = DateTime.strptime(match_data[1], date_time_format)
+      
+      return { :date_time => date_time, :file => file_name }
+    end
+    private_class_method(:get_rotated_file_and_date)
+    
+  end
+end

data/lib/logrotate/logrotate.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby 
+
+require 'logrotate/impl'
+require 'logrotate/rotateinfo'
+
+#
+# This module provides a few methods for log rotation.
+#
+
+module LogRotate
+
+  # 
+  # ====Description: This method rotates the given files.  
+  # 
+  # ====Parameters:
+  # [file(s)]
+  #      The files to be rotated.
+  # [options = {}]
+  #      A list of optional arguments.
+  #
+  # See rotate_file for details.
+  #
+  # ====Returns:
+  # A list of RotateInfoDateExtension or
+  # RotateInfoIntegerExtension instances depending whether the
+  # option +date_time_ext+ was specified.
+  #
+  def self.rotate_files(files, options)
+    return files.map do |file|
+      Impl.send(:rotate_single_file, file, options)
+    end
+  end
+  
+  # 
+  # ====Description: 
+  # This method rotates the given file(s).  
+  # 
+  # There are 2 types of extensions that can be used: the default
+  # extension which is an integer value (".1", ".2", ..), and a
+  # date/time extension (".2008-08-01", ..).  If a date/time extension
+  # is chosen, the caller can specify a date/time format for this
+  # extension.  
+  #
+  # If this method fails, an exception will be thrown.  If this method
+  # succeeds, the method will return normally, and an instance will be
+  # returned with information about the rotated files.
+  # 
+  # ====Parameters:
+  # [file]
+  #      The file to be rotated.
+  # [options = {}]
+  #      A list of optional arguments.
+  #
+  # The optional arguments are listed below:
+  # +count+:: The number of rotated files to be kept.
+  # +directory+:: The directory to store the newly rotated file.  If this
+  #               option is unspecified, the original file's directory 
+  #               will be used.
+  # +date_time_ext+:: Whether the extension on the rotated files should be
+  #                   a date.  Possible values are: +true+, +false+.
+  # +date_time_format+:: For use with +date_time_ext+.  This specifies the
+  #                      format of the date / time extension.
+  # +date_time+:: For use with +date_time_ext+.  The +DateTime+ that will be
+  #               used to construct the extension of the newly rotated file.  
+  #               If this option is not specified, the current date and time 
+  #               will be used.
+  # +pre_rotate+:: A +Proc+ to be executed before the rotation is started.
+  # +post_rotate+:: A +Proc+ to be executed after the rotation is finished,
+  #                 and before the newly rotated file is zipped.
+  # +gzip+:: Whether the newly rotated file should be gzipped.  Possible 
+  #          values are: +true+, +false+.
+  #
+  # To see the default values of these options, see the DEFAULT_
+  # constants in LogRotate::Impl.
+  #
+  # ====Returns:
+  # A RotateInfoDateExtension or RotateInfoIntegerExtension instance
+  # depending whether the option +date_time_ext+ was specified.  This instance
+  # contains information about the rotated files.
+  #
+  def self.rotate_file(file, options)
+    return Impl.send(:rotate_single_file, file, options)
+  end
+  
+end
+