hoodoo 1.0.2

data/lib/hoodoo/generator.rb

@@ -0,0 +1,139 @@
+########################################################################
+# File::    generator.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: Implement the +hoodoo+ command line interface. See also
+#           +/bin/hoodoo+.
+# ----------------------------------------------------------------------
+#           07-Oct-2014 (JDC): Created.
+########################################################################
+
+require 'singleton'
+require 'fileutils'
+
+module Hoodoo
+
+  # Implement the +hoodoo+ command line interface.
+  #
+  class Generator
+    include Singleton
+
+    # Kernel::exit takes a boolean but defines no constants to describe
+    # what it means; very bad form. This constant equates to the 'success'
+    # boolean value.
+    #
+    KERNEL_EXIT_SUCCESS = true
+
+    # Kernel::exit takes a boolean but defines no constants to describe
+    # what it means; very bad form. This constant equates to the 'failed'
+    # boolean value.
+    #
+    KERNEL_EXIT_FAILURE = false
+
+    # Regular expression describing allowed names of services (A-Z,
+    # a-z, 0-9, underscore or hyphen; between 2 and 30 characters).
+    #
+    NAME_REGEX = /^[a-zA-Z01-9_-]{2,30}$/
+
+    # Run the +hoodoo+ command implementation.
+    #
+    # +args+:: Array of command line arguments, excluding the +hoodoo+
+    #          command itself (so, just any extra arguments passed in).
+    #
+    def run!( args )
+      return show_usage if args_empty?(args)
+
+      name = args.first
+
+      return show_usage if name == '-h' || name == '--help'
+
+      return usage_and_warning( "SERVICE_NAME must match #{ NAME_REGEX.inspect }" ) if naughty_name?( name )
+      return usage_and_warning( "'#{ name }' already exists" ) if File.exist?( "./#{ name }" )
+
+      git = args[ 2 ] if args[ 1 ] == '--from'
+      git ||= 'git@github.com:LoyaltyNZ/service_shell.git'
+
+      return create_service( name, git )
+    end
+
+  private
+
+    def create_service( name, git )
+      if create_dir( name )               &&
+         clone_service_shell( name, git ) &&
+         remove_dot_git( name, git )      &&
+         replace_strings( name )
+
+        puts "Success! ./#{name} created."
+        Kernel::exit( KERNEL_EXIT_SUCCESS )
+      else
+        Kernel::exit( KERNEL_EXIT_FAILURE )
+      end
+    end
+
+    def create_dir( name )
+      `mkdir #{ name }`
+      $?.to_i == 0
+    end
+
+    def clone_service_shell( name, git )
+      `git clone #{ git } #{ name }`
+      $?.to_i == 0
+    end
+
+    def remove_dot_git( name, git )
+      git_folder = "./#{ name }/.git"
+      git_config = "#{ git_folder }/config"
+
+      if File.read( git_config ).include?( "url = #{ git }" ) # Paranoid
+        FileUtils.remove_dir( git_folder )
+        return true
+      else
+        raise 'Did not find a .git folder with a service_shell config file in it!'
+      end
+    end
+
+    def replace_strings( name )
+      human_name = name.split( '_' )
+      human_name = human_name.drop( 1 ) if ( human_name[ 0 ].downcase == 'service' )
+      human_name = human_name.map( &:capitalize ).join( ' ' )
+
+      base_cmd   = "LC_CTYPE=C && LANG=C && find #{ name } -type f -print0 | xargs -0 sed -i \"\" \"s/%s/g\""
+      uscore_cmd = base_cmd % "service_shell/#{ Regexp.escape( name ) }"
+      human_cmd  = base_cmd % "#{ Regexp.escape( 'Platform Service: Generic' ) }/#{ Regexp.escape( 'Platform Service: ' + human_name ) }"
+
+      puts "Replacing shell names with real service name:"
+      puts uscore_cmd
+      `#{ uscore_cmd }`
+      result = $?.to_i == 0
+      return false unless result == true
+
+      puts human_cmd
+      `#{ human_cmd }`
+      result = $?.to_i == 0
+      return result
+    end
+
+    def args_empty?( args )
+      args.empty? || args.first == ''
+    end
+
+    def naughty_name?( name )
+      !( name =~ NAME_REGEX )
+    end
+
+    def show_usage
+      puts "Usage: hoodoo SERVICE_NAME [--from <git-repository>]"
+      puts "  e.g. hoodoo service_cron"
+      puts "       hoodoo service_person --from git@github.com:YOURNAME/service_shell_fork.git"
+
+      Kernel::exit( KERNEL_EXIT_FAILURE )
+    end
+
+    def usage_and_warning( warning )
+      puts "WARNING: #{warning}"
+      puts
+      show_usage
+    end
+  end
+end

data/lib/hoodoo/logger.rb

@@ -0,0 +1,23 @@
+########################################################################
+# File::    logger.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: Include the schema based data validation and rendering code.
+# ----------------------------------------------------------------------
+#           26-Jan-2015 (ADH): Split from top-level inclusion file.
+########################################################################
+
+# Dependencies
+
+require 'hoodoo/communicators'
+
+# Logger code
+
+require 'hoodoo/logger/logger'
+require 'hoodoo/logger/writer_mixin'
+require 'hoodoo/logger/flattener_mixin'
+require 'hoodoo/logger/fast_writer'
+require 'hoodoo/logger/slow_writer'
+require 'hoodoo/logger/writers/file_writer'
+require 'hoodoo/logger/writers/stream_writer'
+require 'hoodoo/logger/writers/log_entries_dot_com_writer'

data/lib/hoodoo/logger/fast_writer.rb

@@ -0,0 +1,27 @@
+########################################################################
+# File::    fast_writer.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: Base class for fast log writers.
+# ----------------------------------------------------------------------
+#           16-Dec-2014 (ADH): Created.
+########################################################################
+
+module Hoodoo
+  class Logger
+
+    # Log writer classes are used through the Hoodoo::Logger class.
+    #
+    # Subclass FastWriter if you are writing a log data output mechanism which
+    # responds very quickly. File output might fall into this category
+    # depending upon target deployment infrastructure; printing to the console
+    # would certainly qualify.
+    #
+    # The subclass only needs to implement
+    # Hoodoo::Logger::WriterMixin#report.
+    #
+    class FastWriter
+      include Hoodoo::Logger::WriterMixin
+    end
+  end
+end

data/lib/hoodoo/logger/flattener_mixin.rb

@@ -0,0 +1,36 @@
+########################################################################
+# File::    flattener_mixin.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: Common code for fast and slow log writer base classes.
+# ----------------------------------------------------------------------
+#           19-Dec-2014 (ADH): Created.
+########################################################################
+
+require 'time'
+
+module Hoodoo
+  class Logger
+
+    # This mixin is used by custom logger subclasses and defines a single
+    # method, Hoodoo::Logger::FlattenerMixin#flatten, which takes
+    # structured log data and turns it into a single line of output
+    # including ISO 8601 time and date in local server time zone (since
+    # that makes log analysis easier for humans working in non-UTC time
+    # zone regions).
+    #
+    # Using this is of course entirely optional, but if you do use it, you
+    # will be ensuring consistent non-structured log output across any
+    # non-structured writers.
+    #
+    module FlattenerMixin
+
+      # Take the parameters from Hoodoo::Logger::WriterMixin#report and
+      # return a single line string representing the "flattened" log data.
+      #
+      def flatten( log_level, component, code, data )
+        "#{ log_level.to_s.upcase } [#{ Time.now.iso8601( 6 ) }] #{ component } - #{ code }: #{ data.inspect }"
+      end
+    end
+  end
+end

data/lib/hoodoo/logger/logger.rb

@@ -0,0 +1,387 @@
+########################################################################
+# File::    logger.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: Multiple output logging via local code or external services.
+# ----------------------------------------------------------------------
+#           16-Dec-2014 (ADH): Created.
+########################################################################
+
+module Hoodoo
+
+  # Multiple output logging via local code or external services. Instantiate
+  # a new Logger, then use #add to add _instances_ of writer classes to the
+  # collection of log writers. When #report, #debug, #info, #warn or #error
+  # are called, a corresponding log message is sent once to each of the
+  # writers, provided that the configured logging level (#level, #level=)
+  # allows it.
+  #
+  # By default, a new logger instance has no configured writers so logged
+  # messages will not go anywhere. You must use #add to add at least one
+  # writer for the instance to be useful.
+  #
+  # Some writer classes are provided by Hoodoo, including:
+  #
+  # * Hoodoo::Logger::StreamWriter - write to output streams, typically
+  #   expected to be fast, e.g. unredirected $stdout or $stderr.
+  #
+  # * Hoodoo::Logger::FileWriter - write to files, typically expected to
+  #   be relatively slow.
+  #
+  # Some loggers can preserve structural logged data (see #report) while others
+  # flatten all log messages. For example, Hoodoo::Logger::StreamWriter must
+  # flatten messages but a custom writer that, say, persisted messages in a
+  # database should be able to preserve structure.
+  #
+  # Writers are either considered fast or slow. Fast writers are called inline
+  # as soon as a message gets logged. Slow writers are called asynchronously
+  # via a Thread. A Queue is used to buffer messages for slow writers; if this
+  # gets too large, messages may be dropped. Once the slow writer catches up,
+  # a +warn+ level log message is automatically logged to report the number of
+  # dropped messages in the interim.
+  #
+  # To create a new custom writer class of any name/namespace, just subclass
+  # Hoodoo::Logger::FastWriter or Hoodoo::Logger::SlowWriter - see those
+  # classes for details.
+  #
+  class Logger
+
+    # Create a new logger instance. Once created, use #add to add writers.
+    #
+    # +component+:: Flat logging methods (see #debug, #info, #warn and #error)
+    #               are internally logged through the structured logger (see
+    #               #report) using the +component+ (again, see #report)
+    #               optionally passed here as a Symbol or String. Default is
+    #               +:Middleware+.
+    #
+    def initialize( component = :Middleware )
+      @level     = :debug
+      @pool      = Hoodoo::Communicators::Pool.new
+      @component = component
+      @writers   = {}
+    end
+
+    # Add a new writer instance to this logger. Example:
+    #
+    #     file_writer   = Hoodoo::Logger::FileWriter.new( 'output.log' )
+    #     stdout_writer = Hoodoo::Logger::StreamWriter.new
+    #
+    #     @logger = Hoodoo::Logger.new
+    #
+    #     logger.add( file_writer   )
+    #     logger.add( stdout_writer )
+    #
+    #     # ...then later...
+    #
+    #     logger.report( ... ) # -> Sends to "output.log" and $stdout
+    #
+    # +writer_instances+:: One or more _instances_ of a subclass of
+    #                      Hoodoo::Logger::FastWriter or
+    #                      Hoodoo::Logger::SlowWriter, passed as one or
+    #                      more comma-separated parameters.
+    #
+    def add( *writer_instances )
+      writer_instances.each do | writer_instance |
+        communicator = if writer_instance.is_a?( Hoodoo::Logger::FastWriter )
+          FastCommunicator.new( writer_instance, self )
+        elsif writer_instance.is_a?( Hoodoo::Logger::SlowWriter )
+          SlowCommunicator.new( writer_instance, self )
+        else
+          raise "Hoodoo::Logger\#add: Only instances of Hoodoo::Logger::FastWriter or Hoodoo::Logger::SlowWriter can be added - #{ writer_instance.class.name } was given"
+        end
+
+        @pool.add( communicator )
+        @writers[ writer_instance ] = communicator
+      end
+    end
+
+    # Remove a writer instance from this logger. If the instance has not been
+    # previously added, no error is raised.
+    #
+    # Slow writers may take a while to finish processing and shut down in
+    # the background. As a result, this method might take a while to return.
+    # Internal default timeouts may even mean that the writer is still running
+    # (possibly entirely hung).
+    #
+    # +writer_instance+:: An _instance_ of a subclass of
+    #                     Hoodoo::Logger::FastWriter or
+    #                     Hoodoo::Logger::SlowWriter.
+    #
+    def remove( writer_instance )
+      communicator = @writers[ writer_instance ]
+      @pool.remove( communicator ) unless communicator.nil?
+    end
+
+    # Remove all writer instances from this logger.
+    #
+    # Slow writers may take a while to finish processing and shut down in
+    # the background. As a result, this method might take a while to return.
+    # Internal default timeouts may even mean that one or more slow writers
+    # are still running (possibly entirely hung).
+    #
+    def remove_all
+      @pool.terminate()
+      @writers = {}
+    end
+
+    # Returns an array of all log writer instances currently in use, in order
+    # of addition. See #add.
+    #
+    def instances
+
+      # Implicit ordering relies on Ruby >= 1.9 documented behaviour of
+      # preserving order of addition to a Hash.
+      #
+      @writers.keys
+
+    end
+
+    # Wait for all writers to finish writing all log messages sent up to the
+    # point of calling. Internal default timeouts for slow writers mean that
+    # hung or extremely slow/backlogged writers may not have finished by the
+    # time the call returns, but it's necessary to enforce a timeout else
+    # this call may never return at all.
+    #
+    def wait
+      @pool.wait()
+    end
+
+    # Return or set the current log level. This is +:debug+ by default.
+    #
+    attr_accessor :level
+
+    # Given the log level configuration of this instance - see #level= and
+    # #level - should a message of the given log level be reported? Returns
+    # +true+ if so else +false+.
+    #
+    # This is mostly for internal use but external callers might find it
+    # useful from time to time, especially in tests.
+    #
+    # +log_level+:: Log level of interest as a Symbol - +debug+, +info+, +warn+
+    #               or +error+.
+    #
+    def report?( log_level )
+      return false if log_level == :debug && @level != :debug
+      return false if log_level == :info  && @level != :debug && @level != :info
+      return false if log_level == :warn  && @level != :debug && @level != :info && @level != :warn
+      return true
+    end
+
+    # Logs a message using the structured logger. Whether or not log data is
+    # written in a stuctured manner depends upon the writer(s) in use (see
+    # #add). Structured writers preserve data structures like hashes or arrays
+    # rather than (say) dumping things out as strings into flat output streams.
+    #
+    # As with flat logging methods #debug, #info, #warn and #error, a
+    # message is only logged if the logging threshold level (see #level=) is
+    # set to an equal or lower level.
+    #
+    # +log_level+:: Log level as a symbol - one of, from most trivial to most
+    #               severe, +:debug+, +:info+, +:warn+ or +:error+.
+    #
+    # +component+:: Component; for example, the resource name for a specific
+    #               resource endpoint implementation, 'Middleware' for Hoodoo
+    #               middleware itself, or some other name you think is useful.
+    #               String or Symbol.
+    #
+    # +code+::      Component-defined code. Think of this in a manner similar
+    #               to platform error codes, appearing after the "."; messages
+    #               related to the same thing should share the same code. The
+    #               intent is to produce log data that someone can filter on
+    #               code to get useful information about that specific aspect
+    #               of a service implementation's behaviour.
+    #
+    # +data+::      A Hash containing the level-, component- and code-dependent
+    #               payload data to be logged.
+    #
+    def report( log_level, component, code, data )
+      return unless self.report?( log_level )
+
+      @pool.communicate(
+        Payload.new(
+          log_level: log_level,
+          component: component,
+          code:      code,
+          data:      data
+        )
+      )
+    end
+
+    # Write a +debug+ log message, provided the log level is +:debug+.
+    #
+    # The logging data is unstructured, but gets passed to #report for
+    # structured logging under the component specified in the constructor
+    # and code 'log'.
+    #
+    # Calling #report is recommended over unstructured direct logging.
+    #
+    # *args:: One or more arguments that will be treated as strings and
+    #         written in the presented order to the log, each on its own
+    #         line of output ("\\n" terminated).
+    #
+    def debug( *args )
+      self.report( :debug, @component, :log, { '_data' => args } )
+    end
+
+    # Write an +info+ log message, provided the log level is +:debug+ or
+    # +:info+.
+    #
+    # The logging data is unstructured, but gets passed to #report for
+    # structured logging under the component specified in the constructor
+    # and code 'log'.
+    #
+    # Calling #report is recommended over unstructured direct logging.
+    #
+    # *args:: One or more arguments that will be treated as strings and
+    #         written in the presented order to the log, each on its own
+    #         line of output ("\\n" terminated).
+    #
+    def info( *args )
+      self.report( :info, @component, :log, { '_data' => args } )
+    end
+
+    # Write a +warn+ log message, provided the log level is +:debug+,
+    # +:info+ or +:warn+.
+    #
+    # The logging data is unstructured, but gets passed to #report for
+    # structured logging under the component specified in the constructor
+    # and code 'log'.
+    #
+    # Calling #report is recommended over unstructured direct logging.
+    #
+    # *args:: One or more arguments that will be treated as strings and
+    #         written in the presented order to the log, each on its own
+    #         line of output ("\\n" terminated).
+    #
+    def warn( *args )
+      self.report( :warn, @component, :log, { '_data' => args } )
+    end
+
+    # Write an +error+ log message, regardless of logging level.
+    #
+    # The logging data is unstructured, but gets passed to #report for
+    # structured logging under the component specified in the constructor
+    # and code 'log'.
+    #
+    # Calling #report is recommended over unstructured direct logging.
+    #
+    # *args:: One or more arguments that will be treated as strings and
+    #         written in the presented order to the log, each on its own
+    #         line of output ("\\n" terminated).
+    #
+    def error( *args )
+      self.report( :error, @component, :log, { '_data' => args } )
+    end
+
+    # Used internally toommunicate details of a log message across the
+    # Hoodoo::Communicators::Pool mechanism and through to a log writer.
+    # Log writer authors do not need to use this class;
+    # Hoodoo::Logger::WriterMixin unpacks it and calls your subclass's
+    # #report implementation with individual parameters for you.
+    #
+    class Payload
+
+      # Log level - see Hoodoo::Logger#report.
+      #
+      attr_reader :log_level
+
+      # Component - see Hoodoo::Logger#report.
+      #
+      attr_reader :component
+
+      # Code - see Hoodoo::Logger#report.
+      #
+      attr_reader :code
+
+      # Data - see Hoodoo::Logger#report.
+      #
+      attr_reader :data
+
+      # Create an instance. Named parameters are:
+      #
+      # +log_level+:: See Hoodoo::Logger#report.
+      # +component+:: See Hoodoo::Logger#report.
+      # +code+::      See Hoodoo::Logger#report.
+      # +data+::      See Hoodoo::Logger#report.
+      #
+      def initialize( log_level:, component:, code:, data: )
+        @log_level = log_level
+        @component = component
+        @code      = code
+        @data      = data
+      end
+    end
+
+    # Mixin used internally for the FastCommunicator and SlowCommunicator
+    # wrappers that hide implementation complexity from log writer subclasses.
+    #
+    module Communicator
+
+      # Create an instance of a logging communicator, based on the given
+      # log writer and owning logger instance.
+      #
+      # +writer_instance+:: Hoodoo::Logger::FastWriter or
+      #                     Hoodoo::Logger::SlowWriter subclass instance
+      #                     that will log things when this Communicator
+      #                     asks it to do so.
+      #
+      # +owning_logger+::   Hoodoo::Logger instance that will be using
+      #                     this communicator instance.
+      #
+      def initialize( writer_instance, owning_logger )
+        @writer_instance = writer_instance
+        @owning_logger   = owning_logger
+      end
+
+      # Implement Hoodoo::Communicators::Base#communicate for both slow and
+      # fast writers. Assumes it will be passed a Hoodoo::Logger::Payload
+      # class instance which describes the structured log data to report; also
+      # assumes it is only called when the calling logger's configured log
+      # level threshold should allow through the level of the log message in
+      # question. Calls through to the #report implementation.
+      #
+      def communicate( payload )
+        @writer_instance.report(
+          payload.log_level,
+          payload.component,
+          payload.code,
+          payload.data
+        )
+      end
+
+      # Implement optional method Hoodoo::Communicators::Slow#dropped on
+      # behalf of subclasses. The method turns the 'messages dropped'
+      # notification into a log message of +:warn+ level and which it reports
+      # internally immediately for the Writer instance only (since different
+      # writers have different queues and thus different dropped message
+      # warnings), provided that the owning Hoodoo::Logger instance log
+      # level lets warnings through.
+      #
+      def dropped( number )
+        if @owning_logger.report?( :warn )
+          @writer_instance.report(
+            :warn,
+            self.class.name,
+            'dropped.messages',
+            "Logging flooded - #{ number } messages dropped"
+          )
+        end
+      end
+    end
+
+    # Used internally as a Hoodoo::Communicator::Pool communicator wrapping
+    # fast log writer instances.
+    #
+    class FastCommunicator < Hoodoo::Communicators::Fast
+      include Communicator
+    end
+
+    # Used internally as a Hoodoo::Communicator::Pool communicator wrapping
+    # slow log writer instances.
+    #
+    class SlowCommunicator < Hoodoo::Communicators::Slow
+      include Communicator
+    end
+
+  end
+end