palletjack 0.1.2

data/lib/palletjack/keytransformer.rb

@@ -0,0 +1,264 @@
+class PalletJack
+  class KeyTransformer
+    class Reader < KeyTransformer
+      def concatenate(param, context = {})
+        context[:value].split(param) if context[:value]
+      end
+    end
+
+    class Writer < KeyTransformer
+      def concatenate(param, context = {})
+        context[:value].join(param) if context[:value]
+      end
+
+      # Internal synthesize* helper method
+      # N.B. rdoc will not be generated, because method is private.
+      #
+      # :call-seq:
+      #   synthesize_internal(param, dictionary) -> string or nil
+      #
+      # Use the single +String+ or +Enumerable+ containing +String+
+      # in +param+ to build and return a substitution value. If any
+      # failure occurs while building the new value, return +nil+.
+      #
+      # Substitutions are made from key-value pairs in +dictionary+
+      #
+      # YAML structure:
+      #
+      #   - some_rule: "rule"
+      #
+      # or
+      #
+      #   - some_rule:
+      #     - "rule"
+      #     - "rule"
+      #     ...
+      #
+      # Rules are strings used to build the new value. The value of
+      # another key is inserted by <tt>#[key]</tt>, and all other
+      # characters are copied verbatim.
+      #
+      # Rules are evaluated in order, and the first one to
+      # successfully produce a value without failing a key lookup is
+      # used.
+      #
+
+      def synthesize_internal(param, dictionary, result=String.new)
+        case param
+        when String
+          rex=/#\[([[:alnum:]._-]+)\]/
+          if md=rex.match(param) then
+            result << md.pre_match
+            return unless lookup = dictionary[md[1]]
+            result << lookup.to_s
+            synthesize_internal(md.post_match, dictionary, result)
+          else
+            result << param
+          end
+        else # Enumerable
+          param.reduce(false) do |found_one, alternative|
+            found_one || synthesize_internal(alternative, dictionary)
+          end
+        end
+      end
+      private :synthesize_internal
+
+      # Synthesize a pallet value by pasting others together.
+      #
+      # :call-seq:
+      #   synthesize(param, context)   -> string or nil
+      #
+      # If +context+ contains a non-nil +:value+, an earlier transform
+      # has already produced a value for this key, so do nothing and
+      # return +nil+.
+      #
+      # Otherwise, use the parsed YAML structure in +param+ to build
+      # and return a new value. If any failure occurs while building
+      # the new value, return +nil+ to let another transform try.
+      #
+      # YAML structure:
+      #
+      #   - synthesize: "rule"
+      #
+      # or
+      #
+      #   - synthesize:
+      #     - "rule"
+      #     - "rule"
+      #     ...
+      #
+      # Rules are strings used to build the new value. The value of
+      # another key is inserted by <tt>#[key]</tt>, and all other
+      # characters are copied verbatim.
+      #
+      # Rules are evaluated in order, and the first one to
+      # successfully produce a value without failing a key lookup is
+      # used.
+      #
+      # Example:
+      #
+      #   - net.dns.fqdn:
+      #     - synthesize: "#[net.ip.name].#[domain.name]"
+      #
+      #   - chassis.nic.name:
+      #     - synthesize:
+      #       - "p#[chassis.nic.pcislot]p#[chassis.nic.port]"
+      #       - "em#[chassis.nic.port]"
+
+      def synthesize(param, context = {})
+        return if context[:value]
+        
+        synthesize_internal(param, context[:pallet])
+      end
+
+      # Synthesize a pallet value from others, using regular
+      # expressions to pull out parts of values.
+      #
+      # :call-seq:
+      #   synthesize_regexp(param, context)   -> string or nil
+      #
+      # If +context+ contains a non-nil +:value+, an earlier transform
+      # has already produced a value for this key, so do nothing and
+      # return +nil+.
+      #
+      # Otherwise, use the parsed YAML structure in +param+ to build
+      # and return a new value. If any failure occurs while building
+      # the new value, return +nil+ to let another transform try.
+      #
+      # YAML structure:
+      #
+      #   - synthesize_regexp:
+      #       sources:
+      #         source0:
+      #           key: "key"
+      #           regexp: "regexp"
+      #         source1:
+      #           key: "key"
+      #           regexp: "regexp"
+      #         ...
+      #       produce: "recipe"
+      #
+      # where:
+      # [+sourceN+] Arbitrary number of sources for partial values,
+      #             with arbitrary names
+      # [+key+]     Name of the key to read a partial value from
+      # [+regexp+]  Regular expression for parsing the value indicated
+      #             by +key+, with named captures used to save
+      #             substrings for producing the final value. Capture
+      #             names must not be repeated within the same
+      #             synthesize_regexp block.
+      # [+produce+] A recipe for building the new value. Named
+      #             captures are inserted by <tt>#[capture]</tt>, and
+      #             all other characters are copied verbatim.
+      #
+      # Example:
+      #
+      # Take strings like +192.168.0.0_24+ from +pallet.ipv4_network+
+      # and produce strings like +192.168.0.0/24+ in +net.ipv4.cidr+.
+      #
+      #  - net.ipv4.cidr:
+      #    - synthesize_regexp:
+      #        sources:
+      #          ipv4_network:
+      #            key: "pallet.ipv4_network"
+      #            regexp: "^(?<network>[0-9.]+)_(?<prefix_length>[0-9]+)$"
+      #        produce: "#[network]/#[prefix_length]"
+
+      def synthesize_regexp(param, context = {})
+        return if context[:value]
+
+        captures = {}
+
+        param["sources"].each do |_, source|
+          # Trying to read values from a non-existent key. Return nil
+          # and let another transform try.
+          return unless lookup = context[:pallet][source["key"]]
+
+          # Save all named captures
+          Regexp.new(source["regexp"]).match(lookup) do |md|
+            md.names.each do |name|
+              captures[name] = md[name.to_sym]
+            end
+          end
+        end
+
+        synthesize_internal(param["produce"], captures)
+      end
+
+      # Synthesized value will override an inherited value for a
+      # +key+, but in some cases the intent is actually to only
+      # synthesize a value when there is no inherited value. This
+      # provides early termination of transforms for such keys.
+      #
+      # Example:
+      #
+      #  - net.layer2.name:
+      #    - inherit: ~
+      #    - synthesize: "#[chassis.nic.name]"
+
+      def inherit(_, context = {})
+        throw context[:abort] if context[:pallet][context[:key]]
+      end
+    end
+
+    def initialize(key_transforms={})
+      @key_transforms = key_transforms
+    end
+
+    # Destructively transform the values in +pallet+ according to the
+    # loaded transform rules.
+    #
+    # YAML structure:
+    #
+    #   - key:
+    #     - transform1:
+    #         [transform-specific configuration]
+    #     - transform2:
+    #         [transform-specific configuration]
+    #     [...]
+    #
+    # Transforms are evaluated in order from top to bottom, and the
+    # first one to successfully produce a value is used.
+    #
+    # Transforms are methods in PalletJack::KeyTransformer::Writer,
+    # called by name. They should return the new value, or +false+ if
+    # unsuccessful.
+    #
+    # Transforms are given two parameters, +param+ and +context+:
+    # [+param+]     transform-specific configuration from transforms.yaml
+    # [+context+]
+    #    [+pallet+] The pallet object being processed
+    #    [+key+]    The key from transforms.yaml being processed
+    #    [+value+]  Current locally assigned value for key in pallet
+    #    [+abort+]  #throw this to abort transforms for current key
+
+    def transform!(pallet)
+      @key_transforms.each do |keytrans_item|
+
+        # Enable early termination of transforms for a key
+        # by wrapping execution in a catch block.
+        catch do |abort_tag|
+          key, transforms = keytrans_item.flatten
+          context = {
+            pallet: pallet,
+            key:    key,
+            value:  pallet[key, shallow: true],
+            abort:  abort_tag
+          }
+
+          transforms.each do |t|
+            transform, param = t.flatten
+            if self.respond_to?(transform.to_sym) then
+              if new_value = self.send(transform.to_sym, param, context)
+              then
+                pallet[key] = new_value
+                break
+              end
+            end
+          end
+        end
+      end
+      @hash
+    end
+  end
+end

data/lib/palletjack/pallet.rb

@@ -0,0 +1,80 @@
+class PalletJack
+  # PalletJack managed pallet of key boxes inside a warehouse.
+  class Pallet < KVDAG::Vertex
+
+    attr_reader :name
+    attr_reader :kind
+
+    # N.B: A pallet should never be created manually; use
+    # +PalletJack::new+ to initialize a complete warehouse.
+    #
+    # [+jack+] PalletJack that will manage this pallet.
+    # [+path+] Filesystem path to pallet data.
+    #
+    # Create PalletJack managed singletonish pallet.
+    #
+    # Use relative path inside of warehouse as kind/name for this
+    # pallet, and make a singletonish object for that key.
+
+    def Pallet.new(jack, path) #:doc:
+      ppath, name = File.split(path)
+      _, kind = File.split(ppath)
+
+      jack.pallets[kind] ||= Hash.new
+      jack.pallets[kind][name] || super
+    end
+
+    # N.B: A pallet should never be created manually; use
+    # +PalletJack::new+ to initialize a complete warehouse.
+    #
+    # [+jack+] PalletJack that will manage this pallet.
+    # [+path+] Filesystem path to pallet data.
+    #
+    # Loads and merges all YAML files in +path+ into this Vertex.
+    #
+    # Follows all symlinks in +path+ and creates edges towards
+    # the pallet located in the symlink target.
+
+    private :initialize
+    def initialize(jack, path) #:notnew:
+      @jack = jack
+      @path = path
+      ppath, @name = File.split(path)
+      _, @kind = File.split(ppath)
+      boxes = Array.new
+
+      super(jack, pallet:{@kind => @name})
+
+      Dir.foreach(path) do |file|
+        next if file[0] == '.'
+        filepath = File.join(path, file)
+        filestat = File.lstat(filepath)
+        case
+        when (filestat.file? and file =~ /\.yaml$/)
+          merge!(YAML::load_file(filepath))
+          boxes << file
+        when filestat.symlink?
+          link = File.readlink(filepath)
+          _, lname = File.split(link)
+
+          pallet = Pallet.new(jack, File.absolute_path(link, path))
+          edge(pallet, pallet:{references:{file => lname}})
+        end
+      end
+      merge!(pallet:{boxes: boxes})
+      @jack.keytrans_writer.transform!(self)
+      @jack.pallets[@kind][@name] = self
+    end
+
+    def inspect
+      "#<%s:%x>" % [self.class, self.object_id, @path]
+    end
+
+    # Override standard to_yaml serialization, because pallet objects
+    # are ephemeral by nature. The natural serialization is that of
+    # their to_hash analogue.
+    def to_yaml
+      to_hash.to_yaml
+    end
+  end
+end

data/lib/palletjack/tool.rb

@@ -0,0 +1,463 @@
+require 'palletjack'
+require 'fileutils'
+require 'optparse'
+require 'singleton'
+require 'rugged'
+
+class PalletJack
+
+  # Superclass for PalletJack tool implementations
+  #
+  # Provides convenience methods for option parsing, file generation,
+  # and warehouse structure management.
+  #
+  # Example:
+  #   require 'palletjack/tool'
+  #
+  #   class MyTool < PalletJack::Tool
+  #     def parse_options(parser)
+  #       parser.on('-o DIR', '--output DIR',
+  #                 'output directory',
+  #                 String) {|dir| options[:output] = dir }
+  #
+  #       required_option :output
+  #     end
+  #
+  #     attr_reader :state
+  #
+  #     def process
+  #       @state = {}
+  #       jack.each(kind:'system') do |sys|
+  #         @state[sys.name] = sys
+  #       end
+  #     end
+  #
+  #     def output
+  #       @state.each do |name, data|
+  #         config_dir :output, name
+  #         config_file :output, name, "dump.yaml" do |file|
+  #           file << data.to_yaml
+  #         end
+  #       end
+  #     end
+  #   end
+  #
+  #   if __FILE__ == $0
+  #     MyTool.run
+  #   end
+
+  class Tool
+    include Singleton
+
+    # v0.1.1 API:
+    #
+    # :call-seq:
+    # run
+    #
+    # Main tool framework driver.
+    #
+    # Run the entire tool; setup, process and output. Actual tools
+    # will want to use this function, while testing and other
+    # activities that require poking around in internal state will
+    # want to run the partial functions instead.
+    #
+    # Example:
+    #
+    #   if MyTool.standalone?(__FILE__)
+    #     MyTool.run
+    #   end
+
+    # v0.1.0 API, retained until all tools have been updated to
+    # v0.1.1:
+    #
+    # :call-seq:
+    # run &block
+    #
+    # Run the +block+ given in the context of the tool singleton instance
+    # as convenience for simple tools.
+    #
+    # More complex tools probably want to override parse_options to
+    # add option parsing, and split functionality into multiple methods.
+    #
+    # Example:
+    #
+    #   MyTool.run { jack.each(kind:'system') {|sys| puts sys.to_yaml } }
+
+    def self.run(&block)
+      if block
+        # v0.1.0 API. When all tools have been ported, remove this and
+        # bump the minor version number.
+        instance.setup
+        instance.instance_eval(&block)
+      else
+        # v0.1.1 API
+        instance.setup
+        instance.process
+        instance.output
+      end
+    end
+
+    # Predicate for detecting if we are being invoked as a standalone
+    # tool, or loaded by e.g. a test framework.
+
+    def self.standalone?(file)
+      File::basename(file) == File::basename($0)
+    end
+
+    # Generate data in an internal format, saving it for later testing
+    # or writing to disk by #output.
+    #
+    # Override this function in specific tool classes.
+    #
+    # Example:
+    #
+    #   class MyTool < PalletJack::Tool
+    #     def process
+    #       @systems = Set.new
+    #       jack.each(kind:'system') do |s|
+    #         @systems << s
+    #       end
+    #     end
+    #   end
+
+    def process
+    end
+
+    # Output data in its final format, probably to disk or stdout.
+    #
+    # Example:
+    #
+    #   class MyTool < PalletJack::Tool
+    #     def output
+    #       @systems.each do |s|
+    #         puts s.name
+    #       end
+    #     end
+    #   end
+
+    def output
+    end
+
+    # Return the command line argument list to be used. Replace this
+    # method when testing.
+
+    def argv
+      ARGV
+    end
+
+    # Set up the singleton instance
+    #
+    # Default setup will add options for --warehouse and --help to the
+    # OptionParser, and set the banner to something useful.
+    #
+    # Any exceptions raised during option parsing will abort execution
+    # with usage information.
+
+    def setup
+      @parser = OptionParser.new
+      @options = {}
+      @option_checks = []
+
+      @parser.banner = "Usage: #{$PROGRAM_NAME} -w <warehouse> [options]"
+      @parser.separator ''
+      @parser.on('-w DIR', '--warehouse DIR',
+                 'warehouse directory', String) {|dir|
+        @options[:warehouse] = dir }
+      @parser.on_tail('-h', '--help', 'display usage information') {
+        raise ArgumentError }
+
+      parse_options(@parser)
+
+      @parser.parse!(argv)
+      @option_checks.each {|check| check.call }
+    rescue
+      abort(usage)
+    end
+
+    # Additional option parsing
+    #
+    # The default instance initalization will add option parsing for
+    # <tt>-w</tt>/<tt>--warehouse</tt> and <tt>-h</tt>/<tt>--help</tt>,
+    # and a simple banner string.
+    #
+    # Implementations needing more options than the default, a more
+    # informative banner, or requirement checks for parsed options should
+    # override this empty method.
+    #
+    # Any exceptions raised will abort execution with usage information.
+    #
+    # Example:
+    #
+    #   class MyTool < PalletJack::Tool
+    #     def parse_options(parser)
+    #       parser.on('-o DIR', '--output DIR',
+    #                 'output directory',
+    #                 String) {|dir| options[:output] = dir }
+    #
+    #       required_option :output
+    #     end
+    #   end
+
+    def parse_options(parser)
+    end
+
+    # Require the presence of one of the given options.
+    #
+    # Must not be called outside the scope of the parse_options method.
+    #
+    # Raises ArgumentError if none exist in options[]
+    #
+    # Example:
+    #
+    #   def parse_options(parser)
+    #     ...
+    #     required_option :output
+    #   end
+
+    def required_option(*opts)
+      @option_checks << lambda do
+        raise ArgumentError unless opts.any? {|opt| options[opt]}
+      end
+    end
+
+    # Require the presence of no more than one of the given options.
+    #
+    # Must not be called outside the scope of the parse_options method.
+    #
+    # Raises ArgumentError if more than one exist in options[]
+    #
+    # Example:
+    #
+    #   def parse_options(parse)
+    #     ...
+    #     required_option :output_file, :output_stdout
+    #     exclusive_options :output_file, :output_stdout
+    #   end
+
+    def exclusive_options(*opts)
+      @option_checks << lambda do
+        raise ArgumentError if opts.count {|opt| options[opt]} > 1
+      end
+    end
+
+    # Usage information from option parser
+    #
+    # Example:
+    #
+    #   abort(usage) unless options[:warehouse]
+
+    def usage
+      @parser.to_s
+    end
+
+    # Hash containing all parsed options.
+
+    attr_reader :options
+
+    # Pallet containing all warehouse defined configuration options
+    #
+    # Configuration options for tools can be stored as pallets in
+    # the warehouse:
+    #
+    #   _config
+    #     |
+    #     +-- MyTool
+    #     |     |
+    #     |     `-- somecfg.yaml
+
+    def config
+      @config ||= jack.fetch(kind:'_config',
+                             name: self.class.to_s) rescue Hash.new
+    end
+
+    # Return the PalletJack object for <tt>--warehouse</tt>
+    # Aborts execution with usage message if the warehouse was
+    # not specified.
+
+    def jack
+      abort(usage) unless options[:warehouse]
+      @jack ||= PalletJack.load(options[:warehouse])
+    end
+
+    # Build a filesystem path from path components
+    #
+    # Symbols are looked up in the options dictionary.
+    # All other types are converted to strings. The
+    # resulting list is fed to File#join to produce a
+    # local filesystem compliant path.
+    #
+    # Example:
+    #   parser.on(...) {|dir| options[:output] = dir }
+    #   ...
+    #   config_path :output, 'subdir1'
+    #   config_path :output, 'subdir2'
+
+    def config_path(*path)
+      File.join(path.map {|item|
+                  case item
+                  when Symbol
+                    options.fetch(item)
+                  else
+                    item.to_s
+                  end
+                })
+    end
+
+    # :call-seq:
+    # config_dir '', 'path', 'name'
+    # config_dir :option, 'subdir', ...
+    #
+    # Creates a directory if it doesn't already exist.
+    #
+    # Uses config_path to construct the path, so any symbols will
+    # be looked up in the options hash.
+    #
+    # Example:
+    #
+    #   config_dir :output, system.name
+
+    def config_dir(*path)
+      Dir.mkdir(config_path(*path))
+    rescue Errno::EEXIST
+      nil
+    end
+
+    # :call-seq:
+    # config_file 'filename' {|file| ... }
+    # config_file :option, 'fragment', 'base.ext' {|file| ... }
+    # config_file ..., mode:0600 {|file| ...}
+    #
+    # Creates a configuration file, with default mode:0644
+    # and calls the given block with the file as argument.
+    #
+    # Uses config_path to construct the path, so any symbols will
+    # be looked up in the options hash.
+    #
+    # N.B! If the file already exists, it will be overwritten!
+    #
+    # Example:
+    #
+    #   config_file :output, system.name, 'dump.yaml' do |file|
+    #     file << system.to_yaml
+    #   end
+
+    def config_file(*path, mode:0644, &block)
+      File.open(config_path(*path),
+                File::CREAT | File::TRUNC | File::WRONLY, mode) do |file|
+        block.call(file)
+      end
+    end
+
+    # Create a new pallet directory inside the warehouse
+    #
+    # Uses config_dir to create the directory, so any symbols will
+    # be looked up in the options hash.
+    #
+    # This is effectively a noop if the pallet already exists.
+    #
+    # Example:
+    #
+    #   pallet_dir 'system', :system_name
+
+    def pallet_dir(kind, name)
+      config_dir :warehouse, kind
+      config_dir :warehouse, kind, name
+    end
+
+    # Write a key box file inside a pallet
+    #
+    # The block should return a hash representing the contents of the box.
+    # All keys will be stringified, so we can use key: short forms for
+    # declaration of the box contents.
+    #
+    # Uses config_file to create the file, so any symbols will
+    # be looked up in the options hash.
+    #
+    # N.B! If the box already exists, it will be overwritten!
+    #--
+    # FIXME: should new values be merged with existing box data instead?
+    #++
+    #
+    # Example:
+    #
+    #   pallet_box 'domain', :domain, 'dns' do
+    #     { dns:{ ns:options[:soa_ns].split(',') } }
+    #   end
+
+    def pallet_box(kind, name, box, &block)
+      config_file :warehouse, kind, name, "#{box}.yaml" do |box_file|
+        box_file << block.call.deep_stringify_keys.to_yaml
+      end
+    end
+
+    # Create links from a pallet to parents
+    #
+    # Uses config_path to construct paths within the warehouse, so any
+    # symbols will be looked up in the options hash.
+    #
+    # +links+ is a hash containing +link_type+=>[+parent_kind+, +parent_name+]
+    #
+    # If the link target is empty (e.g. +link_type+=>[]), the link is removed.
+    #
+    # Example:
+    #
+    #   pallet_links 'system', :system, 'os'=>['os', :os], 'netinstall'=>[]
+
+    def pallet_links(kind, name, links={})
+      links.each do |link_type, parent|
+        link_path = config_path(:warehouse, kind, name, link_type)
+
+        begin
+          File.delete(link_path)
+        rescue Errno::ENOENT
+          nil
+        end
+        unless parent.empty?
+          parent_kind, parent_name = parent
+          parent_path = config_path('..', '..', parent_kind, parent_name)
+
+          File.symlink(parent_path, link_path)
+        end
+      end
+    end
+
+    # Return a string stating the Git provenance of a warehouse directory,
+    # suitable for inclusion at the top of a generated configuration file,
+    # with each line prefixed by +comment_char+.
+    #
+    # If <tt>options[:warehouse]</tt> points within a Git repository,
+    # return a string stating its absolute path and active branch. If
+    # +include_id+ is true, also include the commit ID of the branch's
+    # HEAD.
+    #
+    # If Git information cannot be found for
+    # <tt>options[:warehouse]</tt>, return a string stating its path
+    # and print an error message on stderr.
+
+    def git_header(tool_name, comment_char: '#', include_id: false)
+      repo = Rugged::Repository.discover(options[:warehouse])
+      workdir = repo.workdir
+      branch = repo.head.name
+      commit = repo.head.target_id
+
+      header =
+"#{comment_char}#{comment_char}
+#{comment_char}#{comment_char} Automatically generated by #{tool_name} from
+#{comment_char}#{comment_char} Repository: #{workdir}
+#{comment_char}#{comment_char} Branch: #{branch}\n"
+      if include_id
+      then
+        header +=
+          "#{comment_char}#{comment_char} Commit ID: #{repo.head.target_id}\n"
+      end
+      header += "#{comment_char}#{comment_char}\n"
+      return header
+    rescue
+      STDERR.puts "Error finding Git sourcing information: #{$!}"
+      return "#{comment_char}#{comment_char}
+#{comment_char}#{comment_char} Automatically generated by #{tool_name} from
+#{comment_char}#{comment_char} Warehouse: #{warehouse_path}
+#{comment_char}#{comment_char}\n"
+    end
+  end
+end