sycl 1.0

data/bin/sycl

@@ -0,0 +1,184 @@
+#!/usr/bin/env ruby
+# sycl - command line YAML manipulation tool using Sycl library
+# Andrew Ho (ho@groupon.com)
+#
+# Copyright (c) 2012, Groupon, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# Neither the name of GROUPON nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+require 'optparse'
+require 'sycl'
+require 'pp'
+
+$ME    = File.basename $0
+$USAGE = "usage: #$ME [-h] [-i] [-y] program [file1 [file2 ...]]"
+$HELP  = <<"EndHelp"
+
+#$USAGE
+
+#$ME is a command line utility to run Ruby code on one or more YAML files.
+It is similar to awk, but instead of processing lines of a file, it reads
+full files as YAML; and instead of using awk language, you write Ruby code
+which works with the data parsed from YAML.
+
+The required first argument is a literal Ruby program. When this code is
+run, the following variables are available:
+
+    f    filename of the file being parsed
+    y    the YAML text that was parsed
+    d    data parsed from interpreting YAML
+
+The d variable is a Sycl object (https://github.com/groupon/sycl)
+This lets you access hashes with dot notation (for example, the lookup
+d['foo']['bar'] can be written as d.foo.bar), including safe variants
+with get() and set() methods (d.get('foo.bar') returns nil instead of
+dying if d['foo'] does not exist). YAML output is sorted.
+
+Options:
+    -h, --help       display this help text and exit
+    -i, --inplace    overwrite YAML files in place if d is modified
+    -y, --yaml       output return value of program as YAML
+
+Examples:
+    sycl 'puts "\#{d.hostname} \#{d.interfaces.eth0.inet[0]}"' host*.yml
+    sycl -i 'd.params.ntp_servers = %w{ns1 ns2}' host2.yml
+    sycl -y d unsorted.yml > sorted.yml
+
+EndHelp
+
+def main(argv)
+  in_place = false
+  yaml_output = false
+  opts = OptionParser.new do |opts|
+    opts.on('-h', '--help')    { puts $HELP; exit 0 }
+    opts.on('-i', '--inplace') { in_place = true }
+    opts.on('-y', '--yaml')    { yaml_output = true }
+    begin
+      opts.parse! argv
+    rescue OptionParser::InvalidOption => e
+      abort "#$ME: #{e}\n#$USAGE"
+    end
+  end
+  abort "#$ME: missing required program\n#$USAGE" if argv.empty?
+  code = argv.shift
+  options = {}
+  if argv.empty?
+    abort "#$ME: cannot modify stdin in place" if in_place
+    $stderr.puts "#$ME: waiting for input from stdin..." if $stdin.tty?
+    retval = sycl_process code, $stdin, 'stdin'
+    puts retval.to_yaml if retval && yaml_output
+  else
+    modified = []
+    argv.each do |filename|
+      begin
+        fh = File.open filename
+        retval, new_data = sycl_process code, fh, filename
+        fh.close
+      rescue Errno::ENOENT, Errno::EACCES => e
+        abort "#$ME: could not open file: #{e}"
+      end
+      if in_place && new_data
+        output = new_data.to_yaml
+        output.sub! /^\-\-\- \n/, ''
+        output.gsub! /: $/m, ':'
+        begin
+          tmpfile = "#{filename}.tmp.#$$"
+          tmp = File.open tmpfile, 'w'
+          tmp.puts output
+          tmp.close
+          File.rename tmpfile, filename
+          modified << filename
+        ensure
+          begin
+            File.unlink tmpfile
+          rescue
+          end
+        end
+      end
+      puts retval.to_yaml if retval && yaml_output
+    end
+  end
+  0
+end
+
+
+# sycl_process(code, fh, filename) loads YAML from opened filehandle fh
+# into a local variable y, parses it as a Sycl object into d, and runs
+# the code string on it; the optional filename is used for error
+# reporting, and also exposed as f. It returns the return value from the
+# code that was run, and the new data structure, if it was modified.
+
+def sycl_process(code, fh, f = nil)
+  begin
+    y = fh.read
+    d_orig = Sycl::load y
+    d = Sycl::load y
+  rescue Exception => e
+    abort "#$ME: could not parse YAML from #{f || 'unknown file'}: #{e}"
+  end
+  begin
+    retval = eval code
+  rescue Exception => e
+    abort "#$ME: program error while processing #{f || 'unknown file'}: #{e}"
+  end
+  prune_nil_values! d_orig
+  prune_nil_values! d
+  if d_orig.to_yaml == d.to_yaml
+    return [ retval ]
+  else
+    return [ retval, d ]
+  end
+end
+
+
+# Walk hashes and arrays, and delete any nil values.
+
+def prune_nil_values!(o)
+  if o.is_a? Hash
+    o.keys.each do |k|
+      if o[k].nil?
+        o.delete k
+      else
+        o[k] = prune_nil_values o[k]
+        o.delete k if o[k].nil?
+      end
+    end
+    o
+  elsif o.is_a? Array
+    o.reject { |e| e.nil? }
+    o.collect! { |e| prune_nil_values e }
+  else
+    o
+  end
+end
+
+
+# Run main loop and exit
+
+exit main ARGV

data/lib/sycl.rb

@@ -0,0 +1,483 @@
+# sycl.rb - Simple YAML Configuration Library
+# Andrew Ho (ho@groupon.com)
+#
+# Copyright (c) 2012, Groupon, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# Neither the name of GROUPON nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+require 'yaml'
+
+module Sycl
+
+  # Sycl::load(yaml), Sycl::load_file(filename), and Sycl::dump(object)
+  # function just like their YAML counterparts, but return and act on
+  # Sycl-blessed variants of Hashes and Arrays.
+
+  def self.load(yaml)
+    from_object YAML::load(yaml)
+  end
+
+  def self.load_file(filename)
+    from_object YAML::load_file(filename)
+  end
+
+  def self.dump(object)
+    if (object.is_a?(::Hash)  && !object.is_a?(Sycl::Hash)) ||
+       (object.is_a?(::Array) && !object.is_a?(Sycl::Array))
+      sycl_version = from_object object
+      sycl_version.to_yaml
+    else
+      object.to_yaml
+    end
+  end
+
+  private
+
+  def self.from_object(o)
+    if o.is_a?(::Hash)
+      Sycl::Hash.from_hash(o)
+    elsif o.is_a?(::Array)
+      Sycl::Array.from_array(o)
+    else
+      o
+    end
+  end
+
+
+  # A Sycl::Array is like an Array, but creating one from an array blesses
+  # any child Array or Hash objects into Sycl::Array or Sycl::Hash objects.
+  #
+  # Sycl::Arrays support YAML preprocessing and postprocessing, and having
+  # individual nodes marked as being rendered in inline style. YAML
+  # output is also always sorted.
+
+  class Array < ::Array
+    def initialize(*args)
+      @yaml_preprocessor = nil
+      @yaml_postprocessor = nil
+      @yaml_style = nil
+      super
+    end
+
+    def self.[](*args)
+      Sycl::Array.from_array super
+    end
+
+    def self.load_file(f)
+      Sycl::Array.from_array YAML::load_file f
+    end
+
+    def self.from_array(a)
+      retval = Sycl::Array.new
+      a.each { |e| retval << Sycl::from_object(e) }
+      retval
+    end
+
+
+    # Make sure that if we write to this array, we promote any inputs
+    # to their Sycl equivalents. This lets dot notation, styled YAML,
+    # and other Sycl goodies continue.
+
+    def []=(*args)
+      raise ArgumentError => 'wrong number of arguments' unless args.size > 1
+      unless args[-1].is_a?(Sycl::Hash) || args[-1].is_a?(Sycl::Array)
+        args[-1] = Sycl::from_object(args[-1])
+      end
+      super
+    end
+
+    def <<(e)
+      unless e.is_a?(Sycl::Hash) || e.is_a?(Sycl::Array)
+        e = Sycl::from_object(e)
+      end
+      super
+    end
+
+    def collect!(&block)
+      super { |o| Sycl::from_object(block.call o) }
+    end
+    alias_method :map!, :collect!
+
+    def concat(a)
+      a = Sycl::Array.from_array(a) unless a.is_a?(Sycl::Array)
+      super
+    end
+
+    def fill(*args, &block)
+      raise ArgumentError => 'wrong number of arguments' if args.empty?
+      if block_given?
+        super { |idx| Sycl::from_object(block.call idx) }
+      else
+        unless args[0].is_a?(Sycl::Hash) || args[0].is_a?(Sycl::Array)
+          args[0] = Sycl::from_object(args[0])
+        end
+        super
+      end
+    end
+
+    def insert(i, *args)
+      raise ArgumentError => 'wrong number of arguments' if args.empty?
+      args.collect! do |o|
+        unless o.is_a?(Sycl::Hash) || o.is_a?(Sycl::Array)
+          o = Sycl::from_object(o)
+        end
+      end
+      super
+    end
+
+    def push(*args)
+      raise ArgumentError => 'wrong number of arguments' if args.empty?
+      args.collect! do |o|
+        unless o.is_a?(Sycl::Hash) || o.is_a?(Sycl::Array)
+          o = Sycl::from_object(o)
+        end
+      end
+      super
+    end
+
+    def replace(a)
+      a = Sycl::Array.from_array(a) unless a.is_a?(Sycl::Array)
+      super
+    end
+
+    def unshift(*args)
+      raise ArgumentError => 'wrong number of arguments' if args.empty?
+      args.collect! do |o|
+        unless o.is_a?(Sycl::Hash) || o.is_a?(Sycl::Array)
+          o = Sycl::from_object(o)
+        end
+      end
+      super
+    end
+
+
+    # Make this array, or its children, rendered in inline/flow style YAML.
+    # The default is to render arrays in block (multi-line) style.
+
+    def render_inline!
+      @yaml_style = :inline
+    end
+
+    def render_values_inline!
+      self.each do |e|
+        e.render_inline! if e.respond_to?(:render_inline!)
+      end
+    end
+
+
+    # Hooks to run before and after YAML dumping
+
+    def yaml_preprocessor(&block)
+      @yaml_preprocessor = block if block_given?
+    end
+
+    def yaml_postprocessor(&block)
+      @yaml_postprocessor = block if block_given?
+    end
+
+    def yaml_preprocess!
+      @yaml_preprocessor.call(self) if @yaml_preprocessor
+    end
+
+    def yaml_postprocess(yaml)
+      @yaml_postprocessor ? @yaml_postprocessor.call(yaml) : yaml
+    end
+
+
+    # The Psych YAML engine has a bug that results in infinite recursion
+    # if to_yaml is over-ridden on a non-native type.  So, we fake out
+    # Psych and pretend Sycl::Array is a native type.
+
+    class MockNativeType
+      def source_location
+        ['psych/core_ext.rb']
+      end
+    end
+
+    def method(sym)
+      sym == :to_yaml ? MockNativeType.new : super
+    end
+
+
+    # YAML rendering overrides: run preprocessing and postprocessing,
+    # set flow/inline style if this node is marked accordingly, sort
+    # elements, and suppress taguri on output. For Psych, set a long line
+    # width to more or less suppress line wrap.
+
+    if defined?(YAML::ENGINE) && YAML::ENGINE.yamler == 'psych'
+      def encode_with(coder)
+        coder.style = Psych::Nodes::Sequence::FLOW if @yaml_style == :inline
+        coder.represent_seq nil, sort
+      end
+    end
+
+    def to_yaml(opts = {})
+      yaml_preprocess!
+      if defined?(YAML::ENGINE) && YAML::ENGINE.yamler == 'psych'
+        opts ||= {}
+        opts[:line_width] ||= 999999
+        yaml = super
+      else
+        yaml = YAML::quick_emit(self, opts) do |out|
+          out.seq(nil, @yaml_style || to_yaml_style) do |seq|
+            sort.each { |e| seq.add(e) }
+          end
+        end
+      end
+      yaml_postprocess yaml
+    end
+
+  end
+
+
+  # A Sycl::Hash is like a Hash, but creating one from an hash blesses
+  # any child Array or Hash objects into Sycl::Array or Sycl::Hash objects.
+  #
+  # Hash contents can be accessed via "dot notation" (h.foo.bar means
+  # the same as h['foo']['bar']). However, h.foo.bar dies if h['foo']
+  # does not exist, so get() and set() methods exist: h.get('foo.bar')
+  # will return nil instead of dying if h['foo'] does not exist.
+  # There is also a convenient deep_merge() that is like Hash#merge(),
+  # but also descends into and merges child nodes of the new hash.
+  #
+  # Sycl::Hashes support YAML preprocessing and postprocessing, and having
+  # individual nodes marked as being rendered in inline style. YAML
+  # output is also always sorted by key.
+
+  class Hash < ::Hash
+
+    def initialize(*args)
+      @yaml_preprocessor = nil
+      @yaml_postprocessor = nil
+      @yaml_style = nil
+      super
+    end
+
+    def self.[](*args)
+      Sycl::Hash.from_hash super
+    end
+
+    def self.load_file(f)
+      Sycl::Hash.from_hash YAML::load_file f
+    end
+
+    def self.from_hash(h)
+      retval = Sycl::Hash.new
+      h.each { |k, v| retval[k] = Sycl::from_object(v) }
+      retval
+    end
+
+
+    # Make sure that if we write to this hash, we promote any inputs
+    # to their Sycl equivalents. This lets dot notation, styled YAML,
+    # and other Sycl goodies continue.
+
+    def []=(k, v)
+      unless v.is_a?(Sycl::Hash) || v.is_a?(Sycl::Array)
+        v = Sycl::from_object(v)
+      end
+      super
+    end
+    alias_method :store, :[]=
+
+    def merge!(h)
+      h = Sycl::Hash.from_hash(h) unless h.is_a?(Sycl::Hash)
+      super
+    end
+    alias_method :update, :merge!
+
+
+    # Allow method call syntax: h.foo.bar.baz == h['foo']['bar']['baz'].
+    #
+    # Accessing hash keys whose names overlap with names of Ruby Object
+    # built-in methods (id, type, etc.) will still need to be passed in
+    # with bracket notation (h['type'] instead of h.type).
+
+    def method_missing(method_symbol, *args, &block)
+      key = method_symbol.to_s
+      set = key.chomp!('=')
+      if set
+        self[key] = args.first
+      elsif self.key?(key)
+        self[key]
+      else
+        nil
+      end
+    end
+
+
+    # Safe dotted notation reads: h.get('foo.bar') == h['foo']['bar'].
+    #
+    # This will return nil instead of dying if h['foo'] does not exist.
+
+    def get(path)
+      path = path.split(/\./) if path.is_a?(String)
+      candidate = self
+      while !path.empty?
+        key = path.shift
+        if candidate[key]
+          candidate = candidate[key]
+        else
+          candidate = nil
+          last
+        end
+      end
+      candidate
+    end
+
+
+    # Dotted writes: h.set('foo.bar' => 'baz') means h['foo']['bar'] = 'baz'.
+    #
+    # This will auto-vivify any missing intervening hash keys, and also
+    # promote Hash and Array objects in the input to Scyl variants.
+
+    def set(path, value)
+      path = path.split(/\./) if path.is_a?(String)
+      target = self
+      while path.size > 1
+        key = path.shift
+        if !(target.key?(key) && target[key].is_a?(::Hash))
+          target[key] = Sycl::Hash.new
+        else
+          target[key] = Sycl::Hash.from_hash(target[key])
+        end
+        target = target[key]
+      end
+      target[path.first] = value
+    end
+
+
+    # Deep merge two hashes (the new hash wins on conflicts). Hash or
+    # and Array objects in the new hash are promoted to Sycl variants.
+
+    def deep_merge(h)
+      self.merge(h) do |key, v1, v2|
+        if v1.is_a?(::Hash) && v2.is_a?(Sycl::Hash)
+          self[key].deep_merge(v2)
+        elsif v1.is_a?(::Hash) && v2.is_a?(::Hash)
+          self[key].deep_merge(Sycl::Hash.from_hash(v2))
+        else
+          self[key] = Sycl::from_object(v2)
+        end
+      end
+    end
+
+
+    # Make Sycl::Hashes sortable alongside Sycl::Hashes and Strings.
+    # This makes YAML output in sorted order work.
+
+    include Comparable
+
+    def <=>(another)
+      self.to_str <=> another.to_str
+    end
+
+    def to_str
+      self.keys.sort.first
+    end
+
+
+    # Make this hash, or its children, rendered in inline/flow style YAML.
+    # The default is to render hashes in block (multi-line) style.
+
+    def render_inline!
+      @yaml_style = :inline
+    end
+
+    def render_values_inline!
+      self.values.each do |v|
+        v.render_inline! if v.respond_to?(:render_inline!)
+      end
+    end
+
+
+    # Hooks to run before and after YAML dumping
+
+    def yaml_preprocessor(&block)
+      @yaml_preprocessor = block if block_given?
+    end
+
+    def yaml_postprocessor(&block)
+      @yaml_postprocessor = block if block_given?
+    end
+
+    def yaml_preprocess!
+      @yaml_preprocessor.call(self) if @yaml_preprocessor
+    end
+
+    def yaml_postprocess(yaml)
+      @yaml_postprocessor ? @yaml_postprocessor.call(yaml) : yaml
+    end
+
+
+    # The Psych YAML engine has a bug that results in infinite recursion
+    # if to_yaml is over-ridden on a non-native type.  So, we fake out
+    # Psych and pretend Sycl::Hash is a native type.
+
+    class MockNativeType
+      def source_location
+        ['psych/core_ext.rb']
+      end
+    end
+
+    def method(sym)
+      sym == :to_yaml ? MockNativeType.new : super
+    end
+
+
+    # YAML rendering overrides: run preprocessing and postprocessing,
+    # set flow/inline style if this node is marked accordingly, sort by
+    # key, and suppress taguri on output. For Psych, set a long line
+    # width to more or less suppress line wrap.
+
+    if defined?(YAML::ENGINE) && YAML::ENGINE.yamler == 'psych'
+      def encode_with(coder)
+        coder.style = Psych::Nodes::Mapping::FLOW if @yaml_style == :inline
+        coder.represent_map nil, sort
+      end
+    end
+
+    def to_yaml(opts = {})
+      yaml_preprocess!
+      if defined?(YAML::ENGINE) && YAML::ENGINE.yamler == 'psych'
+        opts ||= {}
+        opts[:line_width] ||= 999999
+        yaml = super
+      else
+        yaml = YAML::quick_emit(self, opts) do |out|
+          out.map(nil, @yaml_style || to_yaml_style) do |map|
+            sort.each { |k, v| map.add(k, v) }
+          end
+        end
+      end
+      yaml_postprocess yaml
+    end
+
+  end
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification 
+name: sycl
+version: !ruby/object:Gem::Version 
+  hash: 15
+  prerelease: 
+  segments: 
+  - 1
+  - 0
+  version: "1.0"
+platform: ruby
+authors: 
+- Andrew Ho
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-03-23 00:00:00 +00:00
+default_executable: 
+dependencies: []
+
+description: This library makes using YAML for configuration files convenient and easy.
+email: rubygems@andrew.zeuscat.com
+executables: 
+- sycl
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/sycl.rb
+- bin/sycl
+has_rdoc: true
+homepage: http://zeuscat.com/andrew/sycl
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.3
+signing_key: 
+specification_version: 3
+summary: Simple YAML Config Library
+test_files: []
+