hash-joiner 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49618706629e42c71fde8cb81f186d1c16a65468
-  data.tar.gz: cee8620a4dc55411b95385d9f01dae8fd1ab40c8
+  metadata.gz: 759dc6bcf51ede5903167adf68613554adcec044
+  data.tar.gz: 3fd0a5e98271be1025d65b5c9610f7d73b127c63
 SHA512:
-  metadata.gz: a57bca28a0d760274250824714f0e8033ec1ad3a59e925fb878910c435ea20c61aae0ac23d20054f591d9a1d133c381c42812001082432e9a0eb650c5e394686
-  data.tar.gz: 0aeba3a5e17a911d7449b10da5efd80f510f56be597b0631d54dc39b196ff0e17cd441e8318055b4cc7fc49551d35af4c31aaf776a601c8afe7358dd38a51cb7
+  metadata.gz: 429156c991502011b29fc54d6aff1b1495b0eeb80f19b55b44394f3bdfd65be180b31407b74ded53a0005c16aad947a72edbfb711066b1e173c7822dc61694fe
+  data.tar.gz: 2658c4cebb926998d05c3add6830c6f55c65324ffede7021e40ddadad4f3df57c4e4efbbd5c8b92f887e977e011db5367e2c73393edb00120cd01f43a6627923

data/README.md

@@ -1,7 +1,7 @@
 ## hash-joiner Gem
 
 [![Gem Version](https://badge.fury.io/rb/hash-joiner.svg)](http://badge.fury.io/rb/hash-joiner)
-[![Build Status](https://travis-ci.org/18F/hash-joiner.svg?branch=doc-update)](https://travis-ci.org/18F/hash-joiner)
+[![Build Status](https://travis-ci.org/18F/hash-joiner.svg?branch=master)](https://travis-ci.org/18F/hash-joiner)
 [![Code Climate](https://codeclimate.com/github/18F/hash-joiner/badges/gpa.svg)](https://codeclimate.com/github/18F/hash-joiner)
 [![Test Coverage](https://codeclimate.com/github/18F/hash-joiner/badges/coverage.svg)](https://codeclimate.com/github/18F/hash-joiner)
 
@@ -140,6 +140,20 @@ This corresponds to the process of joining different collections of Jekyll-impor
       {"name"=>"bazquux", "email"=>"baz.quux@gsa.gov"}]}
 ```
 
+#### Running `filter-yaml-files`
+
+The `filter-yaml-files` program can be used to generate "public" versions of YAML files containing "private" data. For example:
+
+```
+$ export HUB_DATA_DIR=../hub/_data
+
+$ filter-yaml-files ${HUB_DATA_DIR}/private/{team,projects}.yml -o ${HUB_DATA_DIR}/public
+../hub/_data/private/team.yml => ../hub/_data/public/team.yml
+../hub/_data/private/projects.yml => ../hub/_data/public/projects.yml
+```
+
+The `filter-yaml-files` program can also strip other properties besides `private:`, and can promote data contained within a property rather than strip it. Run `filter-yaml-files -h` to see the options that allow this.
+
 ### Contributing
 
 Just fork [18F/hash-joiner](https://github.com/18F/hash-joiner) and start sending pull requests! Feel free to ping [@mbland](https://github.com/mbland) with any questions you may have, especially if the current documentation should've addressed your needs, but didn't.

data/bin/filter-yaml-files

@@ -0,0 +1,159 @@
+#! /usr/bin/env ruby
+# hash-joiner - Pruning, promoting, deep-merging, and joining Hash data
+#
+# Written in 2015 by Mike Bland (michael.bland@gsa.gov)
+# on behalf of the 18F team, part of the US General Services Administration:
+# https://18f.gsa.gov/
+#
+# To the extent possible under law, the author(s) have dedicated all copyright
+# and related and neighboring rights to this software to the public domain
+# worldwide. This software is distributed without any warranty.
+#
+# You should have received a copy of the CC0 Public Domain Dedication along
+# with this software. If not, see
+# <https://creativecommons.org/publicdomain/zero/1.0/>.
+#
+# ---
+#
+# Script to generate 'public' versions of 'private' YAML files.
+#
+# The command line flags support stripping object properties other than
+# `private:`, or promoting the data associated with the property rather than
+# stripping it out.
+#
+# Author: Mike Bland (michael.bland@gsa.gov)
+# Date:   2015-01-11
+
+require 'hash-joiner'
+require 'optparse'
+require 'safe_yaml'
+
+options = {
+  :property => 'private',
+  :output_dir => 'public',
+}
+
+opt_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [-hop] [--promote] [file ...]"
+
+  opts.separator ''
+  opts.separator <<EOF
+By default, reads a collection of YAML files containing "private" data, i.e.
+objects containing a `private:` property, and generates new files with the
+private data stripped out.
+
+Using the option flags, other properties besides `private:` can be removed, or
+the data associated with the target PROPERTY can be promoted rather than
+stripped.
+
+All input files should come from the same parent directory. This is because
+the directory structure beneath the parent directory will be preserved in the
+OUTPUT_DIR.
+
+Options:
+EOF
+
+  opts.on('-h', '--help', "Show this help") do
+    puts opts
+    exit
+  end
+
+  opts.on('-p', '--property PROPERTY',
+    'Property to strip/promote ' +
+    "(default: #{options[:property]})") do |property|
+    options[:property] = property
+  end
+
+  opts.on('-o', '--output_dir OUTPUT_DIR',
+    "Output directory (default: #{options[:output_dir]})") do |output_dir|
+    options[:output_dir] = output_dir
+  end
+
+  opts.on('--promote',
+    'Promote the PROPERTY rather than strip it') do |promote|
+    options[:promote] = promote
+  end
+end
+opt_parser.parse!
+
+if ARGV.length < 1
+  STDERR.puts 'No input files specified'
+  exit 1
+end
+
+input_files = []
+errors = []
+parent_dir = ''
+
+ARGV.each do |input_file|
+  current_parent_dir = File.dirname input_file
+  parent_dir = current_parent_dir if parent_dir.empty?
+  unless (parent_dir.start_with? current_parent_dir or
+    current_parent_dir.start_with? parent_dir)
+    STDERR.puts 'All input files should come from the same parent directory.'
+    STDERR.puts "Detected: #{parent_dir} and #{current_parent_dir}"
+    exit 1
+  end
+  parent_dir = current_parent_dir if current_parent_dir.size < parent_dir.size
+
+  if !File.exists? input_file
+    errors << "File does not exist: #{input_file}"
+  elsif !File.readable? input_file
+    errors << "File not readable: #{input_file}"
+  else
+    input_files << input_file
+  end
+end
+
+unless errors.empty?
+  STDERR.puts errors.join "\n"
+  STDERR.puts "Aborting; no files processed."
+  exit 1
+end
+
+def recursive_mkdir(dirname, parent_dir)
+  parent_dir = File::SEPARATOR if dirname.start_with? File::SEPARATOR
+  dir_components = dirname.split(File::SEPARATOR)
+  current_subdir = parent_dir
+  until dir_components.empty?
+    current_subdir = File.join(current_subdir, dir_components.shift)
+    Dir.mkdir(current_subdir) unless Dir.exists? current_subdir
+  end
+end
+
+FILTERED_OUTPUT_DIR = options[:output_dir]
+FILTERED_PROPERTY = options[:property]
+FILTER_OPERATION = options[:promote] ? :promote_data : :remove_data
+FILTERED_PARENT_DIR_SIZE = parent_dir.concat(File::SEPARATOR).size
+
+recursive_mkdir FILTERED_OUTPUT_DIR, Dir.pwd
+
+input_files.each do |input_file|
+  data = SafeYAML.load_file(input_file, :safe=>true)
+  unless data
+    errors << "Failed to parse #{source}"
+    next
+  end
+
+  begin
+    # Make the output subdirectory hierarchy if necessary.
+    subdirs = File.dirname(input_file)[FILTERED_PARENT_DIR_SIZE..-1] || ''
+    recursive_mkdir subdirs, FILTERED_OUTPUT_DIR
+
+    output_file = File.join(FILTERED_OUTPUT_DIR, subdirs,
+      File.basename(input_file))
+    open(output_file, 'w') do |outfile|
+      puts "#{input_file} => #{output_file}"
+      data = HashJoiner.send FILTER_OPERATION, data, FILTERED_PROPERTY
+      outfile.puts data.to_yaml
+    end
+  rescue SystemCallError => e
+    errors << e.to_s
+  end
+end
+
+unless errors.empty?
+  STDERR.puts "\n*** Errors:"
+  STDERR.puts errors.join "\n"
+  exit 1
+end

data/lib/hash-joiner.rb

@@ -30,28 +30,46 @@ module HashJoiner
   # @return [nil] if +collection+ is not a +Hash+ or +Array<Hash>+
   def self.promote_data(collection, key)
     if collection.instance_of? ::Hash
-      if collection.member? key
-        data_to_promote = collection[key]
-        collection.delete key
-        deep_merge collection, data_to_promote
-      end
-      collection.each_value {|i| promote_data i, key}
-
+      promote_hash_data collection, key
     elsif collection.instance_of? ::Array
-      collection.each do |i|
-        # If the Array entry is a hash that contains only the target key,
-        # then that key should map to an Array to be promoted.
-        if i.instance_of? ::Hash and i.keys == [key]
-          data_to_promote = i[key]
-          i.delete key
-          deep_merge collection, data_to_promote
-        else
-          promote_data i, key
-        end
-      end
+      promote_array_data collection, key
+    end
+  end
 
-      collection.delete_if {|i| i.empty?}
+  # Recursively promotes data within a Hash. Used to implement promote_data.
+  #
+  # @param collection [Hash] collection in which to promote information
+  # @param key [String] property to be promoted within +collection+
+  # @return [Hash] +collection+ after promotion
+  # @see promote_data
+  def self.promote_hash_data(collection, key)
+    if collection.member? key
+      data_to_promote = collection[key]
+      collection.delete key
+      deep_merge collection, data_to_promote
+    end
+    collection.each_value {|i| promote_data i, key}
+  end
+
+  # Recursively promotes data within an Array. Used to implement promote_data.
+  #
+  # @param collection [Array] collection in which to promote information
+  # @param key [String] property to be promoted within +collection+
+  # @return [Array] +collection+ after promotion
+  # @see promote_data
+  def self.promote_array_data(collection, key)
+    collection.each do |i|
+      # If the Array entry is a hash that contains only the target key,
+      # then that key should map to an Array to be promoted.
+      if i.instance_of? ::Hash and i.keys == [key]
+        data_to_promote = i[key]
+        i.delete key
+        deep_merge collection, data_to_promote
+      else
+        promote_data i, key
+      end
     end
+    collection.delete_if {|i| i.empty?}
   end
 
   # Raised by +deep_merge+ if +lhs+ and +rhs+ are of different types.
@@ -59,6 +77,26 @@ module HashJoiner
   class MergeError < ::Exception
   end
 
+  # The set of mergeable classes
+  MERGEABLE_CLASSES = [::Hash, ::Array]
+
+  # Asserts that +lhs+ and +rhs+ are of the same type and can be merged.
+  #
+  # @param lhs [Hash,Array] merged data sink (left-hand side)
+  # @param rhs [Hash,Array] merged data source (right-hand side)
+  # @raise [MergeError] if +lhs+ and +rhs+ are of the different types or are
+  #   of a type that cannot be merged
+  # @return [nil]
+  # @see deep_merge
+  def self.assert_objects_are_mergeable(lhs, rhs)
+    if lhs.class != rhs.class
+      raise MergeError.new("LHS (#{lhs.class}): #{lhs}\n" +
+        "RHS (#{rhs.class}): #{rhs}")
+    elsif !MERGEABLE_CLASSES.include? lhs.class
+      raise MergeError.new "Class not mergeable: #{lhs.class}"
+    end
+  end
+
   # Performs a deep merge of +Hash+ and +Array+ structures. If the collections
   # are +Hash+es, +Hash+ or +Array+ members of +rhs+ will be deep-merged with
   # any existing members in +lhs+. If the collections are +Array+s, the values
@@ -70,30 +108,61 @@ module HashJoiner
   # @raise [MergeError] if +lhs+ and +rhs+ are of different classes, or if
   #   they are of classes other than Hash or Array.
   def self.deep_merge(lhs, rhs)
-    mergeable_classes = [::Hash, ::Array]
-
-    if lhs.class != rhs.class
-      raise MergeError.new("LHS (#{lhs.class}): #{lhs}\n" +
-        "RHS (#{rhs.class}): #{rhs}")
-    elsif !mergeable_classes.include? lhs.class
-      raise MergeError.new "Class not mergeable: #{lhs.class}"
-    end
+    assert_objects_are_mergeable lhs, rhs
 
     if rhs.instance_of? ::Hash
-      rhs.each do |key,value|
-        if lhs.member? key and mergeable_classes.include? value.class
-          deep_merge(lhs[key], value)
-        else
-          lhs[key] = value
-        end
-      end
-
+      deep_merge_hashes lhs, rhs
     elsif rhs.instance_of? ::Array
       lhs.concat rhs
     end
     lhs
   end
 
+  # Asserts that +rhs_value+ can be merged into +lhs_value+ for the property
+  # identified by +key+.
+  #
+  # @param key [String] Hash property name
+  # @param lhs_value [Object] the property value of the data sink Hash
+  #   (left-hand side value)
+  # @param rhs_value [Object] the property value of the data source Hash
+  #   (right-hand side value)
+  # @raise [MergeError] if +lhs_value+ exists and +rhs_value+ is of a
+  #   different class
+  # @return [nil]
+  # @see deep_merge
+  # @see deep_merge_hashes
+  def self.assert_hash_properties_are_mergeable(key, lhs_value, rhs_value)
+    lhs_class = lhs_value == false ? ::TrueClass : lhs_value.class
+    rhs_class = rhs_value == false ? ::TrueClass : rhs_value.class
+
+    unless lhs_value.nil? or lhs_class == rhs_class
+      raise MergeError.new(
+        "LHS[#{key}] value (#{lhs_class}): #{lhs_value}\n" +
+        "RHS[#{key}] value (#{rhs_class}): #{rhs_value}")
+    end
+    nil
+  end
+
+  # Performs a deep merge of Hash structures. Used to implement +deep_merge+.
+  #
+  # @param lhs [Hash] merged data sink (left-hand side)
+  # @param rhs [Hash] merged data source (right-hand side)
+  # @return [Hash] +lhs+
+  # @raise [MergeError] if any value of +rhs+ cannot be merged into +lhs+
+  # @see deep_merge
+  def self.deep_merge_hashes(lhs, rhs)
+    rhs.each do |key,rhs_value|
+      lhs_value = lhs[key]
+      assert_hash_properties_are_mergeable key, lhs_value, rhs_value
+
+      if MERGEABLE_CLASSES.include? lhs_value.class
+        deep_merge lhs_value, rhs_value
+      else
+        lhs[key] = rhs_value
+      end
+    end
+  end
+
   # Raised by +join_data+ if an error is encountered.
   # @see join_data
   class JoinError < ::Exception
@@ -133,9 +202,12 @@ module HashJoiner
   # Asserts that +h+ is a hash containing +key+. Used to ensure that a +Hash+
   # can be joined with another +Hash+ object.
   #
+  # @param h [Hash] object to verify
+  # @param key [String] name of the property to verify
+  # @param error_prefix [String] prefix for error message
   # @raise [JoinError] if +h+ is not a +Hash+, or if +key_field+ is absent
   #   from any element of +lhs+ or +rhs+.
-  # @return [NilClass] +nil+
+  # @return [nil]
   # @see join_data
   # @see join_array_data
   def self.assert_is_hash_with_key(h, key, error_prefix)

metadata

@@ -1,29 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: hash-joiner
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Mike Bland
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-19 00:00:00.000000000 Z
+date: 2015-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: safe_yaml
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -56,11 +84,13 @@ description: Performs pruning or one-level promotion of Hash attributes (typical
   labeled "private:"), and deep merges and joins of Hash objects. Works on Array objects
   containing Hash objects as well.
 email: michael.bland@gsa.gov
-executables: []
+executables:
+- filter-yaml-files
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
+- bin/filter-yaml-files
 - lib/hash-joiner.rb
 homepage: https://github.com/18F/hash-joiner
 licenses:
@@ -87,3 +117,4 @@ signing_key:
 specification_version: 4
 summary: Module for pruning, promoting, deep-merging, and joining Hash data
 test_files: []
+has_rdoc: