hash-joiner 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +15 -1
- data/bin/filter-yaml-files +159 -0
- data/lib/hash-joiner.rb +108 -36
- metadata +36 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 759dc6bcf51ede5903167adf68613554adcec044
|
4
|
+
data.tar.gz: 3fd0a5e98271be1025d65b5c9610f7d73b127c63
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 429156c991502011b29fc54d6aff1b1495b0eeb80f19b55b44394f3bdfd65be180b31407b74ded53a0005c16aad947a72edbfb711066b1e173c7822dc61694fe
|
7
|
+
data.tar.gz: 2658c4cebb926998d05c3add6830c6f55c65324ffede7021e40ddadad4f3df57c4e4efbbd5c8b92f887e977e011db5367e2c73393edb00120cd01f43a6627923
|
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
## hash-joiner Gem
|
2
2
|
|
3
3
|
[](http://badge.fury.io/rb/hash-joiner)
|
4
|
-
[](https://travis-ci.org/18F/hash-joiner)
|
5
5
|
[](https://codeclimate.com/github/18F/hash-joiner)
|
6
6
|
[](https://codeclimate.com/github/18F/hash-joiner)
|
7
7
|
|
@@ -140,6 +140,20 @@ This corresponds to the process of joining different collections of Jekyll-impor
|
|
140
140
|
{"name"=>"bazquux", "email"=>"baz.quux@gsa.gov"}]}
|
141
141
|
```
|
142
142
|
|
143
|
+
#### Running `filter-yaml-files`
|
144
|
+
|
145
|
+
The `filter-yaml-files` program can be used to generate "public" versions of YAML files containing "private" data. For example:
|
146
|
+
|
147
|
+
```
|
148
|
+
$ export HUB_DATA_DIR=../hub/_data
|
149
|
+
|
150
|
+
$ filter-yaml-files ${HUB_DATA_DIR}/private/{team,projects}.yml -o ${HUB_DATA_DIR}/public
|
151
|
+
../hub/_data/private/team.yml => ../hub/_data/public/team.yml
|
152
|
+
../hub/_data/private/projects.yml => ../hub/_data/public/projects.yml
|
153
|
+
```
|
154
|
+
|
155
|
+
The `filter-yaml-files` program can also strip other properties besides `private:`, and can promote data contained within a property rather than strip it. Run `filter-yaml-files -h` to see the options that allow this.
|
156
|
+
|
143
157
|
### Contributing
|
144
158
|
|
145
159
|
Just fork [18F/hash-joiner](https://github.com/18F/hash-joiner) and start sending pull requests! Feel free to ping [@mbland](https://github.com/mbland) with any questions you may have, especially if the current documentation should've addressed your needs, but didn't.
|
@@ -0,0 +1,159 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
# hash-joiner - Pruning, promoting, deep-merging, and joining Hash data
|
3
|
+
#
|
4
|
+
# Written in 2015 by Mike Bland (michael.bland@gsa.gov)
|
5
|
+
# on behalf of the 18F team, part of the US General Services Administration:
|
6
|
+
# https://18f.gsa.gov/
|
7
|
+
#
|
8
|
+
# To the extent possible under law, the author(s) have dedicated all copyright
|
9
|
+
# and related and neighboring rights to this software to the public domain
|
10
|
+
# worldwide. This software is distributed without any warranty.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the CC0 Public Domain Dedication along
|
13
|
+
# with this software. If not, see
|
14
|
+
# <https://creativecommons.org/publicdomain/zero/1.0/>.
|
15
|
+
#
|
16
|
+
# ---
|
17
|
+
#
|
18
|
+
# Script to generate 'public' versions of 'private' YAML files.
|
19
|
+
#
|
20
|
+
# The command line flags support stripping object properties other than
|
21
|
+
# `private:`, or promoting the data associated with the property rather than
|
22
|
+
# stripping it out.
|
23
|
+
#
|
24
|
+
# Author: Mike Bland (michael.bland@gsa.gov)
|
25
|
+
# Date: 2015-01-11
|
26
|
+
|
27
|
+
require 'hash-joiner'
|
28
|
+
require 'optparse'
|
29
|
+
require 'safe_yaml'
|
30
|
+
|
31
|
+
options = {
|
32
|
+
:property => 'private',
|
33
|
+
:output_dir => 'public',
|
34
|
+
}
|
35
|
+
|
36
|
+
opt_parser = OptionParser.new do |opts|
|
37
|
+
opts.banner = "Usage: #{$0} [-hop] [--promote] [file ...]"
|
38
|
+
|
39
|
+
opts.separator ''
|
40
|
+
opts.separator <<EOF
|
41
|
+
By default, reads a collection of YAML files containing "private" data, i.e.
|
42
|
+
objects containing a `private:` property, and generates new files with the
|
43
|
+
private data stripped out.
|
44
|
+
|
45
|
+
Using the option flags, other properties besides `private:` can be removed, or
|
46
|
+
the data associated with the target PROPERTY can be promoted rather than
|
47
|
+
stripped.
|
48
|
+
|
49
|
+
All input files should come from the same parent directory. This is because
|
50
|
+
the directory structure beneath the parent directory will be preserved in the
|
51
|
+
OUTPUT_DIR.
|
52
|
+
|
53
|
+
Options:
|
54
|
+
EOF
|
55
|
+
|
56
|
+
opts.on('-h', '--help', "Show this help") do
|
57
|
+
puts opts
|
58
|
+
exit
|
59
|
+
end
|
60
|
+
|
61
|
+
opts.on('-p', '--property PROPERTY',
|
62
|
+
'Property to strip/promote ' +
|
63
|
+
"(default: #{options[:property]})") do |property|
|
64
|
+
options[:property] = property
|
65
|
+
end
|
66
|
+
|
67
|
+
opts.on('-o', '--output_dir OUTPUT_DIR',
|
68
|
+
"Output directory (default: #{options[:output_dir]})") do |output_dir|
|
69
|
+
options[:output_dir] = output_dir
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.on('--promote',
|
73
|
+
'Promote the PROPERTY rather than strip it') do |promote|
|
74
|
+
options[:promote] = promote
|
75
|
+
end
|
76
|
+
end
|
77
|
+
opt_parser.parse!
|
78
|
+
|
79
|
+
if ARGV.length < 1
|
80
|
+
STDERR.puts 'No input files specified'
|
81
|
+
exit 1
|
82
|
+
end
|
83
|
+
|
84
|
+
input_files = []
|
85
|
+
errors = []
|
86
|
+
parent_dir = ''
|
87
|
+
|
88
|
+
ARGV.each do |input_file|
|
89
|
+
current_parent_dir = File.dirname input_file
|
90
|
+
parent_dir = current_parent_dir if parent_dir.empty?
|
91
|
+
unless (parent_dir.start_with? current_parent_dir or
|
92
|
+
current_parent_dir.start_with? parent_dir)
|
93
|
+
STDERR.puts 'All input files should come from the same parent directory.'
|
94
|
+
STDERR.puts "Detected: #{parent_dir} and #{current_parent_dir}"
|
95
|
+
exit 1
|
96
|
+
end
|
97
|
+
parent_dir = current_parent_dir if current_parent_dir.size < parent_dir.size
|
98
|
+
|
99
|
+
if !File.exists? input_file
|
100
|
+
errors << "File does not exist: #{input_file}"
|
101
|
+
elsif !File.readable? input_file
|
102
|
+
errors << "File not readable: #{input_file}"
|
103
|
+
else
|
104
|
+
input_files << input_file
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
unless errors.empty?
|
109
|
+
STDERR.puts errors.join "\n"
|
110
|
+
STDERR.puts "Aborting; no files processed."
|
111
|
+
exit 1
|
112
|
+
end
|
113
|
+
|
114
|
+
def recursive_mkdir(dirname, parent_dir)
|
115
|
+
parent_dir = File::SEPARATOR if dirname.start_with? File::SEPARATOR
|
116
|
+
dir_components = dirname.split(File::SEPARATOR)
|
117
|
+
current_subdir = parent_dir
|
118
|
+
until dir_components.empty?
|
119
|
+
current_subdir = File.join(current_subdir, dir_components.shift)
|
120
|
+
Dir.mkdir(current_subdir) unless Dir.exists? current_subdir
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
FILTERED_OUTPUT_DIR = options[:output_dir]
|
125
|
+
FILTERED_PROPERTY = options[:property]
|
126
|
+
FILTER_OPERATION = options[:promote] ? :promote_data : :remove_data
|
127
|
+
FILTERED_PARENT_DIR_SIZE = parent_dir.concat(File::SEPARATOR).size
|
128
|
+
|
129
|
+
recursive_mkdir FILTERED_OUTPUT_DIR, Dir.pwd
|
130
|
+
|
131
|
+
input_files.each do |input_file|
|
132
|
+
data = SafeYAML.load_file(input_file, :safe=>true)
|
133
|
+
unless data
|
134
|
+
errors << "Failed to parse #{source}"
|
135
|
+
next
|
136
|
+
end
|
137
|
+
|
138
|
+
begin
|
139
|
+
# Make the output subdirectory hierarchy if necessary.
|
140
|
+
subdirs = File.dirname(input_file)[FILTERED_PARENT_DIR_SIZE..-1] || ''
|
141
|
+
recursive_mkdir subdirs, FILTERED_OUTPUT_DIR
|
142
|
+
|
143
|
+
output_file = File.join(FILTERED_OUTPUT_DIR, subdirs,
|
144
|
+
File.basename(input_file))
|
145
|
+
open(output_file, 'w') do |outfile|
|
146
|
+
puts "#{input_file} => #{output_file}"
|
147
|
+
data = HashJoiner.send FILTER_OPERATION, data, FILTERED_PROPERTY
|
148
|
+
outfile.puts data.to_yaml
|
149
|
+
end
|
150
|
+
rescue SystemCallError => e
|
151
|
+
errors << e.to_s
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
unless errors.empty?
|
156
|
+
STDERR.puts "\n*** Errors:"
|
157
|
+
STDERR.puts errors.join "\n"
|
158
|
+
exit 1
|
159
|
+
end
|
data/lib/hash-joiner.rb
CHANGED
@@ -30,28 +30,46 @@ module HashJoiner
|
|
30
30
|
# @return [nil] if +collection+ is not a +Hash+ or +Array<Hash>+
|
31
31
|
def self.promote_data(collection, key)
|
32
32
|
if collection.instance_of? ::Hash
|
33
|
-
|
34
|
-
data_to_promote = collection[key]
|
35
|
-
collection.delete key
|
36
|
-
deep_merge collection, data_to_promote
|
37
|
-
end
|
38
|
-
collection.each_value {|i| promote_data i, key}
|
39
|
-
|
33
|
+
promote_hash_data collection, key
|
40
34
|
elsif collection.instance_of? ::Array
|
41
|
-
collection
|
42
|
-
|
43
|
-
|
44
|
-
if i.instance_of? ::Hash and i.keys == [key]
|
45
|
-
data_to_promote = i[key]
|
46
|
-
i.delete key
|
47
|
-
deep_merge collection, data_to_promote
|
48
|
-
else
|
49
|
-
promote_data i, key
|
50
|
-
end
|
51
|
-
end
|
35
|
+
promote_array_data collection, key
|
36
|
+
end
|
37
|
+
end
|
52
38
|
|
53
|
-
|
39
|
+
# Recursively promotes data within a Hash. Used to implement promote_data.
|
40
|
+
#
|
41
|
+
# @param collection [Hash] collection in which to promote information
|
42
|
+
# @param key [String] property to be promoted within +collection+
|
43
|
+
# @return [Hash] +collection+ after promotion
|
44
|
+
# @see promote_data
|
45
|
+
def self.promote_hash_data(collection, key)
|
46
|
+
if collection.member? key
|
47
|
+
data_to_promote = collection[key]
|
48
|
+
collection.delete key
|
49
|
+
deep_merge collection, data_to_promote
|
50
|
+
end
|
51
|
+
collection.each_value {|i| promote_data i, key}
|
52
|
+
end
|
53
|
+
|
54
|
+
# Recursively promotes data within an Array. Used to implement promote_data.
|
55
|
+
#
|
56
|
+
# @param collection [Array] collection in which to promote information
|
57
|
+
# @param key [String] property to be promoted within +collection+
|
58
|
+
# @return [Array] +collection+ after promotion
|
59
|
+
# @see promote_data
|
60
|
+
def self.promote_array_data(collection, key)
|
61
|
+
collection.each do |i|
|
62
|
+
# If the Array entry is a hash that contains only the target key,
|
63
|
+
# then that key should map to an Array to be promoted.
|
64
|
+
if i.instance_of? ::Hash and i.keys == [key]
|
65
|
+
data_to_promote = i[key]
|
66
|
+
i.delete key
|
67
|
+
deep_merge collection, data_to_promote
|
68
|
+
else
|
69
|
+
promote_data i, key
|
70
|
+
end
|
54
71
|
end
|
72
|
+
collection.delete_if {|i| i.empty?}
|
55
73
|
end
|
56
74
|
|
57
75
|
# Raised by +deep_merge+ if +lhs+ and +rhs+ are of different types.
|
@@ -59,6 +77,26 @@ module HashJoiner
|
|
59
77
|
class MergeError < ::Exception
|
60
78
|
end
|
61
79
|
|
80
|
+
# The set of mergeable classes
|
81
|
+
MERGEABLE_CLASSES = [::Hash, ::Array]
|
82
|
+
|
83
|
+
# Asserts that +lhs+ and +rhs+ are of the same type and can be merged.
|
84
|
+
#
|
85
|
+
# @param lhs [Hash,Array] merged data sink (left-hand side)
|
86
|
+
# @param rhs [Hash,Array] merged data source (right-hand side)
|
87
|
+
# @raise [MergeError] if +lhs+ and +rhs+ are of the different types or are
|
88
|
+
# of a type that cannot be merged
|
89
|
+
# @return [nil]
|
90
|
+
# @see deep_merge
|
91
|
+
def self.assert_objects_are_mergeable(lhs, rhs)
|
92
|
+
if lhs.class != rhs.class
|
93
|
+
raise MergeError.new("LHS (#{lhs.class}): #{lhs}\n" +
|
94
|
+
"RHS (#{rhs.class}): #{rhs}")
|
95
|
+
elsif !MERGEABLE_CLASSES.include? lhs.class
|
96
|
+
raise MergeError.new "Class not mergeable: #{lhs.class}"
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
62
100
|
# Performs a deep merge of +Hash+ and +Array+ structures. If the collections
|
63
101
|
# are +Hash+es, +Hash+ or +Array+ members of +rhs+ will be deep-merged with
|
64
102
|
# any existing members in +lhs+. If the collections are +Array+s, the values
|
@@ -70,30 +108,61 @@ module HashJoiner
|
|
70
108
|
# @raise [MergeError] if +lhs+ and +rhs+ are of different classes, or if
|
71
109
|
# they are of classes other than Hash or Array.
|
72
110
|
def self.deep_merge(lhs, rhs)
|
73
|
-
|
74
|
-
|
75
|
-
if lhs.class != rhs.class
|
76
|
-
raise MergeError.new("LHS (#{lhs.class}): #{lhs}\n" +
|
77
|
-
"RHS (#{rhs.class}): #{rhs}")
|
78
|
-
elsif !mergeable_classes.include? lhs.class
|
79
|
-
raise MergeError.new "Class not mergeable: #{lhs.class}"
|
80
|
-
end
|
111
|
+
assert_objects_are_mergeable lhs, rhs
|
81
112
|
|
82
113
|
if rhs.instance_of? ::Hash
|
83
|
-
|
84
|
-
if lhs.member? key and mergeable_classes.include? value.class
|
85
|
-
deep_merge(lhs[key], value)
|
86
|
-
else
|
87
|
-
lhs[key] = value
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
114
|
+
deep_merge_hashes lhs, rhs
|
91
115
|
elsif rhs.instance_of? ::Array
|
92
116
|
lhs.concat rhs
|
93
117
|
end
|
94
118
|
lhs
|
95
119
|
end
|
96
120
|
|
121
|
+
# Asserts that +rhs_value+ can be merged into +lhs_value+ for the property
|
122
|
+
# identified by +key+.
|
123
|
+
#
|
124
|
+
# @param key [String] Hash property name
|
125
|
+
# @param lhs_value [Object] the property value of the data sink Hash
|
126
|
+
# (left-hand side value)
|
127
|
+
# @param rhs_value [Object] the property value of the data source Hash
|
128
|
+
# (right-hand side value)
|
129
|
+
# @raise [MergeError] if +lhs_value+ exists and +rhs_value+ is of a
|
130
|
+
# different class
|
131
|
+
# @return [nil]
|
132
|
+
# @see deep_merge
|
133
|
+
# @see deep_merge_hashes
|
134
|
+
def self.assert_hash_properties_are_mergeable(key, lhs_value, rhs_value)
|
135
|
+
lhs_class = lhs_value == false ? ::TrueClass : lhs_value.class
|
136
|
+
rhs_class = rhs_value == false ? ::TrueClass : rhs_value.class
|
137
|
+
|
138
|
+
unless lhs_value.nil? or lhs_class == rhs_class
|
139
|
+
raise MergeError.new(
|
140
|
+
"LHS[#{key}] value (#{lhs_class}): #{lhs_value}\n" +
|
141
|
+
"RHS[#{key}] value (#{rhs_class}): #{rhs_value}")
|
142
|
+
end
|
143
|
+
nil
|
144
|
+
end
|
145
|
+
|
146
|
+
# Performs a deep merge of Hash structures. Used to implement +deep_merge+.
|
147
|
+
#
|
148
|
+
# @param lhs [Hash] merged data sink (left-hand side)
|
149
|
+
# @param rhs [Hash] merged data source (right-hand side)
|
150
|
+
# @return [Hash] +lhs+
|
151
|
+
# @raise [MergeError] if any value of +rhs+ cannot be merged into +lhs+
|
152
|
+
# @see deep_merge
|
153
|
+
def self.deep_merge_hashes(lhs, rhs)
|
154
|
+
rhs.each do |key,rhs_value|
|
155
|
+
lhs_value = lhs[key]
|
156
|
+
assert_hash_properties_are_mergeable key, lhs_value, rhs_value
|
157
|
+
|
158
|
+
if MERGEABLE_CLASSES.include? lhs_value.class
|
159
|
+
deep_merge lhs_value, rhs_value
|
160
|
+
else
|
161
|
+
lhs[key] = rhs_value
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
97
166
|
# Raised by +join_data+ if an error is encountered.
|
98
167
|
# @see join_data
|
99
168
|
class JoinError < ::Exception
|
@@ -133,9 +202,12 @@ module HashJoiner
|
|
133
202
|
# Asserts that +h+ is a hash containing +key+. Used to ensure that a +Hash+
|
134
203
|
# can be joined with another +Hash+ object.
|
135
204
|
#
|
205
|
+
# @param h [Hash] object to verify
|
206
|
+
# @param key [String] name of the property to verify
|
207
|
+
# @param error_prefix [String] prefix for error message
|
136
208
|
# @raise [JoinError] if +h+ is not a +Hash+, or if +key_field+ is absent
|
137
209
|
# from any element of +lhs+ or +rhs+.
|
138
|
-
# @return [
|
210
|
+
# @return [nil]
|
139
211
|
# @see join_data
|
140
212
|
# @see join_array_data
|
141
213
|
def self.assert_is_hash_with_key(h, key, error_prefix)
|
metadata
CHANGED
@@ -1,29 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hash-joiner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Bland
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-01-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: safe_yaml
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
|
-
type: :
|
20
|
+
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.7'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.7'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
27
55
|
- !ruby/object:Gem::Dependency
|
28
56
|
name: minitest
|
29
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -56,11 +84,13 @@ description: Performs pruning or one-level promotion of Hash attributes (typical
|
|
56
84
|
labeled "private:"), and deep merges and joins of Hash objects. Works on Array objects
|
57
85
|
containing Hash objects as well.
|
58
86
|
email: michael.bland@gsa.gov
|
59
|
-
executables:
|
87
|
+
executables:
|
88
|
+
- filter-yaml-files
|
60
89
|
extensions: []
|
61
90
|
extra_rdoc_files: []
|
62
91
|
files:
|
63
92
|
- README.md
|
93
|
+
- bin/filter-yaml-files
|
64
94
|
- lib/hash-joiner.rb
|
65
95
|
homepage: https://github.com/18F/hash-joiner
|
66
96
|
licenses:
|
@@ -87,3 +117,4 @@ signing_key:
|
|
87
117
|
specification_version: 4
|
88
118
|
summary: Module for pruning, promoting, deep-merging, and joining Hash data
|
89
119
|
test_files: []
|
120
|
+
has_rdoc:
|