treet 0.8.2

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.json

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in treet.gemspec
+gemspec
+
+gem 'thor'

data/Guardfile

@@ -0,0 +1,9 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/treet/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch(%r{^(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jason May
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,90 @@
+# Treet
+
+Comparisons and transformation between trees of files and JSON blobs
+
+The "JSON blobs" that are supported are not unlimited in structure, but must define:
+
+* hashes, where are the values are either {hashes where the values are all scalars} or {arrays of hashes where the values are all scalars}
+* or arrays of hashes as described above.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'treet'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install treet
+
+## Usage - Command Line
+
+    treet expand [path] [jsonfile]
+    treet explode [jsonfile] [rootdir]
+    treet import [rootdir] [xrefkey]
+
+## Usage - API
+
+    require 'treet'
+
+    hash = Treet::Hash.new(jsonfile)
+    repo = Treet::Repo.new(directory)
+    farm = Treet::Farm.new(rootdir, :xref => 'label')
+
+    hash = repo.to_hash
+    repo = hash.to_repo(root)
+    hash = farm.export
+
+    Treet.init(jsonfile, root) # when jsonfile contains an array which is exploded to multiple files
+
+## Concepts
+
+A *repo* is a directory that contains other files & directories. Any text files in this tree structure must contain JSON-formatted data.
+
+A *farm* is a directory containing one or more repos. When a farm is exported to JSON, each record is augmented with an xref value that contains the root filename of that repo.
+
+For example:
+
+    farm = Treet::Farm.new(rootdir, :xref => 'keycode')
+    puts farm.export
+
+should produce something like:
+
+    {
+      "subdir1": {
+        "field": "value"
+      },
+      "subdir2": {
+        "field": "value",
+        "field2": "value2"
+      },
+      "xref": {
+        "keycode": "repo-dir-name"
+      }
+    }
+
+## Structures
+
+All the nodes at the top level are mapped to subdirectories.
+
+At the second level, arrays elements are converted to individual subdirectories. Subdirectories are
+named with unique digest values computed from the data contents. This means that duplicate entries
+are not allowed.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## TODO
+
+* Enforce limitation on structure depth (top-level elements can contain flat hashes or arrays, nothing else)
+* refac: move diff stuff from hash.rb to Treet::Diff class, to encapsulate the structure of a diff (array of arrays); create methods for hunting for special stuff in a diff
+* Check all exceptions for explicit classes

data/Rakefile

@@ -0,0 +1,8 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/treet

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+
+require 'treet'
+require "thor"
+
+class TreetCommand < Thor
+  desc "export ROOTDIR", "convert a collection of repositories to a single JSON blob"
+  method_option :xref, :desc => "optional fieldname to be added under `xref` with basename of repository"
+  def export(path)
+    # if !File.directory?(path)
+    #   raise "treet export: could not find #{path}"
+    # end
+
+    farm = Treet::Farm.new(:root => path, :xref => options[:xref])
+    jj farm.export
+  end
+
+
+  desc "create JSONFILE", "build a single repository from a JSON file"
+  method_option :root, :required => true
+  def create(jsonfile)
+    hash = Treet::Hash.new(jsonfile)
+    hash.to_repo(options[:root])
+    $stderr.puts "Wrote repository to #{options[:root]}"
+  end
+
+
+  desc "show DIRECTORY", "convert a single repository to a JSON blob"
+  def show(path)
+    repo = Treet::Repo.new(path)
+    jj repo.to_hash
+  end
+
+
+  desc "explode JSONFILE", "Build a collection of repositories from a JSON file"
+  method_option :root, :required => true, :desc => "where to create repositories (will be created if does not exist)"
+  def explode(jsonfile)
+    if !File.directory?(options[:root])
+      Dir.mkdir(options[:root])
+    end
+
+    farm = Treet::Farm.plant(:json => jsonfile, :root => options[:root])
+    filecount = Dir.glob("#{farm.root}/*").count
+    $stderr.puts "Wrote #{filecount} files to #{farm.root}"
+  end
+
+
+  desc "patch JSONFILE", "Apply patches from file to specified repository tree"
+  method_option :root, :required => true, :desc => "where to find repositories to patch"
+  method_option :xref, :required => true, :desc => "fieldname under `xref` to use for repository identification"
+  def patch(patchfile)
+    patches = JSON.load(File.open(patchfile))
+    farm = Treet::Farm.new(:root => options[:root], :xref => options[:xref])
+    results = farm.patch(patches)
+    $stderr.puts "Patched #{results.count} records."
+  end
+
+  desc "version", "show Treet version"
+  def version
+    puts "Treet #{Treet::VERSION}"
+  end
+end
+
+TreetCommand.start

data/lib/treet/farm.rb

@@ -0,0 +1,70 @@
+# encoding: UTF-8
+
+require "uuidtools"
+
+class Treet::Farm
+  attr_reader :root, :xrefkey
+
+  def initialize(opts)
+    raise Errno::ENOENT unless File.directory?(opts[:root])
+
+    @root = opts[:root]
+    @xrefkey = opts[:xref]
+  end
+
+  def repos
+    @repos_cache ||= Dir.glob("#{root}/*").each_with_object({}) do |subdir,h|
+      # in a Farm we are looking for repositories under the root
+      if File.directory?(subdir)
+        xref = File.basename(subdir)
+        h[xref] = Treet::Repo.new(subdir, :xrefkey => xrefkey, :xref => xref)
+      end
+    end
+  end
+
+  def reset
+    @repos_cache = nil
+  end
+
+  # export as an array, not as a hash
+  # the xref for each repo will be included under `xref.{xrefkey}`
+  def export
+    repos.map {|xref,repo| repo.to_hash}
+  end
+
+  # "plant" a new farm: given an array of hashes (in JSON), create a directory
+  # of Treet repositories, one per hash. Generate directory names for each repo.
+  def self.plant(opts)
+    jsonfile = opts[:json]
+    rootdir = opts[:root]
+
+    array_of_hashes = JSON.load(File.open(jsonfile))
+    Dir.chdir(rootdir) do
+      array_of_hashes.each do |h|
+        uuid = UUIDTools::UUID.random_create.to_s
+        thash = Treet::Hash.new(h)
+        thash.to_repo(uuid)
+      end
+    end
+
+    Treet::Farm.new(:root => rootdir, :xref => opts[:xref])
+  end
+
+  # apply patches to a farm of repos
+  def patch(patches)
+    patches.map do |k,diffs|
+      repos[k].patch(diffs)
+    end
+  end
+
+  def [](xref)
+    repos[xref]
+  end
+
+  # add a new repo, with data from an input hash
+  def add(hash)
+    uuid = UUIDTools::UUID.random_create.to_s
+    thash = Treet::Hash.new(hash)
+    thash.to_repo("#{root}/#{uuid}")
+  end
+end

data/lib/treet/hash.rb

@@ -0,0 +1,219 @@
+# encoding: UTF-8
+
+require "json"
+require "digest/sha1"
+
+class Treet::Hash
+  attr_reader :data
+
+  # when loading an Array (at the top level), members are always sorted
+  # so that array comparisons will be order-independent
+  def initialize(source)
+    d = case source
+    when Hash
+      source
+    when String
+      # treat as filename
+      JSON.load(File.read(source))
+    else
+      raise "Invalid source data type #{source.class} for Treet::Hash"
+    end
+
+    @data = normalize(d)
+  end
+
+  def to_repo(root)
+    construct(data, root)
+    Treet::Repo.new(root)
+  end
+
+  def to_hash
+    data.to_hash
+  end
+
+  def compare(target)
+    # HashDiff.diff(data, target.to_hash)
+    Treet::Hash.diff(data.to_hash, target.to_hash)
+  end
+
+  # apply diffs (created via the `#compare` function) to create a new object
+  def patch(diffs)
+    newhash = Treet::Hash.patch(self.to_hash, diffs)
+    Treet::Hash.new(newhash)
+  end
+
+  def self.digestify(hash)
+    Digest::SHA1.hexdigest(hash.to_a.sort.flatten.join)
+  end
+
+  private
+
+  def construct(data, filename)
+    unless filename == '.'
+      # create the root of the repository tree
+      Dir.mkdir(filename) rescue nil
+    end
+
+    Dir.chdir(filename) do
+      data.each do |k,v|
+        case v
+        when Hash
+          File.open(k, "w") {|f| f << JSON.pretty_generate(v)}
+
+        when Array
+          Dir.mkdir(k.to_s)
+          v.each do |v2|
+            case v2
+            when String
+              # create empty file with this name
+              File.open("#{k}/#{v2}", "w")
+
+            else
+              # store object contents as JSON into a generated filename
+              subfile = "#{k}/#{Treet::Hash.digestify(v2)}"
+              File.open(subfile, "w") {|f| f << JSON.pretty_generate(v2)}
+            end
+          end
+
+        when String
+          File.open(k.to_s, "w") {|f| f << v}
+
+        else
+          raise "Unsupported object type #{v.class} for '#{k}'"
+        end
+      end
+    end
+  end
+
+  def normalize(hash)
+    hash.each_with_object({}) do |(k,v),h|
+      case v
+      when Array
+        if v.map(&:class).uniq == Hash
+          # all elements are Hashes
+          h[k] = v.sort do |a,b|
+            a.to_a.sort_by(&:first).flatten <=> b.to_a.sort_by(&:first).flatten
+          end
+        else
+          h[k] =v
+        end
+
+      else
+        h[k] = v
+      end
+    end
+  end
+
+  # Diffs need to be idempotent when applied via patch.
+  # Therefore we can't specify individual index positions for an array, because items can move.
+  # Instead, we must include the entire contents of the sub-hash, and during the patch process
+  # compare that against each element in the array.
+  # This means that an array cannot have exact duplicate entries.
+  def self.diff(hash1, hash2)
+    diffs = []
+
+    keys = hash1.keys | hash2.keys
+    keys.each do |k|
+      # if a value is missing from hash1, create a dummy of the same type that appears in hash2
+      v1 = hash1[k] || hash2[k].class.new
+      v2 = hash2[k] || hash1[k].class.new
+
+      case v1
+      when Hash
+        (v2.keys - v1.keys).each do |k2|
+          # new sub-elements: (-, key, after-value)
+          diffs << ['+', "#{k}.#{k2}", v2[k2]]
+        end
+        (v1.keys - v2.keys).each do |k2|
+          # deleted sub-elements: (-, key, before-value)
+          diffs << ['-', "#{k}.#{k2}", v1[k2]]
+        end
+        (v1.keys & v2.keys).each do |k2|
+          if v1[k2] != v2[k2]
+            # altered sub-elements: (~, key, after-value, before-value-for-reference)
+            diffs << ['~', "#{k}.#{k2}", v2[k2], v1[k2]]
+          end
+        end
+
+      when Array
+        v1.each do |e1|
+          if !v2.include?(e1)
+            # element has been removed
+            diffs << ['-', "#{k}[]", e1]
+          end
+        end
+
+        (v2 - v1).each do |e2|
+          # new array element
+          diffs << ['+', "#{k}[]", e2]
+        end
+
+      else # scalar values
+        if v1 != v2
+          if v1.nil?
+            diffs << ['+', k, v2]
+          elsif v2.nil?
+            diffs << ['-', k, v1]
+          else
+            diffs << ['~', k, v2, v1]
+          end
+        end
+
+      end
+    end
+
+    diffs
+  end
+
+  def self.patch(hash, diffs)
+    result = hash.dup
+
+    diffs.each do |diff|
+      flag, key, v1, v2 = diff
+      if key =~ /\[/
+        keyname, is_array = key.match(/^(.*)(\[\])$/).captures
+      elsif key =~ /\./
+        keyname, subkey = key.match(/^(.*)\.(.*)$/).captures
+      else
+        keyname = key
+      end
+
+      case flag
+      when '~'
+        # change a value in place
+
+        if subkey
+          result[keyname][subkey] = v1
+        else
+          result[keyname] = v1
+        end
+
+      when '+'
+        # add something
+        if subkey
+          result[keyname] ||= {}
+          result[keyname][subkey] = v1
+        elsif is_array
+          result[keyname] ||= []
+          result[keyname] << v1
+        else
+          result[keyname] = v1
+        end
+
+      when '-'
+        # remove something
+        if subkey
+          result[keyname].delete(subkey)
+        elsif is_array
+          result[keyname].delete_if {|v| v == v1}
+        else
+          result.delete(keyname)
+        end
+      end
+    end
+
+    result.delete_if {|k,v| v.nil? || v.empty?}
+
+    result
+  end
+end

data/lib/treet/repo.rb

@@ -0,0 +1,141 @@
+# encoding: UTF-8
+
+# require 'hashdiff'
+
+class Treet::Repo
+  attr_reader :root, :hash, :opts
+
+  def initialize(path, opts = {})
+    # TODO: validate that path exists and is a directory (symlinks should work)
+
+    @root = path
+    raise "Missing or invalid source path #{path}" unless File.directory?(path)
+    @opts = opts
+  end
+
+  def to_hash
+    @hash ||= expand(root)
+  end
+
+  def compare(target)
+    Treet::Hash.diff(to_hash, target.to_hash)
+    # HashDiff.diff(to_hash, hash)
+  end
+
+  # patch keys can look like
+  #   name.first
+  #   emails[]
+  # (address[1] syntax has been eliminated, we recognize array elements by matching the entire content)
+  def self.filefor(keyname)
+    if keyname =~ /\[/
+      keyname, is_array, index = keyname.match(/^(.*)(\[\])$/).captures
+      [keyname, '', nil]
+    elsif keyname =~ /\./
+      # subelement
+      filename,field = keyname.split('.')
+      ['.', filename, field]
+    else
+      [nil, keyname]
+    end
+  end
+
+  # Patching a repo is not the same as patching a hash. Make the changes
+  # directly to the data files.
+  def patch(diffs)
+    @hash = nil # invalidate any cached image
+
+    Dir.chdir(root) do
+      diffs.each do |diff|
+        flag, key, v1, v2 = diff
+        if key =~ /\[/
+          keyname, is_array = key.match(/^(.*)(\[\])$/).captures
+        elsif key =~ /\./
+          keyname, subkey = key.match(/^(.*)\.(.*)$/).captures
+        else
+          keyname = key
+        end
+
+        dirname, filename, fieldname = Treet::Repo.filefor(key)
+        filepath = "#{dirname}/#{filename}"
+        case flag
+        when '~'
+          # change a value in place
+          # load the current data & overwrite with the new value
+          # idempotent: this will overwrite the file with the same contents
+          data = File.exists?(filepath) ? JSON.load(File.open(filepath)) : {}
+          data[fieldname] = v1
+          File.open(filepath, "w") {|f| f << JSON.pretty_generate(data)}
+
+        when '+'
+          # add something
+          if fieldname
+            # writing a value into a hash
+            # idempotent: this will overwrite the file with the same contents
+            data = File.exists?(filepath) ? JSON.load(File.open(filepath)) : {}
+            data[fieldname] = v1
+            Dir.mkdir(dirname) unless Dir.exists?(dirname)
+            File.open(filepath, "w") {|f| f << JSON.pretty_generate(data)}
+          else
+            # writing an entire hash into an array entry
+            # idempotent: this will overwrite the file with the same contents
+            subfile = "#{dirname}/#{Treet::Hash.digestify(v1)}"
+            Dir.mkdir(dirname) unless Dir.exists?(dirname)
+            File.open(subfile, "w") {|f| f << JSON.pretty_generate(v1)}
+          end
+
+        when '-'
+          # remove something
+          if fieldname
+            data = JSON.load(File.open(filepath))
+            data.delete(fieldname)
+            if data.empty?
+              File.delete(filename)
+            else
+              File.open(filepath, "w") {|f| f << JSON.pretty_generate(data)}
+            end
+          else
+            # this is an array, we look for a match on the entire contents via digest
+            subfile = "#{dirname}/#{Treet::Hash.digestify(v1)}"
+            File.delete(subfile) if File.exists?(subfile) # need the existence check for idempotence
+            # TODO: if dirname is now empty, should it be removed? is that worthwhile?
+          end
+        end
+      end
+    end
+
+    to_hash # ?? return the patched data? or no return value? true/false for success?
+  end
+
+  private
+
+  def expand_json(path)
+    if File.file?(path)
+      if File.zero?(path)
+        # empty files are just keys or string elements in an array
+        File.basename(path)
+      else
+        # if file contents is JSON, then parse it
+        # otherwise treat it as a raw string value
+        s = File.read(path)
+        JSON.load(s) rescue s
+      end
+    else
+      # should be a subdirectory containing files named with numbers, each containing JSON
+      files = Dir.entries(path).select {|f|  f !~ /^\./}
+      files.sort_by(&:to_i).each_with_object([]) do |f, ary|
+        ary << expand_json("#{path}/#{f}")
+      end
+    end
+  end
+
+  def expand(path)
+    files = Dir.entries(path).select {|f|  f !~ /^\./}
+    hash = files.each_with_object({}) {|f,h| h[f] = expand_json("#{path}/#{f}")}
+
+    if opts[:xrefkey]
+      hash['xref'] ||= {}
+      hash['xref'][opts[:xrefkey]] = opts[:xref]
+    end
+    hash
+  end
+end

data/lib/treet/version.rb

@@ -0,0 +1,3 @@
+module Treet
+  VERSION = "0.8.2"
+end

data/lib/treet.rb

@@ -0,0 +1,13 @@
+unless Kernel.respond_to?(:require_relative)
+  module Kernel
+    def require_relative(path)
+      require File.join(File.dirname(caller[0]), path.to_str)
+    end
+  end
+end
+
+require_relative "treet/version"
+
+require_relative "treet/repo"
+require_relative "treet/hash"
+require_relative "treet/farm"

data/spec/json/bob1.json

@@ -0,0 +1,24 @@
+{
+  "name": {
+    "first": "Bob",
+    "last": "Smith",
+    "full": "Bob Smith"
+  },
+  "emails": [
+    {
+      "label": "home",
+      "email": "bob@home.com"
+    },
+    {
+      "label": "work",
+      "email": "bob@work.com"
+    },
+    {
+      "label": "other",
+      "email": "bob@vacation.com"
+    }
+  ],
+  "other": {
+    "notes": "some commentary"
+  }
+}