persistent_blocks 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YWVjNDQ5ZGQ1Nzk0OTc2N2UyNjg5ODQwMTM5ZjJjZTUwMjhjOTY4ZA==
+  data.tar.gz: !binary |-
+    ODg5MTcxYzNmN2U3YTZiZTY0MzJiOThkN2NjOGQyNDljMDM0OTBkYg==
+SHA512:
+  metadata.gz: !binary |-
+    MGMzY2QxMDU5ODA5M2UxMDhhMWZmNjhiNDQyZTY5OTViZGM5MmJmMWEyMTY0
+    ODM4YmMzM2RmMjdmYWE2YWNjZDYxYjEyZDE1MzI2ZDhlZjhkZDEzY2U2NDhj
+    M2MyMjdkM2QzZjdkMDI4NjYzMWM3YzRjNDJjZDE2NjQ2OTMxZTI=
+  data.tar.gz: !binary |-
+    YmZjOWUxYWExMzZhZjhkOGM3NjFjMDUwODhiODY2NjI3MmI1OTM3ZjBhNjdh
+    OWRiYzY0NmE3ODA3MzY3NmRmNzk2MmUyZGJiNWYyOGMzNWIzOWQyNzgyZmIw
+    MTQ5Y2E2YzA0MDBlNmEwNTJjM2E3MmI3OGZjZWEzNzI0NmNmZTY=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Morris Feldman
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,49 @@
+# PersistentBlocks
+
+Persist the output of ruby blocks
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'persistent_blocks'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install persistent_blocks
+
+## Usage
+## put the following in a `rakefile.rb`
+
+```ruby
+require 'persistent_blocks'
+extend PersistentBlocks
+		
+persist :some_persistent_data do
+	# Do a long running process to generate some data called :some_persistent_data
+	# You don't want to repeat the calculation unecessarily, so persist will
+	# automatically persist it to disk for you.
+	sleep(1)
+	puts "I will return an array that will be automatically persisted to disk"
+	puts "Next time you run the rakefile I won't be run."
+	["First persisted element", "Second persisted element"]
+end
+    
+persist do |some_persistent_data|
+	puts "Now I will print out the persistent data"
+	p some_persistent_data # => ["First persisted element", "Second persisted element"]
+end
+```
+
+# Then run the rake file using `rake` from the command line.  The
+# first time you run it the block generating some_persistent_data will
+# be run, but the second time you run it :some_persistent_data will
+# just be retrieved from disk.  The name "some_persistent_data"
+# between the pipes in the second block is important because it tells
+# persist which data to lookup from the marshal files and pass to the
+# block of code.
+

data/lib/persistent_blocks.rb

@@ -0,0 +1,142 @@
+require 'rake/clean'
+require 'debugger'
+
+module PersistentBlocks
+  extend self
+
+  def persist(*args, &block)
+    #puts "#{__FILE__}"
+    output_syms, specified_output_files, options = PBsubs.parse_inputs(args)
+    marshal_output_files = PBsubs.sym_ar_to_file_list(output_syms)
+    marshal_dir = PBsubs.ensure_marshal_dir
+    file marshal_output_files[0] => marshal_dir unless marshal_output_files[0].nil?
+    all_output_files = marshal_output_files + specified_output_files
+    target_file = all_output_files[0] # rake tasks only accept one file
+    params = if options[:input_overide]
+               [*options[:input_overide]]
+             else
+               block.parameters.map{|p| p[1]}
+             end
+    prereqs = PBsubs.sym_ar_to_file_list(params)
+    file target_file => prereqs unless prereqs.nil?
+    file target_file do |f|
+      puts "Persistent_blocks: Persisting #{output_syms} and/or #{specified_output_files} from #{[*params]}"
+      raw_inputs = PBsubs.load_inputs(prereqs)
+      inputs = PBsubs.check_for_single_input(raw_inputs, prereqs.count)
+      raw_outputs = block.(inputs)
+      outputs = PBsubs.ensure_array(raw_outputs, output_syms.count)
+      unless output_syms.empty?
+        PBsubs.check_outputs(outputs, output_syms)
+        PBsubs.display_output_info(outputs, output_syms)
+        PBsubs.save_outputs(outputs, marshal_output_files) 
+      end
+      unless specified_output_files.empty?
+        PBsubs.check_specified_output_files(specified_output_files)
+      end
+    end
+    options[:task] ||= PBsubs.default_persistent_blocks_task
+    task options[:task].to_sym => target_file
+    all_output_files.each do |out_file|
+      delete_task = "delete_#{out_file}"
+      task delete_task do
+        rm out_file if File.exists? out_file
+      end
+      task "delete_#{options[:task].to_sym}" => delete_task
+      file out_file => target_file unless out_file == target_file
+    end
+    specified_output_files.each do |out_file|
+      CLOBBER.include(out_file)
+    end
+    return target_file # allows file target_file => extra_dependency
+  end
+end
+
+module PBsubs
+  extend Rake::DSL
+  extend self
+  
+  attr_accessor :marshal_dir, :default_persistent_blocks_task
+  @marshal_dir ||= 'marshal_dir'
+  @default_persistent_blocks_task ||= :default
+
+  def ensure_array(raw, n_expected)
+    (n_expected == 1) ? [raw] : raw
+  end
+
+  def check_for_single_input(raw, n_expected)
+    (n_expected == 1) ? raw[0] : raw
+  end
+  
+  def sym_to_filename(sym)
+    File.join(@marshal_dir, sym.to_s)
+  end
+
+  def sym_ar_to_file_list(sym_ar)
+    sym_ar.map{|sym| sym_to_filename(sym)}
+  end
+  
+  def marshal_save(object, filename)
+    File.open(filename, 'w') {|io| Marshal.dump(object, io)}
+  end
+
+  def marshal_load(filename)
+    File.open(filename, 'r') {|io| Marshal.load(io)}
+  end
+
+  def load_inputs(filenames)
+    return nil if filenames.nil? || filenames.empty?
+    filenames.map{|f| marshal_load(f)}
+  end
+
+  def save_outputs(objects_to_save, filenames)
+    return if objects_to_save.nil?
+    return if filenames.nil? || filenames.empty?
+    objects_to_save.zip(filenames).each {|o,f| marshal_save(o, f)}
+  end
+
+  def check_outputs(outputs, output_syms)
+    n_expect = output_syms.count
+    case n_expect
+    when 0 # NP, the block might return an argument, but we don't need to save it
+    when 1
+      raise "Error: expecting an output (#{outputs}) from the block but got none" if outputs.nil?
+      # if we get an array of outputs they will just be mapped to a single marshal file
+    else
+      raise "Error: expecting #{n_expect} outputs (#{output_sym}) from block but got none" if outputs.nil?
+      n_received = [*outputs].count
+      raise "Error: expecting #{n_expect} outputs (#{output_syms}) from block but got #{n_received} instead" if n_expect != n_received
+    end
+  end
+
+  def display_output_info(outputs, output_syms)
+    output_ar = output_syms.zip(outputs).map do |sym, output|
+      how_long = (output.respond_to?:length) ? output.length : nil
+      "#{sym} (length = #{how_long})"
+    end
+    puts "Block Outputs:"
+    puts output_ar
+  end
+  
+  def ensure_marshal_dir
+    CLOBBER.include(@marshal_dir) unless CLOBBER.include?(@marshal_dir)
+    directory @marshal_dir
+    @marshal_dir
+  end
+
+  def parse_inputs(input_args)
+    args = input_args.dup
+    options = (args[-1].is_a? Hash) ? args.pop : {}
+    syms = args.select{|out| out.is_a? Symbol}
+    files = args.select{|out| out.is_a? String}
+    return syms, files, options
+  end
+
+  def check_specified_output_files(specified_files)
+    specified_files.each do |file|
+      unless File.exists? file
+        raise "Error: #{file} was not created"
+      end
+    end
+  end
+  
+end

data/persistent_blocks.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |gem|
+  gem.name          = "persistent_blocks"
+  gem.version       = "0.1.0"
+  gem.author        = "Morris Feldman"
+  gem.email         = "morrifeldman@gmail.com"
+  gem.summary       = "Persists the output of Ruby blocks"
+  gem.homepage      = "http://github.com/morrifeldman/persistent_blocks"
+
+  gem.files         = `git ls-files`.split($/)
+  
+  gem.add_runtime_dependency 'rake'
+
+  gem.description       = <<EODESC
+  This gem provides a rake extension to wrap blocks of ruby code so
+  that the output of the block is persisted using marshal.  Blocks
+  with persisted data will not be rerun and their data is available to
+  subsequent blocks which can themselves generate persistent data.
+  This allow a very simple and robust pipeline to be constructed in
+  within a regular rakefile.   
+EODESC
+  
+end

data/rakefile.rb

@@ -0,0 +1,10 @@
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end
+
+task :default => 'test'
+

data/test/persistent_blocks_test.rb

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+require 'minitest/autorun'
+require 'persistent_blocks'
+
+Test_rakefile = 'test_rakefile.rb'
+Test_dir = 'test'
+def run_rakefile(arg = '')
+  `rake -I../lib -f #{Test_rakefile} #{arg}` #../lib require to locate
+  #persistent_blocks because we are shelling out
+end
+
+describe 'persistent_blocks' do
+  before do
+    Dir.chdir(Test_dir) do
+      puts run_rakefile('clobber')
+      
+      puts "\nRunning tests for the first time:\n"
+      puts run_rakefile
+    
+      puts "\nRunning tests for the second time:\n"
+      puts @second_run = run_rakefile
+
+      puts run_rakefile('delete_marshal_dir/test3')
+      puts run_rakefile('delete_marshal_dir/test4')
+      # this last line should not raise an error
+      puts run_rakefile('delete_marshal_dir/test4')
+      # asking for another delete shouldn't cause a problem either
+      
+      # clean up
+      puts run_rakefile('clobber')
+    end
+  end
+  it 'should persist data' do
+    @second_run.must_equal ''
+  end
+end
+
+describe 'specify task when calling perist' do
+  before do
+    Dir.chdir(Test_dir) do
+      puts run_rakefile('clobber')
+      run_rakefile('with_set_task')
+      @test_task_persisted = PBsubs.marshal_load(PBsubs.sym_to_filename(:set_task))
+      run_rakefile('delete_with_set_task')
+      @set_task_exists = File.exists?(File.join(Test_dir, 'marshal_dir','set_task'))
+      puts run_rakefile('clobber')
+    end
+  end
+  it 'should run the test_task' do
+    @test_task_persisted.must_equal 'My task was set with the :task option'
+  end
+  it 'the delete_task should delete the specific task' do
+    @set_task_exists.must_equal false
+  end
+end
+
+describe 'setting default_peristent_blocks_task' do
+  before do
+    Dir.chdir(Test_dir) do
+      puts run_rakefile('clobber')
+      run_rakefile('non_default_task')
+      @non_default_peristed = PBsubs.marshal_load(PBsubs.sym_to_filename(:not_a_default_task))
+      run_rakefile('delete_non_default_task')
+      @non_default_task_exists = File.exists?(File.join(Test_dir, 'marshal_dir', 'not_a_default_task'))
+      run_rakefile('clobber')
+    end
+  end
+  it 'should run the non-default task' do
+    @non_default_peristed.must_equal 'I am not a default task, I am a non_default_task'
+  end
+  it 'should run the delete_task' do
+    @non_default_task_exists.must_equal false
+  end
+end
+
+

data/test/test_rakefile.rb

@@ -0,0 +1,65 @@
+require "persistent_blocks"
+require 'debugger'
+
+extend PersistentBlocks
+
+persist :test do
+  puts "running the first task"
+  puts a = [1,2,3]
+  a
+end
+
+persist :test2 do |test|
+  puts "running the second task"
+  puts "input = #{test}"
+  'I do not want to be packed into an array'
+end
+
+persist :test3, :test4  do |test2|
+  puts "About to simulate a 2 sec calc"
+  sleep(2)
+  puts "test2 = #{test2}"
+  puts "test2 = '#{test2}', it should not be an array"
+  [test2*1, test2.upcase]
+end
+
+persist :task4 do |test3, test4|
+  puts "running the 4th task"
+  puts "test3 = #{test3}"
+  puts "test4 = #{test4}"
+  ['String Output\nSecond Line', {test: 1, :test => [1, 2, 3]}]
+end
+
+persist :no_paren do
+  'no parentheses'
+end
+
+persist :overide_test, input_overide: [:test3, :test4] do |x,y|
+  puts "test3 (#{x}) was mapped to x"
+  puts "test4 (#{y})) was mapped to y"
+  1
+end
+
+persist :overide_test_no_bracket, :input_overide => :test3 do |x|
+  puts "input overide also works without brackets"
+end
+
+persist :set_task, :task => :with_set_task do
+  'My task was set with the :task option'
+end
+
+PBsubs.default_persistent_blocks_task = :non_default_task
+persist :not_a_default_task do
+  'I am not a default task, I am a non_default_task'
+end
+
+#persist(:should_fail1, :should_fail_2) do
+#  'one_output, but expecting two'
+#end
+
+#persist (:should_fail) do
+#  puts "This task should fail because it doesn't return anything"
+#end
+
+
+

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: persistent_blocks
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Morris Feldman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "  This gem provides a rake extension to wrap blocks of ruby code so\n
+  \ that the output of the block is persisted using marshal.  Blocks\n  with persisted
+  data will not be rerun and their data is available to\n  subsequent blocks which
+  can themselves generate persistent data.\n  This allow a very simple and robust
+  pipeline to be constructed in\n  within a regular rakefile.   \n"
+email: morrifeldman@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- LICENSE.txt
+- README.md
+- lib/persistent_blocks.rb
+- persistent_blocks.gemspec
+- rakefile.rb
+- test/persistent_blocks_test.rb
+- test/test_rakefile.rb
+homepage: http://github.com/morrifeldman/persistent_blocks
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.3
+signing_key: 
+specification_version: 4
+summary: Persists the output of Ruby blocks
+test_files: []