pyrosome 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc5a6ff3c7d4e715388878be49a9bd371abc2115
-  data.tar.gz: bc5610d94c47f2bfe40bb55532dd757658eb9373
+  metadata.gz: d1384caa550967b1e266bfccc75c925e9d1738cc
+  data.tar.gz: 4645453d5168e61a62632cad8403ce875fd0e530
 SHA512:
-  metadata.gz: 196c87ac11f16c8837df5367693a6fc9d0671df65e9ffe02b67cd30b7a34fd5e27804413824665a76b3b714db92b83713f9e705a23fed0b8664ffcd6249396f2
-  data.tar.gz: 1c9cbc02a78e8bdf43d99f1334ed3335291b84b636393be0bb3e3398b9c4ec4407bd95c24a944f9924be27dab0a4a7934c4e3fd0295c72461b761ec94e97a93b
+  metadata.gz: ec3f223bf71aa5792cbab2c671cf0e0fa77ae0bde1782e380cc5f3efaf08eb32c263e8972db5bd01385af78507482bc4659c8b55139308f3dc05751672aa965e
+  data.tar.gz: d18366abae8bb47ad510c1a55c4d35ef57acf2ab170987b8a35478950607eee488602fb73b9d9c89ec848f21a922ba268003b8d91b79403da80c971131a3ad89

data/README.md

@@ -1,5 +1,9 @@
 # Pyrosome
 
+![Pyrosome Rider](pyrosome_ride.jpg)
+
+Photo via [Mr Roger Fenwick](http://www.biodiversityexplorer.org/mm/tunicates/pyrostremma_spinosum.htm)
+
 "Pyrosomes, genus Pyrosoma, are free-floating colonial tunicates that live usually in the upper layers of the open ocean in warm seas, although some may be found at greater depths. Pyrosomes are cylindrical- or conical-shaped colonies made up of hundreds to thousands of individuals, known as zooids. Colonies range in size from less than one centimeter to several metres in length."
 
 Like Pyrosomes, files are made up of many individual pieces.  The UNIX philosophy encorages operating on streams of data and Ruby embraces this in part with command line options such as `-n` and `-e` so that you can process a stream of data by specifying a Ruby script from the command line which works with a single piece of a file.  This gem aims to provide the same functionality for more complex datasets such as JSON, CSV, XLS, etc...
@@ -21,17 +25,21 @@ Install the gem:
 
 ## Usage
 
+    psome [options...] script_name
+
 The command to execute is `psome` (pronounced `p-some`, ryhmes with `roam`).  It takes a stream like this:
 
     cat file.csv | psome -i csv -e "puts _[0]"
 
-You can see that we specify the format that is expected and a bit of Ruby code which exects to use a `_` variable
+    cat file.csv | psome -i csv script.rb
+
+You can see that we specify the format that is expected and a bit of Ruby code which exects to use a `_` variable.  You can specify a Ruby script for re-usablility of code.  The `_` variable will be available in your Ruby script as well.
 
 ### Arguments
 
 #### -i [FORMAT] / --input [FORMAT]
 
-Specify a input.  `json` and `csv` currently supported
+Specify a input.  `json` and `csv` currently supported.  If this flag isn't specified then each line of the input will be given, similar to Ruby's `-n` flag
 
 #### -e [CODE] / --exec [CODE]
 
@@ -41,25 +49,35 @@ Give some Ruby code to be executed.  A `_` variable will be in scope and will re
 
 If the data is tabular, should we expect headers?
 
-## TODOs
+#### -f / --forks [FORK_COUNT]
+
+Specify that parallel processing is used via separate processes and specify how many processes to use
 
-By default process lines like Ruby
+#### -t / --threads [THREAD_COUNT]
 
-Parallel option
- * Threads or forks
- * Way to do mutex (`sync` method?)
+Specify that parallel processing is used via separate threads and specify how many threads to use
 
--i to do in place replacement of files
+### Methods
 
-Support file names as arguments.
+#### ``sync``
+
+If you are running in parallel mode you can call the `sync` method within your code to run part of it synchronously.  This is particularly useful for having multiple threads/processes coordinate to write output, even if you don't care what order they do it in.  If you specify that threads are to be used then the Ruby `Mutex` will do the synchronization, while for forks a temporary file with be created to `flock` to.
+
+**Example:**
+
+# Convert CSV to JSON:
+
+some_unix_commands | psome -f4 -i csv -e "name, age = _[0], _[1].to_i; sync { puts({name: name, age: age}.to_json + ',') }"
+
+## TODOs
 
-Support file name for code execution instead of -e?  Could files have both `_` and `ARGV` in scope so that scripts not made for pyrosome be used?
+Something like -i to do in place replacement of files?
 
 Flag to echo sample of what you would get with the specified options (do I get an array or a hash?  Does the hash have keys?)
 
 Support for colors?
 
-Support for different enumerators?  Sort_by comes to mind, but that wouldn't work for streaming (or would it?). Select/reject.  Does any?/all? make sense?
+Support for different enumerators (select/reject particularly)?  #sort_by probably wouldn't work for streaming
 
 Support for loading an app environment (automatically load Rails if inside a project to get models?)
 

data/bin/psome

@@ -1,12 +1,14 @@
 #!/usr/bin/env ruby
 
 require 'optparse'
+require 'json'
+require 'csv'
 
 VALID_FORMATS = %w(json csv)
 
 options = {}
 OptionParser.new do |opts|
-  opts.banner = "Usage: example.rb [options]"
+  opts.banner = "Usage: psome [options...] [script_name]"
 
   opts.on("-i", "--input [FORMAT]", "Format of input (#{VALID_FORMATS.inspect})") do |i|
     fail ArgumentError, "Invalid input format: #{i}" unless VALID_FORMATS.include?(i.to_s)
@@ -31,18 +33,64 @@ OptionParser.new do |opts|
   opts.on('-f', '--forks [FORK_COUNT]', "Number of forks to use (parallel mode)") do |forks|
     fail ArgumentError, "Invalid argument for fork count: #{forks.inspect}" if !forks.match(/^\d+$/)
 
+    fail ArgumentError, 'Cannot specify both fork and thread counts at the same time' if options[:threads]
+
     options[:forks] = forks.to_i
   end
+
+  opts.on('-t', '--threads [THREAD_COUNT]', "Number of threads to use (parallel mode)") do |threads|
+    fail ArgumentError, "Invalid argument for thread count: #{threads.inspect}" if !threads.match(/^\d+$/)
+
+    fail ArgumentError, 'Cannot specify both fork and thread counts at the same time' if options[:forks]
+
+    options[:threads] = threads.to_i
+  end
 end.parse!
 
 def process_datum(_, options)
-  datum = _
+  code = if options[:code]
+           options[:code]
+         elsif ARGV[0]
+           "_ = #{_.inspect}; " + File.read(ARGV[0])
+         else
+           fail ArgumentError, 'Either the `-e`/`--exec` flag or a script should be specified'
+         end
 
-  eval(options[:code]).tap do |result|
-    if options[:print]
-      puts result
-    end
+  result = eval(code)
+
+  puts result if options[:print]
+end
+
+PARALLEL_MODE = options[:forks] || options[:threads]
+
+if PARALLEL_MODE
+  if options[:forks]
+    require 'tempfile'
+    LOCK_FILE_PATH = Tempfile.new('pyrosome_mutex_lockfile').path
+  else
+    MUTEX = Mutex.new
+  end
+
+  # Forks can't use Mutex
+  def sync_with_flock(&block)
+    fail ArgumentError, "No block specified" if block.nil?
+
+    file = File.open(LOCK_FILE_PATH)
+    file.flock(File::LOCK_EX)
+    block.call
+  ensure
+    file.flock(File::LOCK_UN)
+    file.close
+  end
+
+  def sync_with_mutex(&block)
+    fail ArgumentError, "No block specified" if block.nil?
+
+    MUTEX.synchronize { block.call }
   end
+
+  sync_method_name = options[:forks] ? :sync_with_flock : :sync_with_mutex
+  define_method(:sync, method(sync_method_name))
 end
 
 def iterate(stream, options)
@@ -51,16 +99,16 @@ def iterate(stream, options)
                require 'yajl'
                Yajl::Parser.new.parse(STDIN)
              when 'csv'
-               require 'csv'
                CSV.new(STDIN, headers: options[:headers])
+             else
+               STDIN.each_line
              end
 
-  if options[:forks]
-    puts 'parallel!'
+  if PARALLEL_MODE
     require 'parallel'
-    puts 'parallel?'
-    puts 'iterator.shift', iterator.shift.inspect
-    Parallel.each(lambda { iterator.shift || Parallel::Stop }, in_processes: options[:forks]) do |datum|
+    parallel_key = options[:forks] ? :in_processes : :in_threads
+
+    Parallel.each(lambda { iterator.shift || Parallel::Stop }, parallel_key => options[:forks]) do |datum|
       process_datum datum, options
     end
   else

data/lib/pyrosome/version.rb

@@ -1,3 +1,3 @@
 module Pyrosome
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pyrosome
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Brian Underwood
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-27 00:00:00.000000000 Z
+date: 2015-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
@@ -102,6 +102,7 @@ files:
 - lib/pyrosome.rb
 - lib/pyrosome/version.rb
 - pyrosome.gemspec
+- pyrosome_ride.jpg
 homepage: https://github.com/cheerfulstoic/pyrosome
 licenses:
 - MIT