pyrosome 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a8c7f462ebcbd7ad7c2f1fbce5a6bda419e9f7d7
-  data.tar.gz: 1285e2335b94289a86f6268850d054aef584e126
+  metadata.gz: cc5a6ff3c7d4e715388878be49a9bd371abc2115
+  data.tar.gz: bc5610d94c47f2bfe40bb55532dd757658eb9373
 SHA512:
-  metadata.gz: 177a82d33c424b9a5cc7a4a2425088b685096c1c9cf75faf8c7f9d27f1a9a6c525b46a03456f24f2b450487f09edcb8f2a7ab30ef3a830ac9801eebff7af3474
-  data.tar.gz: 84505ba050f17f6904e3bcd76f360898c28871e86c667b244d66b748cc34bc9fd1b55f298138fb28112974554f8902c80db016a604e302532297431760746ceb
+  metadata.gz: 196c87ac11f16c8837df5367693a6fc9d0671df65e9ffe02b67cd30b7a34fd5e27804413824665a76b3b714db92b83713f9e705a23fed0b8664ffcd6249396f2
+  data.tar.gz: 1c9cbc02a78e8bdf43d99f1334ed3335291b84b636393be0bb3e3398b9c4ec4407bd95c24a944f9924be27dab0a4a7934c4e3fd0295c72461b761ec94e97a93b

data/CHANGELOG.md

@@ -3,6 +3,16 @@ All notable changes to this project will be documented in this file.
 This file should follow the standards specified on [http://keepachangelog.com/]
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.3.0] - 11-27-2015
+
+### Changed
+
+- Changed `-f`/`--format` flag to `-i`/`--input`
+
+### Added
+
+- Added `-f`/`--forks` option to specify number of forks to process in parallel
+
 ## [0.2.0] - 11-27-2015
 
 ### Changed

data/README.md

@@ -23,15 +23,15 @@ Install the gem:
 
 The command to execute is `psome` (pronounced `p-some`, ryhmes with `roam`).  It takes a stream like this:
 
-    cat file.csv | psome -f csv -e "puts _[0]"
+    cat file.csv | psome -i csv -e "puts _[0]"
 
 You can see that we specify the format that is expected and a bit of Ruby code which exects to use a `_` variable
 
 ### Arguments
 
-#### -f [FORMAT] / --format [FORMAT]
+#### -i [FORMAT] / --input [FORMAT]
 
-Specify a format.  `json` and `csv` currently supported
+Specify a input.  `json` and `csv` currently supported
 
 #### -e [CODE] / --exec [CODE]
 
@@ -41,6 +41,34 @@ Give some Ruby code to be executed.  A `_` variable will be in scope and will re
 
 If the data is tabular, should we expect headers?
 
+## TODOs
+
+By default process lines like Ruby
+
+Parallel option
+ * Threads or forks
+ * Way to do mutex (`sync` method?)
+
+-i to do in place replacement of files
+
+Support file names as arguments.
+
+Support file name for code execution instead of -e?  Could files have both `_` and `ARGV` in scope so that scripts not made for pyrosome be used?
+
+Flag to echo sample of what you would get with the specified options (do I get an array or a hash?  Does the hash have keys?)
+
+Support for colors?
+
+Support for different enumerators?  Sort_by comes to mind, but that wouldn't work for streaming (or would it?). Select/reject.  Does any?/all? make sense?
+
+Support for loading an app environment (automatically load Rails if inside a project to get models?)
+
+Easy generation of CSV / JSON from Ruby data?
+
+Some way to load libraries?
+
+Refer to https://robm.me.uk/ruby/2015/10/31/dollar-underscore.html
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pyrosome. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.

data/bin/psome

@@ -8,10 +8,10 @@ options = {}
 OptionParser.new do |opts|
   opts.banner = "Usage: example.rb [options]"
 
-  opts.on("-f", "--format [FORMAT]", "Format of input") do |f|
-    fail ArgumentError, "Invalid format: #{f}" unless VALID_FORMATS.include?(f.to_s)
+  opts.on("-i", "--input [FORMAT]", "Format of input (#{VALID_FORMATS.inspect})") do |i|
+    fail ArgumentError, "Invalid input format: #{i}" unless VALID_FORMATS.include?(i.to_s)
 
-    options[:format] = f
+    options[:input] = i
   end
 
   opts.on("-e", "--exec [CODE]", "Ruby code to execute.  `datum` variable will be in scope") do |code|
@@ -27,6 +27,12 @@ OptionParser.new do |opts|
   opts.on('-p', '--print', "Prints the result of the evaluated code") do |print|
     options[:print] = print
   end
+
+  opts.on('-f', '--forks [FORK_COUNT]', "Number of forks to use (parallel mode)") do |forks|
+    fail ArgumentError, "Invalid argument for fork count: #{forks.inspect}" if !forks.match(/^\d+$/)
+
+    options[:forks] = forks.to_i
+  end
 end.parse!
 
 def process_datum(_, options)
@@ -39,15 +45,30 @@ def process_datum(_, options)
   end
 end
 
-case options[:format]
-when 'json'
-  require 'yajl'
-  Yajl::Parser.new.parse(STDIN).each do |datum|
-    process_datum(datum, options)
-  end
-when 'csv'
-  require 'csv'
-  CSV.new(STDIN, headers: options[:headers]).each do |datum|
-    process_datum(datum, options)
+def iterate(stream, options)
+  iterator = case options[:input]
+             when 'json'
+               require 'yajl'
+               Yajl::Parser.new.parse(STDIN)
+             when 'csv'
+               require 'csv'
+               CSV.new(STDIN, headers: options[:headers])
+             end
+
+  if options[:forks]
+    puts 'parallel!'
+    require 'parallel'
+    puts 'parallel?'
+    puts 'iterator.shift', iterator.shift.inspect
+    Parallel.each(lambda { iterator.shift || Parallel::Stop }, in_processes: options[:forks]) do |datum|
+      process_datum datum, options
+    end
+  else
+    iterator.each do |datum|
+      process_datum datum, options
+    end
   end
 end
+
+iterate(STDIN, options)
+

data/lib/pyrosome/version.rb

@@ -1,3 +1,3 @@
 module Pyrosome
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/pyrosome.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'yajl-ruby', '~> 1.2.1'
+  spec.add_dependency 'parallel', '~> 1.6.1'
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pyrosome
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Brian Underwood
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -88,8 +102,6 @@ files:
 - lib/pyrosome.rb
 - lib/pyrosome/version.rb
 - pyrosome.gemspec
-- test.csv
-- test.json
 homepage: https://github.com/cheerfulstoic/pyrosome
 licenses:
 - MIT

data/test.csv

@@ -1,2 +0,0 @@
-test,"foo"
-"bar,baz",2

data/test.json

@@ -1 +0,0 @@
-[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100]