csv-curl 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d04f4437667318ac6a0d869e1ebbc6b72de7d6d1c86ecd55b98dd8d74ae79a5b
-  data.tar.gz: 944c0ef6d3837bbd38e4d1bd11b4e3d36fab95274452cb121f9440e34b0f9748
+  metadata.gz: 381850913b0f63297ab7a5186c9108880d09774930604bd965f7958b573bdeca
+  data.tar.gz: df854cee03e5e4f2c1bc68da0d1c37a845ab8ad9fcb9b19aa0c8f024064999c1
 SHA512:
-  metadata.gz: c432715c775b43c27f38b6e21b9410a934ce87a528a17c32f86ac906de98185d17e3e84bcb700dab4f767d7fae4f0dd3df4eda5cdb475e200e785db5f5fa0b01
-  data.tar.gz: db278224c3a4e4d1f702de80495f4758796916ac69ca363cebe9c94cd580948ec8a3adae0315994f4d62f3c5126831519ddac915cc21fabf59b43aab9a847035
+  metadata.gz: 9582c3888f042d002b1d229d5433c2bfa7c9d4e8cf7845d3caed560219614f393eca5fa659c8f2f286c7222ebcee4b53bbdccbe576356998a2d79df89cd6a755
+  data.tar.gz: c45dd8f7fc333b38b493b2b4904956b678c498c7fa281f41b90a6d7960e3037ab5bdd3732c2a366087e949d3673de0f69aed6ce21e0be18e3bacba443c6109f1

data/.gitignore

@@ -50,3 +50,6 @@ build-iPhoneSimulator/
 .rvmrc
 
 *~
+
+csv-utils.response
+csv-utils.request

data/README.md

@@ -1 +1,55 @@
 # csv-curl
+
+Utility for making multiple curl requests using a CSV file.
+
+### Usage
+```
+Usage: csv-curl[OPTIONS] [CURL OPTIONS]
+        --template FILE              Path to the template file
+        --csv CSV_FILE               Path to the CSV file
+        --exec UTILITY               Utility to run on each response
+```
+
+### Template
+
+File containing the payload to pass to the curl command, the "-d@<file>" option.
+
+### CSV File
+
+A CSV file containing the replacement values to use.  It must have a header.  And have a header for all replacement variables.
+
+### Utility
+
+This is an additional command to run on the response body from the curl command.
+
+Example to output only the "id" from a JSON body
+
+```
+csv-curl --csv emails.csv 'https://example.com/user_lookup?email={{param:email}}' --exec 'jq ".id" -r'
+```
+
+### Replacement variables
+
+A replacement variable is enclosed in handlebars **{{** **format:** **name** **}}**.  Replacement variables can be used in the arguments to curl, in the template and in the utility.
+
+*(format and name a separated by a colon)*
+
+- **name** matches a header in the CSV file
+- **format** escape sequence to apply the the replacement value
+    - _json_ JSON encode
+    - _param_ URL encode
+    - _base64_ Base64 encode
+    - _urlsafe64_ URL safe Base64 encode
+    - _hex_ Hex encode
+    - _shell_ Shell encode
+
+Example: lookup users by email and save response by user id.
+
+```
+csv-curl --csv users.csv 'https://example.com/user_lookup?email={{param:email}}' --exec 'cat > {{id}}.json'
+
+# users.csv
+id,email
+5,bob@example.com
+8,johndoe@example.com
+```

data/bin/csv-curl

@@ -11,30 +11,51 @@ require 'digest/md5'
 options = {
   template_file: nil,
   csv_file: nil,
-  response_filter: nil,
+  exec: nil,
   default_format: 'json'
 }
 
-OptionParser.new do |opts|
-  opts.banner = "Usage: " + File.basename(__FILE__) + " [CURL OPTIONS]"
+OPTPARSE = OptionParser.new do |opts|
+  opts.banner = "Usage: " + File.basename(__FILE__) + "[OPTIONS] [CURL OPTIONS]"
 
-  opts.on('--template REQUEST', 'Request template') do |v|
+  opts.on('--template FILE', 'Path to the template file') do |v|
     options[:template_file] = v
   end
 
-  opts.on('--csv CSV_FILE', 'Request data') do |v|
+  opts.on('--csv CSV_FILE', 'Path to the CSV file') do |v|
     options[:csv_file] = v
   end
 
-  opts.on('--response-filter COMMAND', 'Command to run on each response') do |v|
-    options[:response_filter] = v
+  opts.on('--exec UTILITY', 'Utility to run on each response') do |v|
+    options[:exec] = v
   end
-end.parse!
+end
+
+CSV_CURL_ARGV = []
+ARGV.size.times.to_a.reverse.each do |idx|
+  case ARGV[idx]
+  when '--template',
+       '--csv',
+       '--exec'
+    CSV_CURL_ARGV << ARGV.delete_at(idx)
+    CSV_CURL_ARGV << ARGV.delete_at(idx)
+  end
+end
+OPTPARSE.parse(CSV_CURL_ARGV)
+
+def failed(msg, errno = 1)
+  $stderr.puts msg
+  if errno == 1
+    puts ""
+    puts OPTPARSE
+  end
+  exit errno
+end
 
-raise('no csv input file specified') unless options[:csv_file]
-raise("csv input file #{options[:csv_file]} not found") unless File.exist?(options[:csv_file])
+failed('no csv input file specified') unless options[:csv_file]
+failed("csv input file #{options[:csv_file]} not found") unless File.exist?(options[:csv_file])
 
-raise("template file #{options[:template_file]} not found") if options[:template_file] && !File.exist?(options[:template_file])
+failed("template file #{options[:template_file]} not found") if options[:template_file] && !File.exist?(options[:template_file])
 
 template = options[:template_file] ? File.read(options[:template_file]) : nil
 input = CSV.open(options[:csv_file])
@@ -44,18 +65,20 @@ required_replacements = ARGV.map do |arg|
   arg.scan(/\{\{(.*?)\}\}/)
 end
 
-required_replacements += options[:response_filter].scan(/\{\{(.*?)\}\}/) if options[:response_filter]
+required_replacements += options[:exec].scan(/\{\{(.*?)\}\}/) if options[:exec]
 required_replacements += template.scan(/\{\{(.*?)\}\}/) if template
 required_replacements.flatten!
 required_replacements.map! { |v| v.split(':', 2).last }
 required_replacements.uniq!
 
 missing_replacements = required_replacements - input_headers
-raise("missing replacement values for #{missing_replacements.join(', ')}") unless missing_replacements.empty?
+failed("missing replacement values for #{missing_replacements.join(', ')}", 2) unless missing_replacements.empty?
 
 TMP_REQUEST_BODY_FILE = 'csv-utils.request'
 
 def format_value(value, format)
+  return '' unless value
+
   case format
   when 'json'
     value = value.to_json
@@ -67,6 +90,8 @@ def format_value(value, format)
     CGI.escape(value)
   when 'base64'
     Base64.strict_encode64(value)
+  when 'urlsafe64'
+    Base64.urlsafe_encode64(value)
   when 'hex'
     value.unpack('H*').first
   when 'shellword',
@@ -92,7 +117,7 @@ end
 
 def run_command_safely(cmd)
   res = `#{cmd}`
-  raise("failed to run command: #{cmd}") unless $?.success?
+  failed("failed to run command: #{cmd}", 2) unless $?.success?
   res
 end
 
@@ -104,9 +129,9 @@ def build_curl_command(curl_args, data, request_file, response_file)
   cmd
 end
 
-def build_response_filter_command(response_filter, data, response_file)
+def build_exec_command(exec, data, response_file)
   cmd = "cat #{response_file} | "
-  cmd += generate_string(response_filter, data)
+  cmd += generate_string(exec, data)
   cmd
 end
 
@@ -128,12 +153,16 @@ while (row = input.shift)
   run_command_safely(cmd)
 
   output =
-    if options[:response_filter]
-      cmd = build_response_filter_command(options[:response_filter], data, response_file)
+    if options[:exec]
+      cmd = build_exec_command(options[:exec], data, response_file)
       run_command_safely(cmd)
     else
       File.read(response_file)
     end
 
   puts output
+
+  File.unlink(response_file) if File.exist?(response_file)
 end
+
+File.unlink(TMP_REQUEST_BODY_FILE) if File.exist?(TMP_REQUEST_BODY_FILE)

data/csv-curl.gemspec

@@ -2,14 +2,14 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-curl'
-  s.version     = '0.1.0'
+  s.version     = '0.2.0'
   s.licenses    = ['MIT']
   s.summary     = 'CSV Curl'
   s.description = 'Tools making mulitple calls using curl'
   s.authors     = ['Doug Youch']
   s.email       = 'dougyouch@gmail.com'
   s.homepage    = 'https://github.com/dougyouch/csv-curl'
-  s.files       = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  s.files       = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|examples)/}) }
   s.bindir      = 'bin'
   s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-curl
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Doug Youch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-22 00:00:00.000000000 Z
+date: 2019-08-23 00:00:00.000000000 Z
 dependencies: []
 description: Tools making mulitple calls using curl
 email: dougyouch@gmail.com