elasticrawl 1.0.0

data/lib/elasticrawl/job_step.rb

@@ -0,0 +1,46 @@
+module Elasticrawl
+  # Represents an Elastic MapReduce job flow step.  For a parse job this will
+  # process a single Common Crawl segment.  For a combine job a single step
+  # will aggregate the results of multiple parse jobs.
+  class JobStep < ActiveRecord::Base
+    belongs_to :job
+    belongs_to :crawl_segment
+
+    # Returns a custom jar step that is configured with the jar location,
+    # class name and input and output paths.
+    #
+    # For parse jobs optionally specifies the maximum # of Common Crawl
+    # data files to process before the job exits.
+    def job_flow_step(job_config)
+      jar = job_config['jar']
+      max_files = self.job.max_files 
+
+      step_args = []
+      step_args[0] = job_config['class']
+      step_args[1] = self.input_paths
+      step_args[2] = self.output_path
+      # All arguments must be strings.
+      step_args[3] = max_files.to_s if max_files.present?
+
+      step = Elasticity::CustomJarStep.new(jar)
+      step.name = set_step_name
+      step.arguments = step_args
+
+      step
+    end
+
+  private
+    # Sets the Elastic MapReduce job flow step name based on the type of job it
+    # belongs to.
+    def set_step_name
+      case self.job.type
+        when 'Elasticrawl::ParseJob'
+          segment =self.crawl_segment.segment_name if self.crawl_segment.present?
+          "Segment: #{segment}"
+        when 'Elasticrawl::CombineJob'
+          paths = self.input_paths.split(',')
+          "Combining #{paths.count} jobs"
+      end
+    end
+  end
+end

data/lib/elasticrawl/parse_job.rb

@@ -0,0 +1,84 @@
+module Elasticrawl
+  # Represents an Elastic MapReduce job flow that parses segments of
+  # Common Crawl data. A job step is created per segment.
+  #
+  # Inherits from Job which is the ActiveRecord model class.
+  class ParseJob < Job
+    # Populates the job from the list of segments to be parsed.
+    def set_segments(crawl_segments, max_files = nil)
+      self.job_name = set_job_name
+      self.job_desc = set_job_desc(crawl_segments, max_files)
+      self.max_files = max_files
+
+      crawl_segments.each do |segment|
+        self.job_steps.push(create_job_step(segment))
+      end
+    end
+
+    # Runs the job by calling Elastic MapReduce API.  If successful the
+    # parse time is set for each segment.
+    def run
+      emr_config = job_config['emr_config']
+      job_flow_id = run_job_flow(emr_config)
+
+      if job_flow_id.present?
+        self.job_flow_id = job_flow_id
+
+        self.job_steps.each do |step|
+          segment = step.crawl_segment
+          segment.parse_time = DateTime.now
+          segment.save
+        end
+
+        self.save
+        self.result_message
+      end
+    end
+
+    # Returns the S3 location for storing Elastic MapReduce job logs.
+    def log_uri
+      s3_path = "/logs/1-parse/#{self.job_name}/"
+      build_s3_uri(s3_path)
+    end
+
+  private
+    # Creates a job step for the crawl segment.
+    def create_job_step(segment)
+      JobStep.create(:job => self,
+                     :crawl_segment => segment,
+                     :input_paths => segment_input(segment),
+                     :output_path => segment_output(segment))
+    end
+
+    # Returns the S3 location for reading a crawl segment. The input filter
+    # determines which type of Common Crawl data files are parsed.
+    def segment_input(segment)
+      segment.segment_s3_uri + job_config['input_filter']
+    end
+
+    # Returns the S3 location for storing the step results.  This includes
+    # the segment name.
+    def segment_output(segment)
+      job_path = "/data/1-parse/#{self.job_name}"
+      s3_path = "#{job_path}/segments/#{segment.segment_name}/"
+      build_s3_uri(s3_path)
+    end
+
+    # Sets the job description which forms part of the Elastic MapReduce
+    # job flow name.
+    def set_job_desc(segments, max_files)
+      if segments.count > 0
+        crawl_name = segments[0].crawl.crawl_name if segments[0].crawl.present?
+        file_desc = max_files.nil? ? 'all files' : "#{max_files} files per segment"
+      end
+
+      "Crawl: #{crawl_name} Segments: #{segments.count} Parsing: #{file_desc}"
+    end
+
+    # Returns the parse job configuration from ~/.elasticrawl.jobs.yml.
+    def job_config
+      config = Config.new
+      config.load_config('jobs')['steps']['parse']
+    end
+  end
+end

data/lib/elasticrawl/version.rb

@@ -0,0 +1,3 @@
+module Elasticrawl
+  VERSION = "1.0.0"
+end

data/lib/elasticrawl.rb

@@ -0,0 +1,21 @@
+require 'aws-sdk'
+require 'active_record'
+require 'active_support'
+require 'elasticity'
+require 'highline/import'
+require 'thor'
+
+module Elasticrawl
+  require 'elasticrawl/version'
+
+  require 'elasticrawl/config'
+  require 'elasticrawl/error'
+
+  require 'elasticrawl/cluster'
+  require 'elasticrawl/crawl'
+  require 'elasticrawl/crawl_segment'
+  require 'elasticrawl/job'
+  require 'elasticrawl/combine_job'
+  require 'elasticrawl/parse_job'
+  require 'elasticrawl/job_step'
+end

data/spec/fixtures/aws.yml

@@ -0,0 +1,4 @@
+# Configures the AWS credentials used when accessing the AWS EMR and S3 APIs.
+# This file is populated by the elasticrawl init command.
+access_key_id: 'ACCESS_KEY_ID'
+secret_access_key: 'SECRET_ACCESS_KEY'

data/spec/fixtures/cluster.yml

@@ -0,0 +1,44 @@
+# Configures the Elastic MapReduce cluster that is launched to run parse and
+# combine jobs. The list of EC2 instance types can be found at
+# http://aws.amazon.com/ec2/instance-types/#instance-details
+
+# Using spot instances is recommended to reduce costs. However if the spot
+# price rises above your bid price the cluster may be terminated. Elasticrawl
+# tries to reduce the effect of this by parsing each Commmon Crawl segment
+# in a separate job flow step.
+
+# The master node manages the cluster.
+master_instance_group:
+  instance_type: m1.medium
+  use_spot_instances: true
+  bid_price: 0.120
+
+# Core nodes run map and reduce tasks and store data using HDFS.
+core_instance_group:
+  instance_type: m1.medium
+  instance_count: 2
+  use_spot_instances: true
+  bid_price: 0.120
+
+# Task nodes are optional and only run map and reduce tasks.
+task_instance_group:
+  instance_type: m1.small
+  instance_count: 0
+  use_spot_instances: true
+  bid_price: 0.080
+
+# Array of bootstrap scripts that will be applied when the cluster nodes are
+# initialized. The example installs the Ganglia distributed monitoring system.
+bootstrap_scripts: #['s3://elasticmapreduce/bootstrap-actions/install-ganglia']
+
+# Specifying an EC2 key pair allows SSH access to the master node. This also
+# allows accessing the Hadoop Web UI over an SSH tunnel.
+ec2_key_name: 'elasticrawl'
+
+# Availability Zone (AZ) to launch instances in. An AZ in the US-East region is
+# recommended since the Common Crawl corpus is stored there. Otherwise inter
+# region data transfer charges will apply.
+placement: 'us-east-1c'
+
+# The AMI version to use when launching instances.
+emr_ami_version: 'latest'

data/spec/fixtures/jobs.yml

@@ -0,0 +1,31 @@
+# Configures the AWS Elastic MapReduce jobs launched against the Common Crawl
+# corpus. 
+
+# An S3 bucket is created by the init command and is used to store data and logs.
+s3_bucket_name: 'elasticrawl'
+
+# A parse step is created per Common Crawl segment.  A combine step takes the
+# results from multiple segments to create a single set of output files.
+
+# The parse input filter is used to specify the Common Crawl file type.
+
+# WARC: 'warc/*.warc.gz' - Full HTTP requests and responses.
+# WAT: 'wat/*.warc.wat.gz' - Metadata extractions from WARC files.
+# WET: 'wet/*.warc.wet.gz' - Text extractions from WARC files.
+
+# The EMR config is an XML file that sets Hadoop properties.  If a config file
+# is specified then a bootstrap action is run on each node to apply it.
+steps:
+  # Parse step for the Example Elasticrawl JAR.  This does a word count
+  # against the text extractions of the corpus.
+  parse:
+    jar: 's3://elasticrawl/jar/elasticrawl-examples-1.0.0.jar'
+    class: 'com.rossfairbanks.elasticrawl.examples.WordCount'
+    input_filter: 'wet/*.warc.wet.gz'
+    emr_config: #'s3://elasticrawl/jar/parse-mapred-site.xml'
+  # Combine step for the Example Elasticrawl JAR.
+  combine:
+    jar: 's3://elasticrawl/jar/elasticrawl-examples-1.0.0.jar'
+    class: 'com.rossfairbanks.elasticrawl.examples.SegmentCombiner'
+    input_filter: 'part-*'
+    emr_config: #'s3://elasticrawl/jar/combine-mapred-site.xml'

data/spec/spec_helper.rb

@@ -0,0 +1,35 @@
+require 'elasticrawl'
+require 'rspec'
+require 'mocha'
+require 'database_cleaner'
+require 'shoulda-matchers'
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    # Return S3 paths that are used to create a crawl object with 3 crawl segments.
+    segment_paths = []
+    segment_paths[0] = 'common-crawl/crawl-data/CC-MAIN-2013-20/segments/1368696381249/'
+    segment_paths[1] = 'common-crawl/crawl-data/CC-MAIN-2013-20/segments/1368696381630/'
+    segment_paths[2] = 'common-crawl/crawl-data/CC-MAIN-2013-20/segments/1368696382185/'
+    Elasticrawl::Crawl.any_instance.stubs(:s3_segment_paths).returns(segment_paths)
+
+    # Load config from spec/fixtures/ rather than ~/.elasticrawl/
+    config_dir = File.join(File.dirname(__FILE__), 'fixtures')
+    Elasticrawl::Config.any_instance.stubs(:config_dir).returns(config_dir)
+
+    # Load sqlite database. For testing this is stored at db/elasticrawl.sqlite3
+    config = Elasticrawl::Config.new
+    config.load_database
+  end
+
+  # Run each test in a transaction and rollback data on completion.
+  DatabaseCleaner.strategy = :transaction
+
+  config.before(:each) do
+    DatabaseCleaner.start
+  end
+
+  config.after(:each) do
+    DatabaseCleaner.clean
+  end
+end

data/spec/unit/cluster_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe Elasticrawl::Cluster do
+  describe '#create_job_flow' do
+    let(:crawl) { Elasticrawl::Crawl.create(:crawl_name => 'CC-MAIN-2013-20') }
+    let(:job) { Elasticrawl::ParseJob.new }
+    let(:cluster) { Elasticrawl::Cluster.new }
+    subject { cluster.create_job_flow(job) }
+
+    before do
+      job.set_segments(crawl.crawl_segments)
+    end
+
+    it 'should be an Elasticity::JobFlow' do
+      expect(subject).to be_a Elasticity::JobFlow
+    end
+
+    it 'should have a job flow name' do
+      expect(subject.name).to eq "Job Name: #{job.job_name} #{job.job_desc}"
+    end
+
+    it 'should have a log uri' do
+      expect(subject.log_uri).to eq job.log_uri
+    end
+
+    it 'should have an ec2 key name' do
+      expect(subject.ec2_key_name).to eq 'elasticrawl'
+    end
+
+    it 'should have a placement az name' do
+      expect(subject.placement).to eq 'us-east-1c'
+    end
+
+    it 'should have an ami version' do
+      expect(subject.ami_version).to eq 'latest'
+    end
+  end
+
+  describe '#cluster_desc' do
+    let(:cluster_desc) {
+      cluster_desc = <<-HERE
+Cluster configuration
+Master: 1 m1.medium  (Spot: 0.12)
+Core:   2 m1.medium  (Spot: 0.12)
+Task:   --
+      HERE
+    }
+    subject { Elasticrawl::Cluster.new } 
+
+    it 'should describe configured instance groups' do
+      expect(subject.cluster_desc).to eq cluster_desc
+    end
+  end
+end

data/spec/unit/combine_job_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+describe Elasticrawl::CombineJob do
+  describe '#set_input_jobs' do
+    let(:job_name) { (Time.now.to_f * 1000).to_i.to_s }
+    let(:crawl) { Elasticrawl::Crawl.create(:crawl_name => 'CC-MAIN-2013-20') }
+    let(:segment_list_1) { crawl.crawl_segments[0..1] }
+    let(:segment_list_2) { [crawl.crawl_segments[2]]}
+
+    let(:parse_job_1) { Elasticrawl::ParseJob.new }
+    let(:parse_job_2) { Elasticrawl::ParseJob.new }
+    let(:combine_job) { Elasticrawl::CombineJob.new }
+
+    before do
+      crawl.create_segments
+      parse_job_1.set_segments(segment_list_1)
+      parse_job_2.set_segments(segment_list_2)
+
+      input_jobs = [parse_job_1.job_name, parse_job_2.job_name]
+      combine_job.set_input_jobs(input_jobs)
+    end
+
+    it 'should have a job name based on current time' do
+      expect(combine_job.job_name.slice(0, 8)).to eq job_name.slice(0, 8)
+    end
+
+    it 'should have a job desc' do
+      expect(combine_job.job_desc.end_with?('Combining: 3 segments')).to eq true
+    end
+
+    it 'should create 1 job step' do
+      expect(combine_job.job_steps.count).to eq 1
+    end
+
+    it 'should set 1 input path per parse job' do
+      input_paths = combine_job.job_steps[0].input_paths
+      expect(input_paths.split(',').count).to eq 2
+    end
+
+    it 'should set input path including parse job name' do
+      input_paths = combine_job.job_steps[0].input_paths
+      expect(input_paths.include?(parse_job_1.job_name)).to eq true
+    end
+
+    it 'should set input path without segment names' do
+      input_paths = combine_job.job_steps[0].input_paths
+      segment_name = segment_list_1[0].segment_name
+      expect(input_paths.include?(segment_name)).to eq false
+    end
+
+    it 'should set output path including job name' do
+      output_path = combine_job.job_steps[0].output_path
+      expect(output_path.include?(combine_job.job_name)).to eq true
+    end
+  end
+
+  describe '#run' do
+    let(:crawl) { Elasticrawl::Crawl.create(:crawl_name => 'CC-MAIN-2013-20') }
+    let(:parse_job_1) { Elasticrawl::ParseJob.new }
+    let(:parse_job_2) { Elasticrawl::ParseJob.new }
+    let(:combine_job) { Elasticrawl::CombineJob.new }
+    let(:job_flow_id) { 'j-3QHDKKBT6VAIS' }
+
+    before do
+      crawl.create_segments
+      parse_job_1.set_segments(crawl.crawl_segments[0..1])
+      parse_job_2.set_segments([crawl.crawl_segments[2]])
+
+      input_jobs = [parse_job_1.job_name, parse_job_2.job_name]
+      combine_job.set_input_jobs(input_jobs)
+    end
+
+    it 'should set a job flow id' do
+      Elasticity::JobFlow.any_instance.stubs(:run).returns(job_flow_id)
+      combine_job.run
+
+      expect(combine_job.job_flow_id).to eq job_flow_id
+    end
+  end
+
+  describe '#log_uri' do
+    let(:crawl) { Elasticrawl::Crawl.create(:crawl_name => 'CC-MAIN-2013-20') }
+    let(:parse_job) { Elasticrawl::ParseJob.new }
+    let(:job) { Elasticrawl::CombineJob.new }
+
+    before do
+      crawl.create_segments
+      parse_job.set_segments(crawl.crawl_segments)
+
+      job.set_input_jobs([parse_job.job_name])
+    end
+
+    it 'should set a log uri including the job name' do
+      expect(job.log_uri).to eq "s3://elasticrawl/logs/2-combine/#{job.job_name}/"
+    end
+  end
+end

data/spec/unit/config_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe Elasticrawl::Config do
+  describe '#load_config' do
+    subject { Elasticrawl::Config.new }
+
+    it 'should return a hash of config data' do
+      config_data = subject.load_config('jobs')
+      expect(config_data).to be_a Hash
+    end
+
+    it 'should load yaml config file' do
+      config_data = subject.load_config('jobs')
+      expect(config_data['s3_bucket_name']).to eq 'elasticrawl'
+    end
+  end
+end

data/spec/unit/crawl_segment_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe Elasticrawl::CrawlSegment do
+  it { should belong_to(:crawl) }
+  it { should have_many(:job_steps) }
+  it { should have_db_column(:segment_name).of_type(:string) }
+  it { should have_db_column(:segment_s3_uri).of_type(:string) }
+  it { should have_db_column(:parse_time).of_type(:datetime) }
+
+  describe '#initialize' do
+    let(:crawl) { Elasticrawl::Crawl.create(:crawl_name => 'CC-MAIN-2013-20') }
+    subject { crawl.crawl_segments[0]}
+
+    before do
+      crawl.create_segments
+    end
+
+    it 'should have a segment name' do
+      expect(subject.segment_name).to eq '1368696381249'
+    end
+
+    it 'should have an s3 uri' do
+      expect(subject.segment_s3_uri).to eq \
+        's3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2013-20/segments/1368696381249/'
+    end
+  end
+end

data/spec/unit/crawl_spec.rb

@@ -0,0 +1,137 @@
+require 'spec_helper'
+
+describe Elasticrawl::Crawl do
+  it { should have_many(:crawl_segments) }
+  it { should have_db_column(:crawl_name).of_type(:string) }
+
+  describe '#has_segments?' do
+    let(:crawl_name) { 'CC-MAIN-2013-20' }
+    subject { Elasticrawl::Crawl.new(:crawl_name => crawl_name) }
+
+    it 'should have segments' do
+      expect(subject.has_segments?).to eq true
+    end
+  end
+
+  describe '#create_segments' do
+    let(:crawl_name) { 'CC-MAIN-2013-20' }
+    subject { Elasticrawl::Crawl.create(:crawl_name => crawl_name) }
+
+    before do
+      subject.create_segments
+    end
+
+    it 'should set crawl name' do
+      expect(subject.crawl_name).to eq crawl_name
+    end
+
+    it 'should create correct # of segments' do
+      expect(subject.crawl_segments.count).to eq 3
+    end
+
+    it 'should create segment names' do
+      expect(subject.crawl_segments[0].segment_name).to eq '1368696381249'
+    end
+
+    it 'should create segment s3 uris' do
+      expect(subject.crawl_segments[0].segment_s3_uri).to eq \
+        's3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2013-20/segments/1368696381249/'
+    end
+  end
+
+  describe '#next_segments' do
+    let(:crawl_name) { 'CC-MAIN-2013-20' }
+    subject { Elasticrawl::Crawl.create(:crawl_name => crawl_name) }
+
+    before do
+      subject.create_segments
+    end
+
+    it 'should return all segments' do
+      crawl_segments = subject.next_segments
+
+      expect(crawl_segments.count).to eq 3
+      expect(crawl_segments[0].crawl.crawl_name).to eq crawl_name
+      expect(crawl_segments[0].segment_name).to eq '1368696381249'
+    end
+
+    it 'should return first # segments' do
+      crawl_segments = subject.next_segments(2)
+
+      expect(crawl_segments.count).to eq 2
+      expect(crawl_segments[0].crawl.crawl_name).to eq crawl_name
+      expect(crawl_segments[0].segment_name).to eq '1368696381249'
+    end
+  end
+
+  describe '#select_segments' do
+    let(:crawl_name) { 'CC-MAIN-2013-20' }
+    subject { Elasticrawl::Crawl.create(:crawl_name => crawl_name) }
+
+    before do
+      subject.create_segments
+    end
+
+    it 'should select no segments' do
+      segments_list = ['test', 'segment']
+      crawl_segments = subject.select_segments(segments_list)
+
+      expect(crawl_segments.count).to eq 0
+    end
+
+    it 'should select only segments in list' do
+      segments_list = ['1368696381249', '1368696382185']
+      crawl_segments = subject.select_segments(segments_list)
+
+      expect(crawl_segments.count).to eq 2
+    end
+  end
+
+  describe '#reset' do
+    let(:crawl) { Elasticrawl::Crawl.create(:crawl_name => 'CC-MAIN-2013-20') }
+    let(:job) { Elasticrawl::ParseJob.new }
+    let(:job_flow_id) { 'j-3QHDKKBT6VAIS' }
+
+    before do
+      crawl.create_segments
+      job.set_segments(crawl.crawl_segments[0..1])
+
+      Elasticity::JobFlow.any_instance.stubs(:run).returns(job_flow_id)
+      job.run
+
+      crawl.reset
+    end
+
+    it 'should set parse time of all segments to null' do
+      unparsed_segments = Elasticrawl::CrawlSegment.where(:parse_time => nil).count
+      expect(crawl.crawl_segments.count).to eq unparsed_segments
+    end
+  end
+
+  describe '.status' do
+    let(:job_desc) { 'Crawl: CC-MAIN-2013-20 Segments: 2 Parsing: 5 files per segment' }
+    let(:crawl) { Elasticrawl::Crawl.create(:crawl_name => 'CC-MAIN-2013-20') }
+    let(:max_files) { 5 }
+    let(:job) { Elasticrawl::ParseJob.new }
+    let(:job_flow_id) { 'j-3QHDKKBT6VAIS' }
+
+    before do
+      crawl.create_segments
+      job.set_segments(crawl.crawl_segments[0..1], max_files)
+
+      Elasticity::JobFlow.any_instance.stubs(:run).returns(job_flow_id)
+      job.run
+    end
+
+    it 'should display status of crawl segments' do
+      expect(Elasticrawl::Crawl.status.split("\n")[1]).to eq \
+        'CC-MAIN-2013-20 Segments: to parse 1, parsed 2, total 3'
+    end
+
+    it 'should display parse job desc' do
+      crawl_status = Elasticrawl::Crawl.status.split("\n")[4]
+      expect(crawl_status.include?(job.job_name)).to eq true
+      expect(crawl_status.include?(job.job_desc)).to eq true
+    end
+  end
+end

data/spec/unit/job_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe Elasticrawl::Job do
+  it { should have_many(:job_steps) }
+  it { should have_db_column(:type).of_type(:string) }
+  it { should have_db_column(:job_name).of_type(:string) }
+  it { should have_db_column(:job_desc).of_type(:string) }
+  it { should have_db_column(:max_files).of_type(:integer) }
+  it { should have_db_column(:job_flow_id).of_type(:string) }
+end

data/spec/unit/job_step_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+describe Elasticrawl::JobStep do
+  it { should belong_to(:job) }
+  it { should belong_to(:crawl_segment) }
+  it { should have_db_column(:input_paths).of_type(:text) }
+  it { should have_db_column(:output_path).of_type(:text) }
+
+  describe '#job_flow_step' do
+    let(:job) { Elasticrawl::ParseJob.create(:job_name => '1389789645620',
+                                              :max_files => 5) }
+    let(:crawl) { Elasticrawl::Crawl.create(:crawl_name => 'CC-MAIN-2013-20') }
+    let(:crawl_segment) { crawl.crawl_segments[0] }
+    let(:input_paths) {
+      's3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2013-20/segments/1368696381249/wet/*.warc.wet.gz'
+    }
+    let(:output_path) {
+      's3://elasticrawl/data/1-parse/1389789645620/segments/1368696381249/'
+    }
+    let(:config) {
+      { 'jar' => 's3://elasticrawl/jar/elasticrawl-example-0.0.1.jar',
+        'class' => 'com.rossfairbanks.commoncrawl.elasticrawl.ParserDriver'
+      }
+    }
+
+    let(:job_step) { Elasticrawl::JobStep.create(:job => job,
+                                          :crawl_segment => crawl_segment,
+                                          :input_paths => input_paths,
+                                          :output_path => output_path) }
+    subject { job_step.job_flow_step(config) } 
+
+    it 'should be a CustomJarStep' do
+      expect(subject).to be_a Elasticity::CustomJarStep
+    end
+
+    it 'should have a jar location' do
+      expect(subject.jar).to eq config['jar']
+    end
+
+    it 'should have 4 jar args' do
+      expect(subject.arguments.count).to eq 4
+    end
+
+    it 'should have a class argument' do
+      expect(subject.arguments[0]).to eq config['class']
+    end
+
+    it 'should have an input path arg' do
+      expect(subject.arguments[1]).to eq input_paths
+    end
+
+    it 'should have an output path arg' do
+      expect(subject.arguments[2]).to eq output_path
+    end
+
+    it 'should have a max files arg' do
+      expect(subject.arguments[3]).to eq '5'
+    end
+  end
+end