hasta 0.1.0

data/spec/hasta/env_spec.rb

@@ -0,0 +1,30 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/env'
+
+describe Hasta::Env do
+  describe '#setup' do
+    subject { described_class.new(variables, files, combined_storage) }
+
+    let(:combined_storage) { double(Hasta::CombinedStorage) }
+    let(:file) { double('File', :body => contents, :key => s3_uri.path) }
+    let(:contents) { "one,two,three\nfour,five,six" }
+
+    let(:variables) { { 'API_KEY' => '123456' } }
+    let(:files) { { 'DATA_FILE_PATH' => s3_uri } }
+    let(:s3_uri) { Hasta::S3URI.parse('s3://my-bucket/path/to/data.csv') }
+
+    it 'writes the file locally and includes the path in the ENV' do
+      combined_storage.
+        should_receive(:write).
+        with(s3_uri, kind_of(Hasta::S3DataSource)).
+        and_return(s3_uri)
+
+      expect(subject.setup).to eq(
+        variables.merge({ 'DATA_FILE_PATH' => Hasta::LocalFilePath.for(s3_uri) })
+      )
+    end
+  end
+end

data/spec/hasta/execution_context_spec.rb

@@ -0,0 +1,67 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/execution_context'
+require 'hasta/s3_data_source'
+require 'hasta/in_memory_data_sink'
+require 'hasta/in_memory_data_source'
+
+describe Hasta::ExecutionContext do
+  describe '#execute' do
+    let(:source_file) { 'spec/fixtures/hasta/lib/test_identity_mapper.rb' }
+    let(:data_source) { Hasta::InMemoryDataSource.new(lines) }
+    let(:data_sink) { Hasta::InMemoryDataSink.new }
+    let(:lines) { %w[First Second Third] }
+    let(:exp_lines) { lines.map { |line| "#{line}\t" } }
+    let(:results) { subject.execute(source_file, data_source, data_sink).data_source }
+
+    it 'returns the execution results' do
+      expect(results.to_a).to eq(exp_lines)
+    end
+
+    context 'given env variables' do
+      subject { described_class.new([], env) }
+
+      let(:env) { { 'LINE_PREFIX' => prefix } }
+      let(:prefix) { 'Copyright 2014 ' }
+      let(:source_file) { 'spec/fixtures/hasta/lib/test_env_mapper.rb' }
+      let(:exp_lines) { lines.map { |line| "#{prefix}#{line}\t" } }
+
+      it 'returns the execution results' do
+        expect(results.to_a).to eq(exp_lines)
+      end
+
+      it 'does not set the ENV of the parent process' do
+        expect(ENV['LINE_PREFIX']).to be_nil
+      end
+    end
+
+    context 'given additional Ruby files' do
+      subject { described_class.new(files) }
+
+      let(:files) { [file] }
+      let(:file) { "#{File.dirname(__FILE__)}/../fixtures/hasta/lib/types.rb" }
+      let(:dir) { File.expand_path(File.dirname(file)) }
+      let(:source_file) { 'spec/fixtures/hasta/lib/test_types_mapper.rb' }
+
+      it 'returns the execution results' do
+        expect(results.to_a).to eq(exp_lines)
+      end
+
+      it 'does not affect the $LOAD_PATH of the parent process' do
+        expect($LOAD_PATH).to_not include(dir)
+      end
+    end
+
+    context 'given job failure' do
+      let(:source_file) { 'spec/fixtures/hasta/lib/failing_mapper.rb' }
+
+      it 'raises' do
+        expect {
+          subject.execute(source_file, data_source, data_sink)
+        }.to raise_error(Hasta::ExecutionError)
+      end
+    end
+  end
+end

data/spec/hasta/filter_spec.rb

@@ -0,0 +1,66 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/filter'
+
+describe Hasta::Filter do
+  context 'given a single regex' do
+    subject { described_class.new(regex) }
+
+    let(:regex) { /\A\d{1,}.*/ }
+    let(:drop_all) { ['First', 'Second', 'Third'] }
+    let(:drop_none) { ['1. First', '2. Second'] }
+    let(:drop_some) { ['1 First', 'Second', '3 Third'] }
+
+    it { expect(drop_all.select(&subject)).to be_empty }
+    it { expect(drop_none.select(&subject)).to eq(drop_none) }
+    it { expect(drop_some.select(&subject)).to eq(['1 First', '3 Third']) }
+  end
+
+  context 'given multiple regexes' do
+    subject { described_class.new(regex1, regex2) }
+
+    let(:regex1) { /\A\d{1,}.*/ }
+    let(:regex2) { /\A[A-Z].*/ }
+
+    let(:drop_all) { ['first', 'second', 'third'] }
+    let(:drop_none) { ['1. First', '2. Second', 'Third'] }
+    let(:drop_some) { ['First', 'second', '3 Third'] }
+
+    it { expect(drop_all.select(&subject)).to be_empty }
+    it { expect(drop_none.select(&subject)).to eq(drop_none) }
+    it { expect(drop_some.select(&subject)).to eq(['First', '3 Third']) }
+  end
+
+  context 'given multiple identical regexes' do
+    subject { described_class.new(regex1, regex1) }
+
+    let(:regex1) { /\A\d{1,}.*/ }
+    let(:single_regex_filter) { described_class.new(regex1) }
+
+    it { expect(subject.to_s).to eq(single_regex_filter.to_s) }
+  end
+
+  context 'given distinct regexes in a different order' do
+    subject { described_class.new(regex1, regex2) }
+
+    let(:regex1) { /\A\d{1,}.*/ }
+    let(:regex2) { /\A[A-Z].*/ }
+    let(:different_order_filter) { described_class.new(regex2, regex1) }
+
+    it { expect(subject.to_s).to eq(different_order_filter.to_s) }
+  end
+
+  describe '.from_file' do
+    let(:non_existent_file) { 'spec/fixtures/hasta/non_existent_file.dll' }
+
+    context 'given an non-existent file' do
+      it 'raises a ConfigurationError' do
+        expect {
+          described_class.from_file(non_existent_file)
+        }.to raise_error(Hasta::ConfigurationError)
+      end
+    end
+  end
+end

data/spec/hasta/filtered_s3_file_spec.rb

@@ -0,0 +1,45 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/filtered_s3_file'
+
+describe Hasta::FilteredS3File do
+  describe '#each_line' do
+    subject { described_class.new(s3_file, filter) }
+
+    let(:s3_file) { Hasta::S3File.new(fog_file) }
+    let(:fog_file) {
+      double('Fog::File',
+        :key => 'code/1986/HELLO.BAS',
+        :body => lines.join,
+        :etag => '696b61f4be8b11e383a37831c1ce6688'
+      )
+    }
+    let(:lines) {
+      [
+        "10 PRINT \"Hello World\"\n",
+        "20 GOTO 10\n",
+        "30 END\n",
+      ]
+    }
+
+    context 'given a filter that drops every line' do
+      let(:filter) { Hasta::Filter.new(/.^/) }
+      let(:exp_fingerprint) { '89a35b5f6745b5bab65cb918e9933df4' }
+
+      it { expect(subject.each_line.to_a).to be_empty }
+      it { expect(subject.body).to be_empty }
+      it { expect(subject.fingerprint).to eq(exp_fingerprint) }
+    end
+
+    context 'given a filter that drops some lines' do
+      let(:filter) { Hasta::Filter.new(/\A[1,3]/) }
+      let(:exp_fingerprint) { '46426cde7e24ab46ed95913c02623593' }
+
+      it { expect(subject.each_line.to_a).to eq(lines.values_at(0, 2)) }
+      it { expect(subject.body).to eq(lines.values_at(0, 2).join) }
+      it { expect(subject.fingerprint).to eq(exp_fingerprint) }
+    end
+  end
+end

data/spec/hasta/identity_mapper_spec.rb

@@ -0,0 +1,22 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/identity_mapper'
+require 'hasta/execution_context'
+require 'hasta/in_memory_data_sink'
+require 'hasta/in_memory_data_source'
+
+describe Hasta::IdentityMapper do
+  describe '#map' do
+    subject { described_class }
+
+    let(:sink) { Hasta::InMemoryDataSink.new }
+    let(:lines) { [["First\n"], ["Second\n", "Third\n"]] }
+    let(:exp_lines) { lines.flatten.map(&:rstrip).map { |line| "#{line}\t" } }
+    let(:sources) { lines.map { |source_lines| Hasta::InMemoryDataSource.new(source_lines) } }
+    let(:context) { double(Hasta::ExecutionContext) }
+
+    it { expect(subject.map(context, sources, sink).data_source.to_a).to eq(exp_lines) }
+  end
+end

data/spec/hasta/identity_reducer_spec.rb

@@ -0,0 +1,20 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/identity_reducer'
+require 'hasta/execution_context'
+
+describe Hasta::IdentityReducer do
+  describe '#reduce' do
+    subject { described_class }
+
+    let(:sink) { Hasta::InMemoryDataSink.new }
+    let(:lines) { ["Key1\tFirst\n", "Key2\tSecond\n", "Key3\tThird\n"] }
+    let(:exp_lines) { lines.map(&:rstrip) }
+    let(:source) { Hasta::InMemoryDataSource.new(lines) }
+    let(:context) { double(Hasta::ExecutionContext) }
+
+    it { expect(subject.reduce(context, source, sink).data_source.to_a).to eq(exp_lines) }
+  end
+end

data/spec/hasta/interpolate_string_spec.rb

@@ -0,0 +1,44 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/interpolate_string'
+
+describe Hasta::InterpolateString do
+  describe '#evaluate' do
+    subject { described_class.new(text) }
+
+    let(:context) { { 'scheduledStartTime' => Time.parse('2014-03-28T18:05:11Z') } }
+
+    context 'given static text' do
+      let(:text) { 'Static text' }
+
+      it { expect(subject.evaluate(context)).to eq(text) }
+    end
+
+    context 'given text with a known interpolate time expression' do
+      let(:text) {
+        's3://my-bucket/path/to/dir/#{format(@scheduledStartTime,\'YYYY-MM-dd_HHmmss\')}/files/'
+      }
+      let(:interpolated_text) { 's3://my-bucket/path/to/dir/2014-03-28_180511/files/' }
+
+      it { expect(subject.evaluate(context)).to eq(interpolated_text) }
+    end
+
+    context 'given text with a known interpolate date expression' do
+      let(:text) {
+        's3://my-bucket/path/to/dir/#{format(@scheduledStartTime,\'YYYY-MM-dd\')}/files/'
+      }
+      let(:interpolated_text) { 's3://my-bucket/path/to/dir/2014-03-28/files/' }
+
+      it { expect(subject.evaluate(context)).to eq(interpolated_text) }
+    end
+
+    context 'given text with an unknown interpolate expression' do
+      let(:time) { '#{format(minusMinutes(@scheduledStartTime,30),\'YYYY-MM-dd hh:mm:ss\')}' }
+      let(:text) { "s3://my-bucket/path/to/dir/#{time}/files/" }
+
+      it { expect(subject.evaluate(context)).to eq(text) }
+    end
+  end
+end

data/spec/hasta/local_file_path_spec.rb

@@ -0,0 +1,18 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/local_file_path'
+
+describe Hasta::LocalFilePath do
+  describe '.for' do
+    subject { described_class.for(s3_uri) }
+
+    let(:s3_uri) { Hasta::S3URI.new(bucket_name, path) }
+    let(:bucket_name) { 'my-bucket' }
+    let(:path) { 'path/to/my/file.txt' }
+    let(:exp_path) { File.expand_path("#{Hasta.local_storage_root}/#{bucket_name}/#{path}") }
+
+    it { expect(subject).to eq(exp_path) }
+  end
+end

data/spec/hasta/local_storage_spec.rb

@@ -0,0 +1,52 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/local_file_path'
+
+describe Hasta::LocalStorage do
+  subject { described_class.new(fog_storage) }
+
+  after do
+    FileUtils.rm_rf(tmpdir)
+  end
+
+  let(:fog_storage) {
+    Fog::Storage.new(
+      :provider => 'Local',
+      :local_root => tmpdir,
+      :endpoint => 'http://example.com'
+    )
+  }
+  let(:tmpdir) { Dir.mktmpdir('hasta_local_storage_test') }
+
+  describe '#files_for' do
+    it_should_behave_like 'a storage service'
+  end
+
+  describe '#write' do
+    let(:s3_uri) { Hasta::S3URI.new(bucket_name, path) }
+    let(:bucket_name) { 'my-bucket' }
+    let(:content) { "Hi\nBye\nWhy?\n" }
+    let(:data_source) { StringIO.new(content) }
+
+    let(:result) { subject.write(s3_uri, data_source) }
+    let(:local_file_path) { Hasta::LocalFilePath.for(result) }
+
+    context 'given a directory uri' do
+      let(:path) { 'path/to/files/' }
+      let(:expected_uri) { s3_uri.append('part-00000') }
+
+      it { expect(result).to eq(expected_uri) }
+      it { expect(File.read(local_file_path)).to eq(content) }
+    end
+
+    context 'given a file uri' do
+      let(:path) { 'path/to/files/file.txt' }
+      let(:expected_uri) { s3_uri }
+
+      it { expect(result).to eq(expected_uri) }
+      it { expect(File.read(local_file_path)).to eq(content) }
+    end
+  end
+end

data/spec/hasta/mapper_spec.rb

@@ -0,0 +1,26 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/mapper'
+
+describe Hasta::Mapper do
+  let(:mapper_file) { 'spec/fixtures/hasta/lib/test_identity_mapper.rb' }
+
+  describe '#map' do
+    subject { described_class.new(mapper_file) }
+
+    let(:input_source) { Hasta::InMemoryDataSource.new(input_lines, "Test Input") }
+    let(:input_sources) { [input_source] }
+    let(:input_lines) { ["Key1\tOne", "Key2\tTwo", "Key3\tThree", "Key4\tFour"] }
+    let(:output_lines) { input_lines }
+    let(:context) { Hasta::ExecutionContext.new }
+
+    let(:sink) { Hasta::InMemoryDataSink.new }
+
+    it 'writes all of the mapped lines to the sink' do
+      expect(subject.map(context, input_sources, sink)).to eq(sink)
+      expect(sink.data_source.to_a).to eq(input_lines)
+    end
+  end
+end

data/spec/hasta/reducer_spec.rb

@@ -0,0 +1,26 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/reducer'
+require 'hasta/in_memory_data_source'
+
+describe Hasta::Reducer do
+  describe '#reduce' do
+    subject { described_class.new(reducer_file) }
+
+    let(:reducer_file) { 'spec/fixtures/hasta/lib/unconventional_reducer.rb' }
+    let(:input_source) { Hasta::InMemoryDataSource.new(input_lines) }
+    let(:input_lines) { ["Small", "Medium", "Large"] }
+    let(:sorted_lines) { ["Large", "Medium", "Small"] }
+    let(:exp_lines) { sorted_lines.map { |line| "#{line}\t"} }
+    let(:context) { Hasta::ExecutionContext.new }
+
+    let(:sink) { Hasta::InMemoryDataSink.new }
+
+    it 'reducers over all of the input lines in sorted order' do
+      expect(subject.reduce(context, input_source, sink)).to eq(sink)
+      expect(sink.data_source.to_a).to eq(exp_lines)
+    end
+  end
+end

data/spec/hasta/resolved_cached_s3_file_spec.rb

@@ -0,0 +1,68 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+
+require 'hasta/resolve_cached_s3_file'
+
+describe Hasta::ResolveCachedS3File do
+  describe '#resolve' do
+    subject { described_class.new(file_cache, child_resolver) }
+
+    let(:file_cache) { double(Hasta::S3FileCache) }
+    let(:child_resolver) { Hasta::ResolveFilteredS3File.new(filter) }
+    let(:filter) { Hasta::Filter.new(/.*/) }
+
+    let(:fog_file) {
+      double('Fog::File',
+        :directory => fog_bucket,
+        :key => path,
+        :body => body
+      )
+    }
+    let(:fog_bucket) { double('Fog::Directory', :key => bucket_name) }
+    let(:bucket_name) { 'my-bucket' }
+    let(:path) { 'path/to/my/file.txt' }
+    let(:body) { "Parts\n" }
+    let(:s3_uri) { Hasta::S3URI.new(bucket_name, path) }
+
+    let(:exp_fingerprint) {
+      Digest::MD5.hexdigest("#{Digest::MD5.hexdigest(body)}_#{filter.to_s}")
+    }
+
+    let(:result) { subject.resolve(fog_file) }
+
+    context 'given the file is not cached' do
+      before do
+        file_cache.stub(:get).and_return(nil)
+      end
+
+      it 'caches the file' do
+        file_cache.should_receive(:put).with(exp_fingerprint, body)
+
+        expect(result.body).to eq(body)
+        expect(result.s3_uri).to eq(s3_uri)
+      end
+    end
+
+    context 'given the file is cached' do
+      before do
+        file_cache.stub(:get).with(exp_fingerprint).and_return(cached_file)
+      end
+
+      let(:cached_file) {
+        double('Fog::File',
+          :directory => double('Fog::Bucket', :key => 'cache_dir'),
+          :key => exp_fingerprint,
+          :body => body
+        )
+      }
+
+      it 'retrieves the cached file' do
+        file_cache.should_not_receive(:put)
+
+        expect(result.body).to eq(body)
+        expect(result.s3_uri).to eq(s3_uri)
+      end
+    end
+  end
+end