hasta 0.1.0

data/lib/hasta/s3_file_cache.rb

@@ -0,0 +1,23 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+module Hasta
+  # Caches data in a flat namespace using Fog storage
+  class S3FileCache
+    def initialize(fog_storage, bucket_name = 'cache')
+      directories = fog_storage.directories
+      @bucket = directories.get(bucket_name) || directories.create(:key => bucket_name)
+    end
+
+    def get(key)
+      bucket.files.get(key)
+    end
+
+    def put(key, data)
+      bucket.files.create(:key => key, :body => data)
+    end
+
+    private
+
+    attr_reader :bucket
+  end
+end

data/lib/hasta/s3_storage.rb

@@ -0,0 +1,21 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'fog'
+
+require 'hasta/s3_uri'
+require 'hasta/storage'
+
+module Hasta
+  # The read-only file storage interface to the actual S3 data used by the local map/reduce jobs
+  class S3Storage
+    include Storage
+
+    private
+
+    def fog_files(s3_bucket, s3_uri)
+      s3_bucket.files.all('prefix' => s3_uri.path).reject { |file|
+        file.key == s3_uri.path || (file_s3_uri(file).parent != s3_uri)
+      }
+    end
+  end
+end

data/lib/hasta/s3_uri.rb

@@ -0,0 +1,60 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+module Hasta
+  # Represents a URI to a file or directory on S3
+  class S3URI
+    attr_reader :bucket, :path
+
+    def self.parse(uri)
+      if match = /\As3n?:\/\/([^\/]+?)(\/.*)?\z/.match(uri)
+        canonical_path = match[2] && match[2][1..-1]
+        new(match[1], canonical_path)
+      else
+        raise ArgumentError, "Invalid S3 URI: #{uri}"
+      end
+    end
+
+    def initialize(bucket, path)
+      @bucket = bucket
+      @path = path
+    end
+
+    def directory?
+      path.end_with?('/')
+    end
+
+    def file?
+      !directory?
+    end
+
+    def basename
+      if path
+        path.split('/').last
+      else
+        ''
+      end
+    end
+
+    def parent
+      if path.nil?
+        nil
+      else
+        elements = path.split('/')
+        self.class.new(bucket, "#{elements.take(elements.length - 1).join('/')}/")
+      end
+    end
+
+    def append(append_path)
+      raise ArgumentError, "Cannot append to a file path: #{self}" if file?
+      self.class.new(bucket, File.join(path, append_path))
+    end
+
+    def ==(other)
+      self.class === other && (self.bucket == other.bucket && self.path == other.path)
+    end
+
+    def to_s
+      ["s3:/", bucket, path].compact.join('/')
+    end
+  end
+end

data/lib/hasta/sorted_data_source.rb

@@ -0,0 +1,36 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+module Hasta
+  # Decorator for a data source that yields the contents in sorted order
+  class SortedDataSource
+    def initialize(data_source)
+      @data_source = data_source
+    end
+
+    def name
+      data_source.name
+    end
+
+    def each_line
+      return enum_for(:each_line) unless block_given?
+
+      sorted_lines.each do |line|
+        yield line
+      end
+    end
+
+    def to_s
+      "#<#{self.class.name}:#{name} size=#{lines.count} lines>"
+    end
+
+    private
+
+    attr_reader :data_source
+
+    def sorted_lines
+      data_source.to_a.sort.tap do
+        Hasta.logger.debug "Finished sorting data for source: #{data_source}"
+      end
+    end
+  end
+end

data/lib/hasta/storage.rb

@@ -0,0 +1,82 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/s3_file'
+
+module Hasta
+  # Common file storage methods used by the local and S3 storage providers
+  module Storage
+    # Creates the appropriate Hasta S3 file instance given a Fog file
+    module ResolveS3File
+      def self.resolve(fog_file)
+        S3File.wrap(fog_file)
+      end
+    end
+
+    def initialize(fog_storage, s3_file_resolver = ResolveS3File)
+      @fog_storage = fog_storage
+      @s3_file_resolver = s3_file_resolver
+    end
+
+    def exists?(s3_uri)
+      if s3_uri.file?
+        !!fog_file(s3_uri)
+      elsif s3_bucket = bucket(s3_uri)
+        !fog_files(s3_bucket, s3_uri).empty?
+      end
+    end
+
+    def files_for(s3_uri)
+      if s3_uri.file?
+        [s3_file!(s3_uri)]
+      else
+        s3_files(bucket!(s3_uri), s3_uri)
+      end
+    end
+
+    private
+
+    attr_reader :fog_storage, :s3_file_resolver
+
+    def bucket(s3_uri)
+      fog_storage.directories.get(s3_uri.bucket)
+    end
+
+    def s3_file!(s3_uri)
+      s3_file_resolver.resolve(fog_file!(s3_uri))
+    end
+
+    def s3_files(bucket, s3_uri)
+      fog_files(bucket, s3_uri).map { |fog_file| s3_file_resolver.resolve(fog_file) }
+    end
+
+    def fog_file(s3_uri)
+      (s3_bucket = bucket(s3_uri)) && s3_bucket.files.get(s3_uri.path)
+    end
+
+    def bucket!(s3_uri)
+      bang!(s3_uri) { bucket(s3_uri) }
+    end
+
+    def fog_file!(s3_uri)
+      bang!(s3_uri) { fog_file(s3_uri) }
+    end
+
+    def create_bucket(bucket_name)
+      fog_storage.directories.create(:key => bucket_name)
+    end
+
+    def create_file(bucket, key, content)
+      bucket.files.create(:body => content, :key => key)
+    end
+
+    def file_s3_uri(file, path=file.key)
+      S3URI.new(file.directory.key, path)
+    end
+
+    def bang!(s3_uri)
+      yield.tap do |result|
+        raise NonExistentPath.new(s3_uri) unless result
+      end
+    end
+  end
+end

data/lib/hasta/tasks.rb

@@ -0,0 +1,8 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/tasks/runner'
+
+module Hasta
+  module Tasks
+  end
+end

data/lib/hasta/tasks/runner.rb

@@ -0,0 +1,84 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+
+require 'rake'
+require 'rake/tasklib'
+require 'hasta'
+
+require 'hasta/emr_job_definition'
+require 'hasta/runner'
+
+module Hasta
+  module Tasks
+    # Rakes task that runs a local test of an EMR job
+    class Runner < ::Rake::TaskLib
+      include ::Rake::DSL if defined?(::Rake::DSL)
+
+      # Name of task.
+      #
+      # default:
+      #   :runner
+      attr_accessor :name
+
+      # Path to the AWS Data Pipeline definition file
+      attr_accessor :definition_file
+
+      # The Scheduled Start Time to use when evaluating the definition
+      #
+      # default:
+      #  Time.now
+      attr_accessor :scheduled_start_time
+
+      # The id of the EMR job to perform
+      attr_accessor :job_id
+
+      # The root directory of the project containing the EMR code that is being tested
+      attr_accessor :project_root
+
+      # Use verbose output. If this is set to true, the task will print the
+      # local and remote paths of each step file it uploads to S3.
+      #
+      # default:
+      #   true
+      attr_accessor :verbose
+
+      def initialize(*args, &task_block)
+        setup_ivars(args)
+
+        desc "Runs the specified EMR job"
+        task name, [:job_id, :scheduled_start_time] do |_, task_args|
+          RakeFileUtils.send(:verbose, verbose) do
+            if task_block
+              task_block.call(*[self, task_args].slice(0, task_block.arity))
+            end
+
+            run_task verbose
+          end
+        end
+      end
+
+      def setup_ivars(args)
+        @name = args.shift || :runner
+        @verbose = true
+        @path = "definitions"
+        @scheduled_start_time = Time.now
+      end
+
+      def run_task(verbose)
+        Hasta.configure do |config|
+          config.project_root = project_root
+        end
+
+        definition = Hasta::EmrJobDefinition.load(definition_file, job_id, scheduled_start_time)
+        runner = Hasta::Runner.new(definition.id, definition.mapper, definition.reducer)
+
+        result = runner.run(
+          definition.data_sources,
+          definition.data_sink,
+          definition.ruby_files,
+          definition.env
+        )
+      end
+    end
+  end
+end

data/lib/hasta/version.rb

@@ -0,0 +1,3 @@
+module Hasta
+  VERSION = "0.1.0"
+end

data/spec/fixtures/hasta/filter_config.txt

@@ -0,0 +1 @@
+\AOnly Allow This Line\z

data/spec/fixtures/hasta/json/emr_node.json

@@ -0,0 +1,10 @@
+{
+  "id": "EMRJob1",
+  "type": "EmrActivity",
+  "onFail": { "ref": "FailureNotify" },
+  "schedule": { "ref": "Nightly" },
+  "runsOn": { "ref": "MenuIntelEMRCluster" },
+  "step": "/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input1/,-output,s3://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output/,-mapper,cat,-reducer,s3n://steps-bucket/path/to/reducer.rb,-cacheFile,s3://data-bucket/path/to/mappings.yml#mappings.yml,-cacheFile,s3://data-bucket/path/to/ignored.yml#ignored.yml,-cmdenv,API_KEY=123456,-cmdenv,ENVIRONMENT_NAME=uat",
+  "input": [ { "ref": "S3Input1" }, { "ref": "S3Input2" }, { "ref": "S3Input3" } ],
+  "output": { "ref": "S3Output" }
+}

data/spec/fixtures/hasta/json/pipeline_definition.json

@@ -0,0 +1,135 @@
+{
+  "objects": [
+    {
+      "id": "Default",
+      "role": "role-aggregator",
+      "resourceRole": "role-aggregator",
+      "failureAndRerunMode": "cascade",
+      "scheduleType": "cron",
+      "onFail": { "ref": "FailureNotify" }
+    },
+
+    {
+      "id": "Nightly",
+      "type": "Schedule",
+      "startDateTime": "2014-03-26T18:00:00",
+      "period": "12 hours"
+    },
+
+    {
+      "id": "SuccessNotify",
+      "type": "SnsAlarm",
+      "topicArn": "arn:aws:sns:us-east-1:999999999999:datapipeline-name",
+      "subject": "SUCCESS: pipeline step #{node.name}",
+      "message": "pipeline step SUCCESS\n\nScheduled start: #{node.@scheduledStartTime}\nActual start: #{node.@actualStartTime}\nActual end:\n#{node.@actualEndTime}"
+    },
+
+    {
+      "id": "FailureNotify",
+      "type": "SnsAlarm",
+      "topicArn": "arn:aws:sns:us-east-1:999999999999:datapipeline-name",
+      "subject": "FAILURE: pipeline step #{node.name}",
+      "message": "pipeline step FAILED #{node.name}\n\nScheduled start: #{node.@scheduledStartTime}\nError message:\n#{node.errorMessage}\nError stack trace:\n#{node.errorStackTrace}"
+    },
+
+    {
+      "id": "S3FirstInput",
+      "type": "S3DataNode",
+      "schedule": { "ref": "Nightly" },
+      "directoryPath": "s3://data-bucket/path/to/data/dir1/"
+    },
+
+    {
+      "id": "S3SecondInput",
+      "type": "S3DataNode",
+      "schedule": { "ref": "Nightly" },
+      "directoryPath": "s3://data-bucket/path/to/data/dir2/"
+    },
+
+    {
+      "id": "S3ThirdInput",
+      "type": "S3DataNode",
+      "schedule": { "ref": "Nightly" },
+      "filePath": "s3://datapipeline-assets/path/to/data/file.csv"
+    },
+
+    {
+      "id": "S3Output",
+      "type": "S3DataNode",
+      "schedule": { "ref": "Nightly" },
+      "directoryPath": "s3://data-bucket/path/to_data/results/"
+    },
+
+    {
+      "id": "TestPipelineEC2Resource",
+      "type": "Ec2Resource",
+      "instanceType": "m1.large",
+      "schedule": { "ref": "Nightly" },
+      "logUri": "s3://logs-bucket/path/to/test_pipeline/logs/TestPipelineEC2Resource",
+      "terminateAfter": "6 hours",
+      "keyPair": "pipeline-debug",
+      "securityGroups": ["s-pipe-appdb"]
+    },
+
+    {
+      "id": "TestPipelineEMRCluster",
+      "type": "EmrCluster",
+      "masterInstanceType": "m1.large",
+      "taskInstanceType": "m1.large",
+      "coreInstanceType": "m1.large",
+      "coreInstanceCount": "2",
+      "terminateAfter": "1 hour",
+      "schedule": { "ref": "Nightly" },
+      "enableDebugging": "true",
+      "bootstrapAction": "s3://steps-bucket/test_pipeline/steps/bootstrap_emr.sh",
+      "emrLogUri": "s3://logs-bucket/path/to/test_pipeline/logs/RecentSalesPipelineEMRLogs",
+      "logUri": "s3://logs-bucket/path/to/test_pipeline/logs/TestPipelineEMRCluster"
+    },
+
+    {
+      "id": "BootstrapEnvironment",
+      "type": "ShellCommandActivity",
+      "onFail": { "ref": "FailureNotify" },
+      "stdout": "s3://logs-bucket/path/to/test_pipeline/logs/BootstrapEnvironment/stdout",
+      "stderr": "s3://logs-bucket/path/to/test_pipeline/logs/BootstrapEnvironment/stderr",
+      "schedule": { "ref": "Nightly" },
+      "runsOn": { "ref": "TestPipelineEC2Resource" },
+      "scriptUri": "s3://steps-bucket/test_pipeline/steps/bootstrap_ec2.sh"
+    },
+
+    {
+      "id": "TouchEMRCluster",
+      "type": "ShellCommandActivity",
+      "onFail": { "ref": "FailureNotify" },
+      "schedule": { "ref": "Nightly" },
+      "runsOn": { "ref": "TestPipelineEMRCluster" },
+      "command": "true"
+    },
+
+    {
+      "id": "EMRJob1",
+      "type": "EmrActivity",
+      "onFail": { "ref": "FailureNotify" },
+      "schedule": { "ref": "Nightly" },
+      "runsOn": { "ref": "TestPipelineEMRCluster" },
+      "step": "/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3://data-bucket/path/to/data/dir1/,-input,s3://data-bucket/path/to/data/dir2/,-input,s3://data-bucket/path/to/data/file.csv,-output,s3://data-bucket/path/to_data/results/,-mapper,s3n://steps-bucket/test_pipeline/steps/mapper.rb,-reducer,s3n://steps-bucket/test_pipeline/steps/reducer.rb,-cacheFile,s3n://steps-bucket/test_pipeline/steps/types.rb#types.rb,-cacheFile,s3://data-bucket/path/to/data/notes.yml#notes.yml,-cmdenv,API_KEY=123456",
+      "input": [ { "ref": "S3FirstInput" }, { "ref": "S3SecondInput" }, { "ref": "S3ThirdInput" } ],
+      "output": { "ref": "S3Output" }
+    },
+
+    {
+      "id": "Complete",
+      "type": "ShellCommandActivity",
+      "onFail": { "ref": "FailureNotify" },
+      "schedule": { "ref": "Nightly" },
+      "onSuccess": { "ref": "SuccessNotify" },
+      "runsOn": { "ref": "TestPipelineEC2Resource" },
+      "dependsOn": [
+        { "ref": "TouchEMRCluster" },
+        { "ref": "EMRJob1" }
+      ],
+      "command": "true"
+    }
+
+  ]
+}