pyper_rb 1.2.0

data/lib/pyper/pipes/field_rename.rb

@@ -0,0 +1,23 @@
+module Pyper::Pipes
+  # @param attr_map [Hash] A map of old field names to new field names, which will be used to rename attributes.
+  class FieldRename < Struct.new(:attr_map)
+
+    # @param args [Hash|Enumerator<Hash>] One or more item hashes
+    # @param status [Hash] The mutable status field
+    # @return [Hash|Enumerator<Hash>] The item(s) with fields renamed
+    def pipe(attrs_or_items, status = {})
+      case attrs_or_items
+      when Hash then rename(attrs_or_items)
+      else attrs_or_items.map { |item| rename(item) }
+      end
+    end
+
+    def rename(item)
+      attr_map.each do |old,new|
+        item[new.to_sym] = item.delete(old.to_sym) if item.has_key?(old.to_sym)
+        item[new.to_s] = item.delete(old.to_s) if item.has_key?(old.to_s)
+      end
+      item
+    end
+  end
+end

data/lib/pyper/pipes/force_enumerator.rb

@@ -0,0 +1,13 @@
+module Pyper::Pipes::Model
+  # Typically at the end of a pipeline, makes sure any lazy computations on the items are evaluated.
+  # Returning a lazy enumerator can be unexpected by the consumer, and may cause the enumerator to
+  # be evaluated more than once with unexpected results.
+  class ForceEnumerator
+    # @param items [Enumerable::Lazy<Hash>] A list of items
+    # @param status [Hash] The mutable status field
+    # @return [Enumerable<Hash>] A list of items, deserialized according to the type mapping
+    def self.pipe(items, status = {})
+      items.force
+    end
+  end
+end

data/lib/pyper/pipes/model/attribute_deserializer.rb

@@ -0,0 +1,27 @@
+require 'json'
+
+module Pyper::Pipes::Model
+  # @param type_mapping [Hash<Symbol, Class>] A map from field names to types. fields will be deserialized according
+  # to these types.
+  class AttributeDeserializer < Struct.new(:type_mapping)
+    # @param items [Enumerable<Hash>] A list of items
+    # @param status [Hash] The mutable status field
+    # @return [Enumerable<Hash>] A list of items, deserialized according to the type mapping
+    def pipe(items, status = {})
+      items.map do |item|
+        new_item = item.dup
+        type_mapping.each do |field, type|
+          new_item[field] = deserialize(new_item[field], type) if new_item[field]
+        end
+        new_item
+      end
+    end
+
+    def deserialize(value, type)
+      if (type == Array) || (type == Hash) then JSON.parse(value)
+      elsif (type == Integer) then value.to_i
+      elsif (type == Float) then value.to_f
+      else value end
+    end
+  end
+end

data/lib/pyper/pipes/model/attribute_serializer.rb

@@ -0,0 +1,34 @@
+require 'json'
+
+module Pyper::Pipes::Model
+  # Provides a way to serialize attributes to JSON.
+  class AttributeSerializer
+
+    # @param attributes [Hash] Unserialized attributes
+    # @param status [Hash] The mutable status field
+    # @return [Hash] The serialized attributes
+    def pipe(attributes, status = {})
+      attributes.each_with_object({}) do |attr, serialized_attrs|
+        value = force_encode_to_UTF8(attr.last)
+        serialized_attrs[attr.first] = case value
+                                       when Array, Hash then JSON.generate(value)
+                                       when DateTime then value.to_time
+                                       else value
+                                       end
+      end
+    end
+
+    def force_encode_to_UTF8(value)
+      case value
+      when Array
+        value.map { |v| force_encode_to_UTF8(v) }
+      when Hash
+        Hash[value.map { |k,v| [k, force_encode_to_UTF8(v)] }]
+      when String
+        value.dup.force_encoding('UTF-8')
+      else
+        value
+      end
+    end
+  end
+end

data/lib/pyper/pipes/model/attribute_validation.rb

@@ -0,0 +1,57 @@
+module Pyper::Pipes::Model
+  class AttributeValidation
+
+    # Raised when validation fails.
+    class Failure < ::StandardError; end
+
+    # Array of attributes that are allowed to be set. If empty, all attributes
+    # are allowed.
+    attr_reader :allowed
+
+    # Array of attributes that are required to be present (non-nil).
+    attr_reader :required
+
+    # Hash of attributes whose value must be restricted in some way.
+    # Format :attribute => lambda { |value| #Return boolean indicating pass/fail }
+    attr_reader :restricted
+
+    # @param opts [Hash] Options defining how attributes should be validated.
+    # @option opts [Array<Symbol>] :allowed A list of attributes that are allowed
+    #   to be set. If empty, all attributes are assumed to be allowed.
+    # @option opts [Array<Symbol>] :required A list of attributes that are required
+    #   to be present (non-nil).
+    # @option opts [Hash] :restricted A Hash of attributes whose value must be
+    #   restricted in some way.
+    #   Format :attribute => lambda { |value| #Return boolean indicating pass/fail }
+    def initialize(opts={})
+      @allowed = opts[:allowed] if opts[:allowed]
+      @required = opts[:required] if opts[:required]
+      @restricted = opts[:restricted]
+    end
+
+    # @param attributes [Hash] The un-validated attributes
+    # @param status [Hash] The mutable status field
+    # @return [Hash] The original attributes
+    def pipe(attributes, status = {})
+      if allowed.present?
+        attributes.keys.each do |attr|
+          raise Failure.new("Attribute #{attr} is not allowed.") unless allowed.include?(attr)
+        end
+      end
+
+      if required.present?
+        required.each do |attr|
+          raise Failure.new("Missing required attribute #{attr}.") if attributes[attr].nil?
+        end
+      end
+
+      if restricted.present?
+        restricted.each do |attr, test|
+          raise Failure.new("Invalid value for attribute #{attr}.") unless test.call(attributes[attr])
+        end
+      end
+
+      attributes
+    end
+  end
+end

data/lib/pyper/pipes/model/virtus_deserializer.rb

@@ -0,0 +1,39 @@
+require 'json'
+
+module Pyper::Pipes::Model
+  # Provides a way to deserialize serialized fields from an item. This is intended to be used with a Virtus
+  # model class, and will use the attribute names and type information from that model to determine how to
+  # deserialize.
+  #
+  # All serialization is as JSON.
+  class VirtusDeserializer
+
+    attr_reader :type_mapping
+
+    # @param attribute_set [Virtus::AttributeSet] A Virtus AttributeSet
+    def initialize(attribute_set)
+      @type_mapping = Hash[attribute_set.map { |attr| [attr.name.to_s, attr.type.primitive] }]
+    end
+
+    # @param items [Enumerator<Hash>] A list of items
+    # @param status [Hash] The mutable status field
+    # @return [Enumerator<Hash>] A list of items, deserialized according to the type mapping
+    def pipe(items, status = {})
+      items.map do |item|
+        new_item = item.dup
+        type_mapping.each do |field, type|
+          new_item[field] = deserialize(new_item[field], type) if new_item[field]
+        end
+
+        new_item
+      end
+    end
+
+    def deserialize(value, type)
+      if type == Array || type == Hash then JSON.parse(value)
+      elsif type == Integer then value.to_i
+      elsif type == Float then value.to_f
+      else value end
+    end
+  end
+end

data/lib/pyper/pipes/model/virtus_parser.rb

@@ -0,0 +1,13 @@
+module Pyper::Pipes::Model
+  # Transform a series of items into model classes (based on Virtus model objects)
+  # @param virtus_model_class [Class] the model class to instantiate. Should respond to `new(item_attributes)`
+  class VirtusParser < Struct.new(:virtus_model_class)
+
+    # @param items [Enumerable<Hash>]
+    # @param status [Hash] The mutable status field
+    # @return [Enumerable<Hash>] The unchanged list of items
+    def pipe(items, status = {})
+      items.map { |item| virtus_model_class.new(item) }
+    end
+  end
+end

data/lib/pyper/pipes/model.rb

@@ -0,0 +1,5 @@
+require_relative 'model/attribute_serializer'
+require_relative 'model/attribute_validation'
+require_relative 'model/attribute_deserializer'
+require_relative 'model/virtus_deserializer'
+require_relative 'model/virtus_parser'

data/lib/pyper/pipes/no_op.rb

@@ -0,0 +1,15 @@
+module Pyper::Pipes
+  # This pipe performs no operation. Usful if want to potentially skip a pipe in
+  # in the pipeline.
+  # @example
+  #   (some condition) ? some_pipe(conent) : Pyper::Pipes::NoOp
+  class NoOp
+
+    # @param attrs_or_items [Object]
+    # @param status [Hash] The mutable status field
+    # @return [Object]
+    def self.pipe(attrs_or_items, status = {})
+      attrs_or_items
+    end
+  end
+end

data/lib/pyper/pipes/pry.rb

@@ -0,0 +1,9 @@
+module Pyper::Pipes
+  # For debugging purposes
+  class Pry
+    def pipe(items, status)
+      binding.pry
+      items
+    end
+  end
+end

data/lib/pyper/pipes/remove_fields.rb

@@ -0,0 +1,22 @@
+module Pyper::Pipes
+  # A generic pipe to remove fields from a pipeline
+  class RemoveFields
+
+    attr_reader :fields_to_remove
+
+    # @param fields_to_remove [Array] fields to be removed from pipe
+    def initialize(fields_to_remove)
+      @fields_to_remove = Array.wrap(fields_to_remove)
+    end
+
+    # @param attributes [Hash] The attributes from which to remove the specified fields
+    # @param status [Hash] The mutable status field
+    # @return [Hash] attributes with the specified fields removed
+    def pipe(attributes, status = {})
+      attributes = attributes.dup
+      fields_to_remove.each { |field| attributes.delete(field) }
+
+      attributes
+    end
+  end
+end

data/lib/pyper/pipes.rb

@@ -0,0 +1,8 @@
+module Pyper::Pipes; end
+
+# require the individual generic pipes
+require_relative 'pipes/field_rename'
+require_relative 'pipes/default_values'
+require_relative 'pipes/no_op'
+require_relative 'pipes/remove_fields'
+require_relative 'pipes/force_enumerator'

data/lib/pyper/version.rb

@@ -0,0 +1,3 @@
+module Pyper
+  VERSION = "1.2.0"
+end

data/lib/pyper.rb

@@ -0,0 +1,4 @@
+require "pyper/version"
+require "pyper/pipeline"
+
+module Pyper; end

data/pyper_rb.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pyper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pyper_rb"
+  spec.version       = Pyper::VERSION
+  spec.authors       = ["Arron Norwell"]
+  spec.email         = ["anorwell@datto.com"]
+  spec.summary       = %q{Create pipelines for storing data.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/test/fixtures/cass_schema_config.yml

@@ -0,0 +1,6 @@
+datastores:
+  test_datastore:
+    hosts: 127.0.0.1
+    port: 9242
+    keyspace: test_keyspace
+    replication: "{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }"

data/test/fixtures/test_datastore/schema.cql

@@ -0,0 +1,23 @@
+# CREATE KEYSPACE test_keyspace with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }
+
+CREATE TABLE test(
+  id text,
+  a text,
+  b text,
+  PRIMARY KEY((id), a)
+  );
+
+CREATE TABLE test2(
+  id text,
+  x text,
+  y text,
+  PRIMARY KEY((id), x)
+  );
+
+CREATE TABLE mod_test(
+ row int,
+ id text,
+ mod_key int,
+ val text,
+ PRIMARY KEY ((row, mod_key), id)
+ );

data/test/test_helper.rb

@@ -0,0 +1,34 @@
+require 'rubygems'
+require 'pry'
+require 'minitest/autorun'
+require 'minitest/should'
+
+class Minitest::Should::TestCase
+  def self.xshould(*args)
+    puts "Disabled test: #{args}"
+  end
+end
+
+def setup_cass_schema
+  base_path = "#{File.dirname(__FILE__)}/fixtures"
+  ::CassSchema::Runner.datastores = ::CassSchema::YamlHelper.datastores(File.join(base_path, 'cass_schema_config.yml'))
+  ::CassSchema::Runner.schema_base_path = base_path
+  ::CassSchema::Runner.create_all
+end
+
+def teardown_cass_schema
+  ::CassSchema::Runner.drop_all
+end
+
+def create_cass_client(datastore_name)
+  keyspace = ::CassSchema::Runner.datastore_lookup(datastore_name).keyspace
+  c = Cassandra.cluster(port: 9242)
+  session = c.connect(keyspace)
+  Cassava::Client.new(session)
+end
+
+
+require 'pyper/all'
+require 'storage_strategy'
+require 'cass_schema'
+require 'cassava'

data/test/unit/pyper/pipeline_test.rb

@@ -0,0 +1,81 @@
+require 'test_helper'
+
+module Pyper
+  class PipelineTest < Minitest::Should::TestCase
+    context '::create' do
+      context 'with a block' do
+        should 'yield the instance to the block' do
+          el = 'hello'
+
+          pl = Pyper::Pipeline.create do
+            add el
+          end
+
+          assert pl.pipes.include? el
+        end
+
+        should 'allow access to externally defined methods' do
+          def external
+            'external'
+          end
+
+          pl = Pyper::Pipeline.create do
+            add external
+          end
+
+          assert pl.pipes.include? external
+        end
+
+        should 'return a new pipeline' do
+          pl = Pyper::Pipeline.create do
+            add 'hello'
+          end
+
+          assert pl.is_a? Pyper::Pipeline
+        end
+      end
+
+      context 'without a block' do
+        should 'return a new pipeline' do
+          pl = Pyper::Pipeline.create
+
+          assert pl.is_a? Pyper::Pipeline
+        end
+      end
+    end
+
+    context '#add' do
+      should 'add the pipe to the pipes' do
+        pl = Pyper::Pipeline.new
+        el = 'hello'
+        pl.add el
+
+        assert pl.pipes.include? el
+      end
+    end
+
+    context '#push' do
+      should 'accept pipes that respond to #pipe' do
+        pl = Pyper::Pipeline.create do
+          add ExamplePipe
+        end
+        res = pl.push({})
+        assert res.status[:called]
+      end
+
+      should 'accept pipes that respond to #call' do
+        pl = Pyper::Pipeline.create do
+          add ->(attrs, status = {}) { status[:called] = true  }
+        end
+        res = pl.push({})
+        assert res.status[:called]
+      end
+    end
+
+    class ExamplePipe
+      def self.pipe(attrs, status = {})
+        status[:called] = true
+      end
+    end
+  end
+end

data/test/unit/pyper/pipes/cassandra/all_items_reader_test.rb

@@ -0,0 +1,47 @@
+require 'test_helper'
+
+module Pyper::Pipes::Cassandra
+  class AllItemsReaderTest < Minitest::Should::TestCase
+    context 'a cassandra paging_reader pipe' do
+      setup do
+        setup_cass_schema
+
+        @client = create_cass_client('test_datastore')
+        @pipe = AllItemsReader.new(:test, @client)
+
+        # populate some test data
+        @client.insert(:test, {id: 'ida', a: '1', b: '2'})
+        @client.insert(:test, {id: 'ida', a: '2', b: '3'})
+        @client.insert(:test, {id: 'ida', a: '3', b: '4'})
+        @client.insert(:test, {id: 'idb', a: '1', b: '2'})
+        @client.insert(:test, {id: 'idb', a: '2', b: '3'})
+      end
+
+      teardown do
+        teardown_cass_schema
+      end
+
+      should 'page through cassandra result and return all items' do
+        @client.insert(:test, {id: 'ida', a: '4', b: '4'})
+
+        all_items = AllItemsReader.new(:test, @client)
+        result = all_items.pipe(id: 'ida').to_a
+        assert_equal 4, result.size
+        assert_equal %w(1 2 3 4).to_set, result.map { |i| i['a'] }.to_set
+      end
+
+      context "columns selecting" do
+        should "only select given columns from columns argument" do
+          out = @pipe.pipe({id: 'idb', :columns => [:a]}).to_a
+          assert_equal({'a' => '1'}, out.first)
+          assert_equal({'a' => '2'}, out.last)
+        end
+
+        should  "select all columns when columns argument not given" do
+          out = @pipe.pipe({id: 'idb'}).to_a
+          assert_equal({ "id" => "idb", "a" => "1", "b" => "2" }, out.first)
+        end
+      end
+    end
+  end
+end

data/test/unit/pyper/pipes/cassandra/deleter_test.rb

@@ -0,0 +1,37 @@
+require 'test_helper'
+
+module Pyper::Pipes::Cassandra
+  class DeleterTest < Minitest::Should::TestCase
+    context 'a cassandra delete pipe' do
+      setup do
+        setup_cass_schema
+
+        @client = create_cass_client('test_datastore')
+        @deleter = Deleter.new(:test, @client)
+        @writer = Writer.new(:test, @client)
+      end
+
+      teardown do
+        teardown_cass_schema
+      end
+
+      should 'delete from the specified table' do
+        attributes = {id: 'id', a: 'a', b: 'b'}
+        @writer.pipe(attributes)
+        assert @client.select(:test).execute.first
+
+        @deleter.pipe({:id => 'id'})
+        refute @client.select(:test).execute.first
+      end
+
+      should 'delete certain columns' do
+        attributes = {id: 'id', a: 'a', b: 'b'}
+        @writer.pipe(attributes)
+        assert_equal 'b', @client.select(:test).execute.first['b']
+
+        @deleter.pipe({:id => 'id', :a => 'a', :columns => [:b]})
+        refute @client.select(:test).execute.first['b']
+      end
+    end
+  end
+end

data/test/unit/pyper/pipes/cassandra/mod_key_reader_test.rb

@@ -0,0 +1,47 @@
+require 'test_helper'
+
+module Pyper::Pipes::Cassandra
+  class ModKeyReaderTest < Minitest::Should::TestCase
+    context 'a cassandra reader pipe' do
+
+      setup do
+        setup_cass_schema
+
+        @client = create_cass_client('test_datastore')
+        @pipe = ModKeyReader.new(:mod_test, @client, mod_size = 4)
+
+        # populate some test data
+        @client.insert(:mod_test, {row: 1, id: 'ida', mod_key: 0, val: '1'})
+        @client.insert(:mod_test, {row: 1, id: 'idb', mod_key: 1, val: '2'})
+        @client.insert(:mod_test, {row: 1, id: 'idc', mod_key: 2, val: '3'})
+      end
+
+      teardown do
+        teardown_cass_schema
+      end
+
+      should 'return all items in an unordered enumerator from cassandra' do
+        result = @pipe.pipe(row: 1).to_a
+        assert_equal 3, result.size
+        assert_equal %w(1 2 3).to_set, result.map { |i| i['val'] }.to_set
+      end
+
+      should 'page through cassandra result and return all items' do
+        @client.insert(:mod_test, {row: 1, id: 'idd', mod_key: 0, val: '4'})
+        @client.insert(:mod_test, {row: 1, id: 'ide', mod_key: 1, val: '5'})
+        @client.insert(:mod_test, {row: 1, id: 'idf', mod_key: 2, val: '6'})
+
+        paging_pipe = ModKeyReader.new(:mod_test, @client, mod_size = 4, page_size = 1)
+        result = paging_pipe.pipe(row: 1).to_a
+        assert_equal 6, result.size
+        assert_equal %w(1 2 3 4 5 6).to_set, result.map { |i| i['val'] }.to_set
+      end
+
+      should 'not return items with a mod key outside the mod key range' do
+        @client.insert(:mod_test, {row: 1, id: 'idz', mod_key: 4, val: '1'})
+        result = @pipe.pipe(row: 1).to_a
+        assert_equal 3, result.size
+      end
+    end
+  end
+end

data/test/unit/pyper/pipes/cassandra/pagination_decoding_test.rb

@@ -0,0 +1,29 @@
+require 'test_helper'
+
+module Pyper::Pipes::Cassandra
+  class PaginationDecodingTest < Minitest::Should::TestCase
+    setup do
+      @pipe = PaginationDecoding.new
+    end
+
+    should 'decode the :paging_state argument' do
+      decoded = 'sdf'
+      encoded = Base64.urlsafe_encode64(decoded)
+
+      args = {paging_state: encoded}
+      new_attrs = @pipe.pipe(args)
+      assert_equal decoded, new_attrs[:paging_state]
+    end
+
+    should 'allow missing paging states' do
+      args = {}
+      new_args = @pipe.pipe(args)
+      assert_equal nil, new_args[:paging_state]
+    end
+
+    should 'not modify other args' do
+      args = {other: 1}
+      assert_equal args, @pipe.pipe(args)
+    end
+  end
+end

data/test/unit/pyper/pipes/cassandra/pagination_encoding_test.rb

@@ -0,0 +1,29 @@
+require 'test_helper'
+
+module Pyper::Pipes::Cassandra
+  class PaginationEncodingTest < Minitest::Should::TestCase
+    setup do
+      @pipe = PaginationEncoding.new
+    end
+
+    should 'encode the :paging_state status' do
+      state = 'sdf'
+      encoded = Base64.urlsafe_encode64(state)
+
+      status = {paging_state: state}
+      @pipe.pipe([], status)
+      assert_equal encoded, status[:paging_state]
+    end
+
+    should 'allow missing paging states' do
+      status = {}
+      @pipe.pipe([], status)
+      assert_equal nil, status[:paging_state]
+    end
+
+    should 'not modify the items' do
+      items = %w(a b)
+      assert_equal items, @pipe.pipe(items, {})
+    end
+  end
+end