piah 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fe582fcfd679d1c51dcb5a7327fbeee7359b800e
+  data.tar.gz: 3905b220ab0c6dcc79d87910f3b1c7d966f33629
+SHA512:
+  metadata.gz: c4089dde4306f8494ed6166632060801aa4a0903b922fe166908ecae0c7139f9327c3dbe582f7cbe8f825985f4c4dc922f609332da55c93463aaf41f58d166c9
+  data.tar.gz: 7e520595425f48ca64406b0100869c472851c879341682d69536beefc7d81ae93682e6c652d40bee3d3c702c473e5988bb14bb941bb75b85e7c28cbfba6576e8

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at matt.edlefsen@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,10 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in piah.gemspec
+#ruby=ruby-2.4
+ruby '2.4.1'
+gemspec
+
+gem 'rake'
+gem 'alki-console'
+gem 'alki-testing'

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Matt Edlefsen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.adoc

@@ -0,0 +1,64 @@
+[[piah]]
+piah
+----
+
+Piah is a library for constructing large data transformations out
+of smaller individual processes. When given input data, and a set of
+of processors, piah will automatically order and connect the processors
+into a pipeline to produce output.
+
+[[installation]]
+Installation
+~~~~~~~~~~~~
+
+Add this line to your application's Gemfile:
+
+[source,ruby]
+----
+gem 'piah'
+----
+
+And then execute:
+
+....
+$ bundle
+....
+
+Or install it yourself as:
+
+....
+$ gem install piah
+....
+
+[[usage]]
+Usage
+~~~~~
+
+TODO: Write usage instructions here
+
+
+[[contributing]]
+Contributing
+~~~~~~~~~~~~
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/alki-project/piah. This project is intended to be a
+safe, welcoming space for collaboration, and contributors are expected
+to adhere to the http://contributor-covenant.org[Contributor Covenant]
+code of conduct.
+
+[[license]]
+License
+~~~~~~~
+
+The gem is available as open source under the terms of the
+http://opensource.org/licenses/MIT[MIT License].
+
+[[code-of-conduct]]
+Code of Conduct
+~~~~~~~~~~~~~~~
+
+Everyone interacting in the Piah project’s codebases, issue trackers,
+chat rooms and mailing lists is expected to follow the
+https://github.com/alki-project/piah/blob/master/CODE_OF_CONDUCT.md[code
+of conduct].

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+require 'alki/testing/tasks'
+task :default => :test

data/config/assembly.rb

@@ -0,0 +1,15 @@
+Alki do
+  set :processors do
+    []
+  end
+
+  service :root_processor do
+    require 'piah/processor_list'
+    Piah::ProcessorList.new(processors.to_a)
+  end
+
+  factory :process do
+    require 'piah/config_processor'
+    Piah::ConfigProcessor.new(root_processor).method(:process)
+  end
+end

data/lib/alki_loader.rb

@@ -0,0 +1 @@
+Alki::Loader.register 'piah/dsl.rb', builder: 'alki/dsls/dsl'

data/lib/piah.rb

@@ -0,0 +1,2 @@
+require 'alki'
+Alki.project_assembly!

data/lib/piah/config_processor.rb

@@ -0,0 +1,137 @@
+module Piah
+  class ConfigMergeError < StandardError
+    def initialize(a,b)
+      super "Failed to merge config values #{a.pretty_print_inspect}\n  and\n#{b.pretty_print_inspect}"
+    end
+  end
+
+  class ConfigProcessorResultError < StandardError
+    def initialize(v)
+      super "Config processors must return a Hash or an Array of Hashes, got #{v.inspect}"
+    end
+  end
+
+  class ConfigProcessor
+    def initialize(processor)
+      @processor = processor
+    end
+
+    # Basic idea is that we have our list of config processors, along with the config types
+    # each one accepts. The list becomes our initial "chain", which is a particular ordering
+    # of the processors. We then run through the chain, providing each processor with the
+    # config types it wants, and adding it's outputs to the set of config types. The result
+    # is then a hash of all unhandled config types.
+    #
+    # If, when processing the chain, a processor (A) outputs a config type that has already been
+    # processed by an earlier (in the chain) processor (B), we abort and rearrange the chain so that
+    # the A processor will run before the B processor, and then execute the new chain.
+    # This repeats until we get all the way through the chain without encountering processors out
+    # of order.
+    def process(config)
+      types = @processor.types
+      chain = types.keys.map{|p| [p,{}] }
+      result = nil
+      config = copy(config)
+      until result
+        result = process_chain(types,chain,copy(config))
+        if result.is_a?(Set)
+          # Rearrange chain
+          move = chain.size.times.select do |i|
+            types[chain[i][0]].to_set.intersect? result
+          end
+          move.each do |i|
+            chain.push [chain[i][0],{}]
+            chain[i] = nil
+          end
+          chain.compact!
+          result = nil
+        end
+      end
+      result
+    end
+
+    private
+
+    # Returns final output or a Set of keys that have been generated
+    # after we've already processed them (requiring re-ordering)
+    def process_chain(types,chain,input)
+      output = input.dup
+      seen = Set.new
+      chain.each do |(p,cache)|
+        args = input.select{|k| types[p].include?(k) }
+        args.keys.each {|k| output.delete k }
+
+        unless cache[:result]
+          result = @processor.process(p, copy(args))
+          if result.is_a? Array
+            cache[:result] = result.inject({}) do |r,c|
+              merge(r,copy(c))
+            end
+          else
+            unless result.is_a? Hash
+              raise Piah::ConfigProcessorResultError.new result
+            end
+            cache[:result] = copy(result)
+          end
+          unless cache[:result].keys.all?{|k| k && k.size > 0}
+            raise Piah::ConfigProcessorResultError.new cache[:result]
+          end
+        end
+        keys = cache[:result].keys.to_set
+        out_of_order = seen & keys
+        # If chain is out of order, abort
+        return out_of_order unless out_of_order.empty?
+
+        seen.merge types[p]
+        begin
+          merge(output,copy(cache[:result]))
+        rescue => e
+          raise ConfigMergeError.new(output,cache[:result])
+        end
+        cache[:result].each_key {|k| input[k] = output[k] unless input.key? k }
+      end
+      output
+    end
+
+    def merge(a,b)
+      if a.is_a?(Hash) && b.is_a?(Hash)
+        a.merge!(b) do |key,av,bv|
+          merge av, bv
+        end
+      elsif a.is_a?(Array) && b.is_a?(Array)
+        a.push *b.reject{|e| a.include?(b)}
+      elsif a.is_a?(Proc) && b.is_a?(Proc)
+        b
+      elsif a != b
+        raise ConfigMergeError.new(a,b)
+      end
+      a
+    end
+
+    def copy(obj)
+      if obj.is_a?(Hash)
+        obj.each_with_object({}) do |(k,v),h|
+          new_k = k.to_sym rescue k
+          h[new_k] = copy(v)
+        end
+      elsif obj.is_a?(Array)
+        obj.map {|e| copy e }
+      else
+        obj.dup
+      end
+    end
+
+    def symbolize_keys(obj)
+      if obj.is_a?(Hash)
+        obj.each_with_object({}) do |(k,v),h|
+          new_k = k.to_sym rescue k
+          h[new_k] = symbolize_keys(v, &blk)
+        end
+      elsif obj.is_a?(Array)
+        obj.map{|e| symbolize_keys(e, &blk)}
+      else
+        obj
+      end
+    end
+  end
+end

data/lib/piah/dsl.rb

@@ -0,0 +1,85 @@
+Alki do
+  require_dsl 'alki/dsls/service'
+
+  init do
+    @auto_outputs = []
+    @types = {}
+    @has_process =false
+  end
+
+  dsl_method :input do |type,*args|
+    raise ArgumentError.new "Too many arguments" if args.size > 1
+    if type.is_a?(Hash) && type.size == 1
+      type, *args = type.to_a.first
+    end
+    has_default = !args.empty?
+    default = args.first
+    @types[type] = !has_default
+    add_method type, private: true, subclass: 'Instance' do
+      @_values[type] || default
+    end
+  end
+
+  dsl_method :passthrough do |type,*args|
+    input type, *args
+    @auto_outputs << type
+  end
+
+  dsl_method :process do |&blk|
+    @has_process = true
+    add_method :process, subclass: 'Instance', &blk
+  end
+
+  finish do
+    unless @has_process
+      raise "Processor must have process block!"
+    end
+    types = @types.keys
+    required_types = @types.inject([]) {|a,(n,r)| r ? a << n : a }.to_set
+    auto_outputs = @auto_outputs
+
+    ivs = class_builder[:initialize_params]&.map {|n,_| :"@#{n}"} || []
+
+    add_module ctx[:module], subclass: 'Instance'
+    ctx[:module] = nil
+    add_method :initialize, subclass: 'Instance' do |config_type,values,output|
+      @_config_type = config_type
+      ivs.each do |iv|
+        instance_variable_set iv, config_type.instance_variable_get(iv)
+      end
+      @_values = values
+      @_output = output
+    end
+
+    add_method :output, subclass: 'Instance' do |*keys,value,&blk|
+      if blk
+        keys << value
+        value = blk
+      end
+      @_output << keys.reverse.inject(value) do |v,k|
+        {k => v}
+      end
+    end
+
+    add_method :types do
+      types
+    end
+
+    add_method :process do |args|
+      output = []
+      unless auto_outputs.empty?
+        output << auto_outputs.map do |o|
+          args.key?(o) ? [o,args[o]] : nil
+        end.compact.to_h
+      end
+      unless required_types <= args.keys.to_set
+        return output
+      end
+      instance = self.class::Instance.new self, args, output
+      if instance.respond_to? :process
+        instance.process
+      end
+      output
+    end
+  end
+end

data/lib/piah/processor_list.rb

@@ -0,0 +1,15 @@
+module Piah
+  class ProcessorList
+    def initialize(*processors)
+      @processors = processors.flatten
+    end
+
+    def types
+      @types ||= @processors.each_index.zip(@processors.map(&:types)).to_h
+    end
+
+    def process(p,args)
+      @processors[p].process args
+    end
+  end
+end

data/lib/piah/version.rb

@@ -0,0 +1,3 @@
+module Piah
+  VERSION = "0.1.0"
+end

data/piah.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "piah/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "piah"
+  spec.version       = Piah::VERSION
+  spec.authors       = ["Matt Edlefsen"]
+  spec.email         = ["matt.edlefsen@gmail.com"]
+
+  spec.summary       = %q{A library for constructing data pipelines.}
+  spec.homepage      = "https://github.com/alki-project/piah"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "alki", "~> 0.13.0"
+end

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification
+name: piah
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matt Edlefsen
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-11-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: alki
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.0
+description: 
+email:
+- matt.edlefsen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.adoc
+- Rakefile
+- config/assembly.rb
+- lib/alki_loader.rb
+- lib/piah.rb
+- lib/piah/config_processor.rb
+- lib/piah/dsl.rb
+- lib/piah/processor_list.rb
+- lib/piah/version.rb
+- piah.gemspec
+homepage: https://github.com/alki-project/piah
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: A library for constructing data pipelines.
+test_files: []