flowckerc 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6618e3366d09596072f37257e2730dd47790511b
+  data.tar.gz: 9767ab7b95d3a1e7198df9bb693d72634b0a2e77
+SHA512:
+  metadata.gz: 4cf94ff7a0f75fcc81468d477ea1710549a397fccc3738f5206e1af45edf5ac5bd010814e96f4aebbe832eca6f9257fce4256fa56467567a1bb7c10547323cfe
+  data.tar.gz: 39706d2669d77631f9458d948676f5cdd9fc54eb61110698974066bf5c34af58ca8e2f4fc8018e6b82724ab9628a2445e4ed476999433993625353a27ed933df

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.0

data/COPYRIGHT

@@ -0,0 +1,13 @@
+    Copyright (C) 2015 Leopoldo Lara Vazquez.
+
+    This program is free software: you can redistribute it and/or  modify
+    it under the terms of the GNU Affero General Public License, version 3,
+    as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in flowckerc.gemspec
+gemspec

data/README.md

@@ -0,0 +1,24 @@
+# Flowcker
+
+Flowcker is a distributed processing platform, based on Flow based programming and linux containers (Docker).
+
+Flow based programing (FBP) is a paradigm that makes distributed computing much easy to understand and develop.
+
+Linux containers is a technology that facilitates the deployment and distribution of server software components.
+
+Flowcker integrates FBP and Docker to make distributed processing software for Data Sciences, Scientific computation, Bioinformatics and the like much more easier and fun to develop.
+
+# Status
+Flowcker is currently at very early stages of development and documentation.
+
+# Components
+In github.com/flowcker/flowcker:
+ + common - Common components
+ + atom - Runtime for Flowcker atoms (FBP components)
+ + runtime - Runtime for Flowcker molecules (FBP graphs)
+
+In github.com/flowcker/flowcker-stdlib:
+ + Standard library of atoms (FBP components)
+
+In github.com/flowcker/flowckerc:
+ + Flowcker formula compiler

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/flowckerc

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+#    Copyright (C) 2015 Leopoldo Lara Vazquez.
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+$:.unshift Dir.pwd
+
+require 'flowckerc'
+require 'json'
+
+if ARGV.length > 1
+  options = JSON.parse(File.read(ARGV[1]))
+else
+  options = {}
+end
+
+formula = Flowckerc::Formula.new ARGV[0]
+
+solution = Flowckerc::Solver.solve(formula, options)
+
+puts JSON.pretty_generate(solution.molecule.toHash)

data/examples/http_parallelsum_redis.ff

@@ -0,0 +1,14 @@
+
+atom 'stdlib readHttp' do
+  name :readFile
+  config url: 'http://pastebin.com/raw.php?i=qc0z66VP'
+  out from: 'output', to: [:sum, 'input']
+end
+
+mix 'parallelsum.ff', :sum
+
+atom 'stdlib updateRedisKey' do
+  name :writeFile
+  config addr: '192.168.59.103:49153', key: 'output'
+  inp to: 'input', from: [:sum, 'output']
+end

data/examples/http_parallelsum_redis_simple.ff

@@ -0,0 +1,32 @@
+
+atom 'stdlib readHttp' do
+  name :readFile
+  config url: 'http://pastebin.com/raw.php?i=qc0z66VP'
+  out from: 'output', to: [:split, 'input']
+end
+
+atom 'stdlib splitByLine' do
+  name :split
+  out from: 'output', to: [:add1, 'input']
+  out from: 'output', to: [:add2, 'input']
+end
+
+atom 'stdlib accumAdd' do
+  name :add1
+  out from: 'output', to: [:reduce, 'input']
+end
+
+atom 'stdlib accumAdd' do
+  name :add2
+  out from: 'output', to: [:reduce, 'input']
+end
+
+atom 'stdlib sum' do
+  name :reduce
+  out from: 'output', to: [:writeFile, 'input']
+end
+
+atom 'stdlib updateRedisKey' do
+  name :writeFile
+  config addr: '192.168.59.103:49153', key: 'output'
+end

data/examples/parallelsum.ff

@@ -0,0 +1,22 @@
+inport 'input', :split, 'input'
+outport 'output', :reduce, 'output'
+
+atom 'stdlib splitByLine' do
+  name :split
+  out from: 'output', to: [:add1, 'input']
+  out from: 'output', to: [:add2, 'input']
+end
+
+atom 'stdlib accumAdd' do
+  name :add1
+  out from: 'output', to: [:reduce, 'input']
+end
+
+atom 'stdlib accumAdd' do
+  name :add2
+  out from: 'output', to: [:reduce, 'input']
+end
+
+atom 'stdlib sum' do
+  name :reduce
+end

data/examples/serialsum.ff

@@ -0,0 +1,21 @@
+
+atom 'stdlib readHttp' do
+  name :readFile
+  config url: 'http://pastebin.com/raw.php?i=qc0z66VP'
+  out from: 'output', to: [:split, 'input']
+end
+
+atom 'stdlib splitByLine' do
+  name :split
+  out from: 'output', to: [:add, 'input']
+end
+
+atom 'stdlib accumAdd' do
+  name :add
+  out from: 'output', to: [:writeFile, 'input']
+end
+
+atom 'stdlib updateRedisKey' do
+  name :writeFile
+  config addr: '192.168.59.103:49153', key: 'output'
+end

data/flowckerc.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'flowckerc/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "flowckerc"
+  spec.version       = Flowckerc::VERSION
+  spec.authors       = ["Leo Lara"]
+  spec.email         = ["leo@leolara.me"]
+
+  spec.summary       = %q{flowckerc is Flowcker formula compiler}
+  spec.description   = %q{Flowcker is a distributed processing platform, based on Flow based programming and linux containers (Docker).}
+  spec.homepage      = "http://www.flowcker.io"
+  spec.license       = "AGPLv3"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "hike", "~> 2.1.3"
+  spec.add_runtime_dependency "json", "~> 1.8.1"
+
+  spec.add_development_dependency "bundler", "~> 1.8"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/flowckerc.rb

@@ -0,0 +1,206 @@
+#    Copyright (C) 2015 Leopoldo Lara Vazquez.
+#
+#    This program is free software: you can redistribute it and/or  modify
+#    it under the terms of the GNU Affero General Public License, version 3,
+#    as published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'flowckerc/version'
+require 'flowckerc/tokens'
+require 'flowckerc/formula'
+
+module Flowckerc
+  module UniqueID
+    @@count = 0
+
+    def self.next
+      @@count = @@count + 1
+      @@count
+    end
+  end
+
+  class SymTable
+    def initialize
+      @table = {}
+    end
+
+    def add sym, value
+      @table[sym] = value
+    end
+
+    def getId sym
+      @table[sym].id
+    end
+
+    def value sym
+      @table[sym]
+    end
+
+    def isAtom sym
+      @table[sym].is_a? Atom
+    end
+
+    def isSolution
+      @table[sym].is_a? FormulaSolution
+    end
+  end
+
+  class SolverClass
+    def solve formula, options={}
+      syms = SymTable.new
+      molecule = Molecule.new
+      molecule_ports = MoleculePorts.new
+
+      context = FormulaExecContext.new syms, molecule, molecule_ports, options
+      formula.eval context
+
+      LinksSolver.new(syms).solve(molecule.links)
+
+      solve_ports_ids syms, molecule_ports
+
+      FormulaSolution.new molecule, molecule_ports
+    end
+
+    private
+
+    def solve_ports_ids syms, molecule_ports
+      molecule_ports.inports.each do |inport|
+        if inport[:to_atom].is_a? Symbol
+          inport[:to_atom] = syms.getId inport[:to_atom]
+        end
+      end
+
+      molecule_ports.outports.each do |outport|
+        if outport[:from_atom].is_a? Symbol
+          outport[:from_atom] = syms.getId outport[:from_atom]
+        end
+      end
+    end
+
+    class LinksSolver
+      def initialize syms
+        @syms = syms
+      end
+
+      def solve links
+        links.each do |link|
+          if link.from_atom.is_a? Symbol
+            solve_src link
+          end
+          if link.to_atom.is_a? Symbol
+            solve_dst link
+          end
+        end
+      end
+
+      def solve_src link
+        name = link.from_atom
+        port = link.from_port
+        if @syms.isAtom name
+          link.from_atom = @syms.getId name
+        else
+          solution = @syms.value name
+          actual_port =  solution.molecule_ports.outportByName port
+          link.from_atom = actual_port[:from_atom]
+          link.from_port = actual_port[:from_port]
+        end
+      end
+
+      def solve_dst link
+        name = link.to_atom
+        port = link.to_port
+        if @syms.isAtom name
+          link.to_atom = @syms.getId name
+        else
+          solution = @syms.value name
+          actual_port =  solution.molecule_ports.inportByName port
+          link.to_atom = actual_port[:to_atom]
+          link.to_port = actual_port[:to_port]
+        end
+      end
+    end
+
+    class AtomExecContext
+      def initialize atom, parent
+        @atom = atom
+        @parent = parent
+      end
+
+      def out spec
+        from_port = spec[:from]
+        spec[:from] = [@atom.id, from_port]
+        @parent.link spec
+      end
+
+      def inp spec
+        to_port = spec[:to]
+        spec[:to] = [@atom.id, to_port]
+        @parent.link spec
+      end
+
+      def name atom_name
+        @parent.syms.add atom_name, @atom
+      end
+
+      def config data={}
+        @atom.config = data
+      end
+
+      def options
+        @parent.options
+      end
+    end
+
+    class FormulaExecContext
+      attr_reader :syms
+      attr_reader :options
+      def initialize syms, molecule, molecule_ports, options
+        @syms = syms
+        @molecule = molecule
+        @molecule_ports = molecule_ports
+        @options = options
+      end
+
+      def atom element, &blk
+        atom = Atom.new element
+        @molecule.atoms.push atom
+        AtomExecContext.new(atom, self).instance_exec &blk
+      end
+
+      def link spec={}
+        @molecule.links.push Link.new spec[:from][0], spec[:from][1], spec[:to][0], spec[:to][1]
+      end
+
+      def inport name, to_atom, to_port
+        @molecule_ports.inports.push  name: name,
+                                      to_atom: to_atom,
+                                      to_port: to_port
+      end
+
+      def outport name, from_atom, from_port
+        @molecule_ports.outports.push   name: name,
+                                        from_atom: from_atom,
+                                        from_port: from_port
+      end
+
+      def mix formula_file, name, options={}
+        formula = Formula.new formula_file
+        solution = Solver.solve formula, options
+        @molecule.atoms += solution.molecule.atoms
+        @molecule.links += solution.molecule.links
+
+        @syms.add name, solution
+      end
+    end
+  end
+
+  Solver = SolverClass.new
+
+end

data/lib/flowckerc/formula.rb

@@ -0,0 +1,33 @@
+#    Copyright (C) 2015 Leopoldo Lara Vazquez.
+#
+#    This program is free software: you can redistribute it and/or  modify
+#    it under the terms of the GNU Affero General Public License, version 3,
+#    as published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'hike'
+
+module Flowckerc
+  class Formula
+
+    @@trail = Hike::Trail.new "/"
+    @@trail.append_extensions ".ff"
+    @@trail.append_paths *$:
+
+    def initialize orig_file
+      @file = @@trail.find orig_file
+      raise 'Cannot find formula '+ orig_file if @file.nil?
+    end
+
+    def eval context
+      context.instance_eval File.read(@file), @file
+    end
+  end
+end

data/lib/flowckerc/tokens.rb

@@ -0,0 +1,127 @@
+#    Copyright (C) 2015 Leopoldo Lara Vazquez.
+#
+#    This program is free software: you can redistribute it and/or  modify
+#    it under the terms of the GNU Affero General Public License, version 3,
+#    as published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module Flowckerc
+  class Link
+    attr_reader :id
+    attr_accessor :from_atom
+    attr_accessor :from_port
+    attr_accessor :to_atom
+    attr_accessor :to_port
+
+    def initialize from_atom, from_port, to_atom, to_port
+      @id = UniqueID.next
+      @from_atom = from_atom
+      @from_port = from_port
+      @to_atom = to_atom
+      @to_port = to_port
+    end
+
+    def toHash
+      {
+          id: @id,
+          from: {
+            atomID: @from_atom,
+            port: {
+              name: @from_port
+            }
+          },
+          to: {
+            atomID: @to_atom,
+            port: {
+              name: @to_port
+            }
+          }
+      }
+    end
+  end
+
+  class Atom
+    attr_reader :id
+    attr_reader :element
+    attr_accessor :config
+
+    def initialize element
+      @id = UniqueID.next
+      @element = element
+      @config = {}
+    end
+
+    def toHash
+      {
+          id: @id,
+          element: @element,
+          config: @config
+      }
+    end
+  end
+
+  class Molecule
+    attr_accessor :atoms
+    attr_accessor :links
+
+    def initialize
+      @atoms = []
+      @links = []
+    end
+
+    def toHash
+      res = {}
+      res[:atoms] = @atoms.map do |atom|
+        atom.toHash
+      end
+
+      res[:links] = @links.map do |link|
+        link.toHash
+      end
+      res
+    end
+  end
+
+  class MoleculePorts
+    attr_accessor :inports
+    attr_accessor :outports
+
+    def initialize
+      @inports = []
+      @outports = []
+    end
+
+    def inportByName name
+      @inports.each do |port|
+        return port if port[:name] == name
+      end
+      return nil
+    end
+
+    def outportByName name
+      @outports.each do |port|
+        return port if port[:name] == name
+      end
+      return nil
+    end
+  end
+
+  class FormulaSolution
+    attr_accessor :molecule
+    attr_accessor :molecule_ports
+
+    def initialize molecule, molecule_ports
+      @molecule = molecule
+      @molecule_ports = molecule_ports
+    end
+  end
+
+
+end

data/lib/flowckerc/version.rb

@@ -0,0 +1,3 @@
+module Flowckerc
+  VERSION = "0.0.0"
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: flowckerc
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Leo Lara
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hike
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Flowcker is a distributed processing platform, based on Flow based programming
+  and linux containers (Docker).
+email:
+- leo@leolara.me
+executables:
+- flowckerc
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- COPYRIGHT
+- Gemfile
+- README.md
+- Rakefile
+- bin/flowckerc
+- examples/http_parallelsum_redis.ff
+- examples/http_parallelsum_redis_simple.ff
+- examples/parallelsum.ff
+- examples/serialsum.ff
+- flowckerc.gemspec
+- lib/flowckerc.rb
+- lib/flowckerc/formula.rb
+- lib/flowckerc/tokens.rb
+- lib/flowckerc/version.rb
+homepage: http://www.flowcker.io
+licenses:
+- AGPLv3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: flowckerc is Flowcker formula compiler
+test_files: []