eclipsed 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b4f70c005d67c21a1d3353fea853d40edecf0893
+  data.tar.gz: 6032d9df663bcab6b50f91c8bba89726e9b19b96
+SHA512:
+  metadata.gz: 3dd8100fb25cb79f677e6901e5136b893efd36dbeb9b315c444b3b8c0c0b2efb779ca6c6730255d48bcc9dde0b729e8cdf021909410f3ec763f6c48282ca265e
+  data.tar.gz: 3a538aca81687a45fb5503865213b211533bbf11beb97b683bbce711a862aaac52205f63e253a02c84910265d3e8058ab92689a27c8c38eaf013ad0c7245e325

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http:contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in eclipsed.gemspec
+gemspec
+
+gem 'ffi'               # For loading C functions, in this case the hash functions
+gem 'awesome_print'     # Pretty-printer
+gem 'table_print'       # Pretty-printer

data/README.md

@@ -0,0 +1,39 @@
+# Eclipsed
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/eclipsed`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'eclipsed'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install eclipsed
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/eclipsed/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "eclipsed"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/eclipsed.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'eclipsed/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "eclipsed"
+  spec.version       = Eclipsed::VERSION
+  spec.authors       = ["Vicente Adolfo Bolea Sanchez"]
+  spec.email         = ["vicente.bolea@gmail.com"]
+
+  if spec.respond_to?(:metadata)
+    #spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com' to prevent pushes to rubygems.org, or delete to allow pushes to any server."
+  end
+
+  spec.summary       = %q{EclipseMR governor}
+  spec.description   = %q{Controls EclipseMR.}
+  spec.homepage      = "http://dicl.unist.ac.kr"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.8"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/exe/eclipsed

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+#
+require 'eclipsed'
+
+EclipseDaemon::CLI_driver.new input: ARGV

data/lib/eclipsed/version.rb

@@ -0,0 +1,3 @@
+module Eclipsed
+  VERSION = "0.1.0"
+end

data/lib/eclipsed.rb

@@ -0,0 +1,287 @@
+require "eclipsed/version"
+require 'optparse'          # For parsing the options
+require 'json'              # 
+require 'ffi'               # For loading C functions, in this case the hash functions
+require 'awesome_print'     # Pretty-printer
+require 'table_print'       # Pretty-printer
+
+module Eclipsed
+  CONFPATH = "@sysconfdirfull@/eclipse.json"
+  BINDIR   = "@bindirfull@"
+  LIBDIR   = "@libdirfull@"
+  CXX      = "@CXX@"
+  module EclipseAPI  #{{{
+    extend FFI::Library
+    ffi_lib "#{LIBDIR}/libecfs.so"
+    attach_function :hash_ruby, [ :string ], :uint32
+  end
+  #}}} 
+  # print_async {{{
+  def print_async(msg) 
+    Thread.new(msg) do |m|
+      print "#{m} "
+      loop do
+        print "\b\\"
+        sleep 0.3
+        print "\b-"
+        sleep 0.3
+        print "\b/"
+        sleep 0.3
+      end
+    end
+  end
+  #}}}
+  # find_confpath {{{
+  def find_confpath
+    home = "#{ENV['HOME']}/.eclipse.json"
+    etc  = "/etc/.eclipse.json"
+
+    if File.exists? home
+      return home
+    elsif File.exists? etc
+      return etc
+    else
+      return CONFPATH
+    end 
+  end
+  # }}}
+
+  class Core 
+    include EclipseDaemon
+    # Initialize {{{
+    def initialize 
+      @nodelist = File.open(find_confpath) { |f| JSON.parse(f.read) }['network']['nodes']
+      @verbose  = false
+    end
+
+    # }}}
+    # launch {{{
+    def launch
+      thr = print_async "Initializing framework..."
+      `#{BINDIR}/master &>/dev/null &`
+      sleep 1
+      @nodelist.each do |node|
+        system "ssh #{node} 'nohup #{BINDIR}/slave </dev/null &>/dev/null & exit'"
+      end
+      thr.exit
+      print "\r"
+    end 
+
+    #}}}
+    # show {{{
+    def show 
+      msg_handler = print_async "Collecting information..."
+
+      instance = [ ]
+      in_english = { true => "Running", false => "Stopped" }
+
+      status = in_english[system "pgrep -u #{`whoami`.chomp} master &>/dev/null"]
+      instance << { :host => "localhost", :status => status, :role => "master" }
+
+      @nodelist.each do |node|
+        mr_status    = in_english[system "ssh #{node} pgrep -u #{`whoami`.chomp} -x slave &>/dev/null"]
+        instance << { :host => node, :status => mr_status, :role => "worker" }
+      end
+
+      msg_handler.exit
+      print "\r"
+      tp instance, "host", "role", "status"
+    end 
+
+    #}}}
+    # close {{{
+    def close
+      thr = print_async "Stopping framework..."
+      `pkill -u #{`whoami`.chomp} master`
+      @nodelist.each do |node|
+        `ssh #{node} pkill -u #{`whoami`.chomp} slave`
+      end 
+      thr.exit
+      print "\r"
+    end #}}}
+    # submit {{{
+    def submit(input)
+    #  @instance[:app]= input
+    #  File.open(@fs_tmpfile, 'w') { |f| f.write(JSON.generate(@instance)) }
+
+      system input.join(' ')
+    end #}}}
+    # kill {{{
+    def kill(input)
+      @nodelist.each do |node|
+        cmd = "ssh #{node} \'pkill -u #{`whoami`.chomp} #{input.join}\'"
+        puts cmd if @verbose
+        system cmd
+      end
+    end #}}}
+    # pry {{{
+    def pry 
+      require 'pry'
+      binding.pry
+    end #}}}
+  end
+
+  class Fs < Core
+    include EclipseAPI
+    def initialize #{{{
+      @files           = {}
+      @config          = File.open(find_confpath) { |f| JSON.parse(f.read) }
+      @fs_path         = @config['path']['filesystem']
+      @fs_scratch_path = @config['path']['scratch']
+      @fs_tmpfile      = @fs_path + "/.list"
+      @files           = File.open(@fs_tmpfile) { |f| JSON.parse(f.read) } if File.exist? @fs_tmpfile
+      alias :hash :hash_ruby
+      super()
+    end
+
+    def node_containing(fn); @nodelist[hash(fn) % @nodelist.length] end
+
+    #}}}
+    # put {{{
+    def put(input)
+      input.each do |fn|
+        node = node_containing(fn)
+        @files[fn] = node
+
+        File.open(@fs_tmpfile, 'w') { |f| f.write(JSON.generate(@files)) }
+        cmd = "scp #{@fs_path}/#{fn} #{node}:#{@fs_scratch_path}/#{fn}"
+        puts cmd if @verbose
+        system cmd
+      end
+    end
+
+    #}}}
+    # cat {{{
+    def cat(input)
+      input.each do |fn|
+        system "ssh #{node_containing(fn)} cat #{@fs_scratch_path}/#{fn}"
+      end
+    end
+
+    #}}}
+    # get {{{
+    def get(input)
+      input.each do |fn|
+        cmd = "scp #{node_containing(fn)}:#{@fs_scratch_path}/#{fn} ."
+        puts cmd if @verbose
+        system cmd
+      end
+    end
+
+    #}}}
+    # rm {{{
+    def rm(input)
+      input = @files.keys.grep(%r[#{@regex}]) if @regex
+
+      input.each do |fn|
+#        raise "\'#{fn}\' not found in Eclipse FS" unless @files[fn]
+
+        node = node_containing(fn)
+        pathtofile = @fs_scratch_path + "/" + fn
+        cmd = "ssh #{node} rm -f #{pathtofile}"
+        puts cmd if @verbose
+        system cmd
+        @files.delete(fn)
+        File.open(@fs_tmpfile, 'w') { |f| f.write(JSON.generate(@files)) }
+      end
+    end
+
+    #}}}
+    def list #{{{
+      thr = print_async "Collecting information..."
+      output = [ ] 
+      @files.each{|k, v| output << { :filename => k, :location => v} }
+
+      @nodelist.each do |node| 
+        `ssh #{node} ls #{@fs_scratch_path}`.each_line do |l| 
+          l.chomp!
+          output << { :filename => l, :location => node} unless @files.has_key?(l)
+        end
+      end
+
+      thr.exit
+      print "\r\e[K" 
+      tp output, :filename, :location
+    end
+
+    #}}}
+    def config #{{{
+      ap self.instance_variables.map{|var|  [var, self.instance_variable_get(var)]}.to_h
+    end  #}}}
+    # compile {{{
+    def compile(input)
+      sources = input.join(' ')
+      raise 'Need to specify output file (-o File)' unless @outputfile 
+
+      cmd = "#{CXX} -static -o #{@outputfile} #{sources} -l ecfs"
+      puts cmd if @verbose
+      system cmd
+    end
+
+    #}}}
+    def clear #{{{
+      rm @files.keys
+      @nodelist.each do |node|
+        cmd = "ssh #{node} rm -rf #{@fs_scratch_path}/.job* #{@fs_scratch_path}/*"
+        puts cmd if @verbose
+        system cmd
+      end
+    end
+    #}}}
+  end
+
+  class CLI_driver < Fs 
+    def initialize input:  #{{{
+      @options = {}
+      super()
+      OptionParser.new do |opts|
+        opts.banner = "ecfs (Eclipse FileSystem) is an script to manage the fs\n" +
+          "Usage: ecfs [options] <actions> [FILE]..."
+        opts.version = 1.0
+        opts.program_name = "Eclipse Launcher"
+        opts.separator "Core actions"
+        opts.separator "    launch       Create new Eclipse network"
+        opts.separator "    close        Close the network"
+        opts.separator "    status       Check the status of the network"
+        opts.separator "    submit       Submit application"
+        opts.separator "    kill         kill application in each node"
+        opts.separator ""
+        opts.separator "Filesystem actions"
+        opts.separator "    put FILE...  insert FILE..."
+        opts.separator "    get FILE...  copy the FILES to the current directory"
+        opts.separator "    rm FILE...   remove FILE..."
+        opts.separator "    cat FILE...  \'cat\' the FILE..."
+        opts.separator "    clear        remove all the files in the FS"
+        opts.separator "    ls           list all the files in the FS"
+        opts.separator "    config       list all the internal variables"
+        opts.separator "    cc -o OUTPUT INPUT...  Compile file with EclipseFS API using OUTPUT name"
+        opts.separator ""
+        opts.separator "Options"
+        opts.on_tail("-h", "--help"   , "recursive this")         { puts opts; exit}
+        opts.on_tail("-v", "--verbose" , "printout verbose info") { @verbose = true }
+        opts.on_tail("-V", "--version" , "printout version") { puts opts.ver; exit }
+        opts.on_tail("-o FILE", "output file, for mcc") { |f| @outputfile = f }
+        opts.on_tail("-r REGEX", "regex for remove or ls") { |r| @regex = r }
+      end.parse! input
+
+      case input.shift
+      when 'launch' then launch
+      when 'close' then  close
+      when 'status' then show
+      when 'submit' then submit input
+      when 'kill' then   kill input
+      when 'put' then    put input
+      when 'get' then    get input
+      when 'cat' then    cat input
+      when 'rm' then     rm input
+      when 'clear' then  clear
+      when 'ls' then     list
+      when 'config' then config 
+      when 'cc' then     compile(input)
+      when 'pry' then    pry
+      else               raise "Not action given"
+      end
+    end #}}}
+  end
+
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: eclipsed
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vicente Adolfo Bolea Sanchez
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-09-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Controls EclipseMR.
+email:
+- vicente.bolea@gmail.com
+executables:
+- eclipsed
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- eclipsed.gemspec
+- exe/eclipsed
+- lib/eclipsed.rb
+- lib/eclipsed/version.rb
+homepage: http://dicl.unist.ac.kr
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: EclipseMR governor
+test_files: []