zillabyte-cli 0.0.9

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YTVkNDVmMWE1NWJiMzhmMTk5M2IxOGJkZDExYjIxMTY3MGRiNDBjMg==
+  data.tar.gz: !binary |-
+    ZGEyMjU3NjM5MTE0NWE1N2I4ZDdkZjZlNTk2NmQzNzYzYmI2ZGZkYw==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NDk2ZTcyMjk1ODg5NTE3NTU0ZGFmNDlkYTY4MWIzOGEyYjEyYjJlNDk0YTI2
+    OTAwYmRkMWU1MzliOWYyMmIyZWFjMTFiZWI0NWE0NmUzZTA3Y2Y4ZTkyNjMz
+    ODBkYWQ1ZDA1MjkzZGE1NTg5MDA0NzMwZmFhYzNhNmRlYzBiNTY=
+  data.tar.gz: !binary |-
+    MjI3YzIzNGMzMjkzY2JjNjZmMjc2YzU4MDU3Y2RlNTQzYjE0OTYzN2VhNTc4
+    ZWRmMGZhMjc2M2FiYjkwYTc1OGY2YzUwNjM3YmRiOTU1NzA2OWFiNzBiNWU4
+    YTRjYTNiMzQyYWM5ZmVhZjdlZDY1MjI5ZDJkNjUzZjc3OWU1NTE=

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in zillabyte-cli.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Zillabyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Zillabyte
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'zillabyte'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install zillabyte
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/bin/zillabyte

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'zillabyte' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+require_relative '../lib/zillabyte-cli'
+Zillabyte::CLI.start(*ARGV)
+

data/lib/zillabyte/api/base.rb

@@ -0,0 +1,11 @@
+class Zillabyte::API::Base
+
+  def initialize(api)
+    @api = api
+  end
+  
+  def request(*args)
+    @api.request(*args)
+  end
+  
+end

data/lib/zillabyte/api/data.rb

@@ -0,0 +1,126 @@
+require 'net/http'
+
+class Zillabyte::API::Data < Zillabyte::API::Base
+
+
+  # # GET /datasets
+  # def list(options = {})
+  # 
+  #   options = {
+  #     # TODO
+  #   }.merge(options)
+  #   
+  #   res = @api.request(
+  #     :expects  => 200,
+  #     :method   => :get,
+  #     :path     => "/datasets"
+  #   )
+  #   
+  #   res.body
+  # 
+  # end
+  
+  
+  
+  
+  # GET /datasets/1
+  def get(id, options = {})
+    
+    options = {
+      :name => id
+      # rows
+      # schema
+      # relations
+    }.merge(options)
+   
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/relations/#{id}",
+      :body     => options.to_json 
+    )
+
+    res.body
+    
+  end
+  
+  
+  # POST /datasets
+  def create(name, options = {})
+    
+    options = {
+      :name => name,
+      # rows
+      # schema
+      # relations
+    }.merge(options)
+    
+    res = @api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/relations",
+      :body     => options.to_json
+    )
+
+    res.body
+    
+  end
+  
+  
+  
+  
+  
+  
+  # POST /datasets/1/append
+  def append(id, options = {})
+    opts = {:name => id}
+    file = options[:file] if options[:file]
+    if(file)
+      opts[:size] = File.size(file)
+      options.delete(:file)
+    end
+    options = opts.merge(options)
+    
+    res = @api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/relations/#{id}/append",
+      :body     => options.to_json
+    )
+
+    hash = res.body
+    if(hash['uri'])
+      uri = URI(hash['uri'])
+      response = nil
+      try_again = 1
+      while(try_again)
+        Net::HTTP.start(uri.host, uri.port, :use_ssl => true) do |http|
+          request = Net::HTTP::Put.new(uri.request_uri)
+          request.body_stream = Zillabyte::Common::Tar.tar(file, [])
+          request['Content-Length'] = request.body_stream.size
+          request['Content-Type'] = ''
+          response = http.request(request)
+          if response.code.to_i >= 300
+            try_again = 1
+            break
+          end
+          try_again = nil
+        end
+      end
+      hash[:appended] = opts[:size]
+      hash.delete('uri')
+      res = @api.request(
+        :expects => 200,
+        :method => :post,
+        :path => "/relations/#{id}/store",
+        :body => options.to_json
+      )
+    end
+    hash
+ 
+  end
+  
+  
+  
+
+end

data/lib/zillabyte/api/flows.rb

@@ -0,0 +1,239 @@
+require 'net/http'
+require 'mkfifo'
+require 'zillabyte/cli/config'
+
+class Zillabyte::API::Flows < Zillabyte::API::Base
+
+
+  # GET /flows
+  def list(options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+    
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows"
+    )
+    
+    res.body
+
+  end
+  
+  
+  
+
+  # GET /flows/id
+  def get(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+    
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{id}",
+      :body     => options.to_json 
+    )
+    
+    res.body
+
+  end
+  
+  
+  
+
+  # GET /flows/id/kill
+  def kill(id, options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+    
+    res = @api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/flows/#{id}/kill",
+      :body     => options.to_json 
+    )
+    
+    res.body
+
+  end
+  
+  
+  
+  
+  
+  
+  
+  
+  # GET /flows/id
+  def pull_to_directory(id, dir, progress = nil, options = {})
+    
+    # Get the resource. note query params
+    progress.display "downloading ##{id}" if progress
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{id}/download" 
+      #:path     => "/flows/#{id}?indirect_url=1" 
+      #:path     => "/flows/#{id}?tar=1" 
+    )
+    
+    hash = res.body
+    unless hash['status'] == 'error'
+      if(hash['tar'])
+        progress.display "unpacking to #{dir}" if progress
+        tar = StringIO.new(Base64.decode64(hash['tar']))
+        hash.delete('tar')
+        Zillabyte::Common::Tar.untar(tar, dir)
+#If you get an uri instead of the tar, then download the file directly from s3
+      elsif(hash['uri'])
+        progress.display "downloading ##{id} to #{dir}" if progress
+        uri = URI(hash['uri'])
+        try_again = 1
+        while(try_again)
+          Net::HTTP.start(uri.host, uri.port, :use_ssl => true) do |http|
+            request = Net::HTTP::Get.new(uri.request_uri)
+            response = http.request(request)
+            if response.code.to_i >= 300
+              try_again = 1
+              break
+            end
+            tar = StringIO.new(response.body)
+            Zillabyte::Common::Tar.untar(tar, dir)
+            try_again = nil
+          end
+        end
+        hash.delete('uri')
+      end
+    end
+    
+    hash
+    
+  end
+  
+  
+  
+  # POST /flows
+  def push_directory(dir, progress = nil, options = {})
+    
+    # Get the meta info
+    options[:local_flag] = 1
+    hash = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, progress, options)
+    if hash.nil?
+      error "unable to extract flow meta information"
+    end
+        
+    # Tar up everything... 
+    hash['ignore_files'] ||= []
+    hash['ignore_files'] << "info_to_ruby.in"
+    hash['ignore_files'] << "vEnv"
+    
+    top_dir = hash['top_dir'] || "."
+    ignore_files = hash['ignore_files'] || []
+    progress.display("packaging directory... ", false) if progress
+    tar = Zillabyte::Common::Tar.tar(top_dir, ignore_files)
+    progress.display("done") if progress
+
+    hash.delete('pwd')
+
+    options = {
+      :name => hash['name'],
+      :meta => hash,
+#      :tar => Base64.encode64(tar.read)
+    }.merge(options)
+    
+    progress.display("uploading #{tar.size} bytes... ", false) if progress
+    res = @api.request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/flows",
+      :body     => options.to_json
+    )
+
+    if(res.body['uri'])
+      uri = URI(res.body['uri'])
+      try_again = 1
+      while(try_again)
+        Net::HTTP.start(uri.host, uri.port, :use_ssl => true) do |http|
+          request = Net::HTTP::Put.new(uri.request_uri)
+          request.body_stream = tar
+          request['Content-Length'] = request.body_stream.size
+          request['Content-Type'] = ''
+          response = http.request(request)
+          if response.code.to_i >= 300
+            try_again = 1
+            break
+          end
+          try_again = nil
+        end
+      end
+      res.body.delete('uri')
+    end
+    progress.display("done")
+    res.body
+    
+  end
+
+####################################################IMPORTANT####################################################
+#                                                                                                               #
+# get_rich_meta_info_from_script only works with zillabyte flows:push now! The pipe "info_to_ruby.in" is        #
+# hardcoded as the meta info passing pipe, so if 2 processes are trying to access it simultaneously there might #
+# be problems since it gets created/destroyed within this script (attempting to create a fifo that already      #
+# exists or deleting a fifo that still has information in it may occur).                                        #
+#                                                                                                               #
+####################################################IMPORTANT####################################################
+  def self.get_rich_meta_info_from_script(dir, progress = nil, options = {}) 
+    hash = Zillabyte::CLI::Config.get_config_info(dir, progress, options)
+    if hash.nil?
+      return {"status" => "error", "error" => "invalid_directory", "error_message" => "The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file."}
+    end
+    
+    full_script = File.join(dir, hash['script'])
+    command = nil
+    
+    info_file = SecureRandom.uuid
+    arg = "--info --file #{info_file}"
+    stderr = ""
+    stderr = " 2> /dev/null" unless options[:test]
+    
+    case hash["language"]
+    when "ruby"
+      command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg} #{stderr}"
+      command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg} #{stderr}"
+    when "python"
+      if(File.directory?("#{dir}/vEnv"))
+        command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg} #{stderr}"
+      else
+        command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg} 2> /dev/null"
+      end
+    when "js"
+#      command = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS}  \"#{full_script}\" --info"
+      command = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr}"
+    else
+      progress.error "unsupported language: #{hash["language"]}" if progress
+      return nil
+    end
+
+    results = Zillabyte::Command::Flows.get_info(command, info_file, options)
+
+    begin
+      meta = JSON.parse(results.split("\n").first.strip)   # Throws error if invalid json
+    rescue Exception => e
+      progress.error "the underlying flow did not return a valid result.  command: #{command} error: #{e.message}" if progress
+      exit
+    end
+      
+    
+    # Merge with the new info
+    return hash.merge(meta)
+    
+  end
+  
+
+end

data/lib/zillabyte/api/locks.rb

@@ -0,0 +1,4 @@
+module Zillabyte
+	module Lock
+	end
+end

data/lib/zillabyte/api/logs.rb

@@ -0,0 +1,32 @@
+require 'net/http'
+require 'open-uri'
+
+class Zillabyte::API::Logs < Zillabyte::API::Base
+
+
+  # GET /flows/ID/logs
+  def get(flow_id, operation_id, options = {})
+    
+    options = {
+    }.merge(options)
+    
+    since = ""
+    if options[:since]
+      since = "?since=#{URI::encode(options[:since])}"
+    end
+    
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{flow_id}/logs/#{operation_id}#{since}"
+    )
+
+    res.body
+
+  end
+  
+  
+  
+  
+
+end

data/lib/zillabyte/api/metrics.rb

@@ -0,0 +1,48 @@
+class Zillabyte::API::Metrics < Zillabyte::API::Base
+
+
+  # GET /metrics
+  def list(options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+    
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/metrics"
+    )
+    
+    res.body
+
+  end
+  
+  
+  
+  # GET /metrics
+  def show(id, options = {}, block = nil)
+    
+    
+    if block
+      res = @api.request(
+        :expects  => 200,
+        :method   => :get,
+        :path     => "/metrics/#{id}?#{options.to_query}",
+        :response_block => block
+      )
+    else
+      res = @api.request(
+        :expects  => 200,
+        :method   => :get,
+        :path     => "/metrics/#{id}?#{options.to_query}",
+      )
+      
+    end
+
+    res.body
+    
+  end
+  
+
+end

data/lib/zillabyte/api/queries.rb

@@ -0,0 +1,58 @@
+class Zillabyte::API::Queries < Zillabyte::API::Base
+
+
+  # GET /sxp
+  def sxp(expression, options = {})
+
+    options = {
+      'sxp' => expression
+    }.merge(options)
+    
+    res = request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/query_sxp",
+      :body     => options.to_json
+    )
+    
+    res.body
+
+  end
+
+
+
+  # GET /sql
+  def sql(expression, options = {})
+
+    options = {
+      'sql' => expression
+    }.merge(options)
+    
+    res = request(
+      :expects  => 200,
+      :method   => :post,
+      :path     => "/query_sql",
+      :body     => options.to_json
+    )
+    
+    res.body
+
+  end
+  
+  
+
+  def agnostic(expression, options = {})
+    case expression
+    when Array
+      return sxp(expression, options)
+    when String
+      return sql(expression, options)
+    else
+      throw "unknown expression type: #{expression}"
+    end
+  end
+  
+
+
+
+end

data/lib/zillabyte/api/semantic_tags.rb

@@ -0,0 +1,24 @@
+
+class Zillabyte::API::SemanticTags < Zillabyte::API::Base
+
+
+  # GET /semantic_tags
+  def list(options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+    
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/semantic_tags"
+    )
+    
+    res.body
+
+  end
+  
+  
+
+end

data/lib/zillabyte/api/settings.rb

@@ -0,0 +1,8 @@
+module Zillabyte
+  class API
+#    PY_VIRTUAL_ENV_DIR = "/tmp/vEnv"
+    CASPERJS_BIN = "/usr/local/bin/casperjs"
+    API_CLIENT_JS = File.join(ENV['HOME'], "zb1/api.client.js/api.coffee")
+    NODEJS_BIN = "/usr/local/bin/node"
+  end
+end

data/lib/zillabyte/api/sources.rb

@@ -0,0 +1,28 @@
+class Zillabyte::API::Sources < Zillabyte::API::Base
+
+
+  # GET /sources
+  def list(options = {})
+
+    options = {
+      # TODO
+    }.merge(options)
+    
+    res = @api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/sources"
+    )
+    
+    res.body
+
+  end
+  
+  
+  
+  
+  
+  
+  
+
+end