zillabyte-cli 0.0.9

data/lib/zillabyte/cli/relations.rb

@@ -0,0 +1,326 @@
+require "zillabyte/cli/base"
+require "csv"
+
+# manage custom relations
+#
+class Zillabyte::Command::Relations < Zillabyte::Command::Base
+
+  # relations
+  #
+  #
+  def index
+    self.list
+  end
+
+
+  # relations:list
+  #
+  # lists your custom relations
+  #
+  def list
+    
+    response = api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/relations"
+    )
+      
+    headings = []
+    rows = response.body.map do |row|
+      headings = row.keys if headings.size == 0
+      row["columns"] = row["columns"].map{|c| c['type']}.join(',')
+      vals = row.values_at *headings
+      vals 
+    end
+    
+    display "relations:\n" + Terminal::Table.new(:headings => headings, :rows => rows).to_s
+    display "Total number of relations: "+rows.length.to_s
+    
+    
+  end
+  
+  
+  alias_command "list", "relations:list"
+  alias_command "relations:list", "relations:list"
+  alias_command "relation:list", "relations:list"
+
+
+
+  # relations:status RELATION
+  #
+  # shows status of a relation
+  #
+  def status
+    
+    name = options[:relation_name] || shift_argument
+    error "no name given" if name.nil?
+
+    res = api.request(
+      :path => "/metrics/#{name}",
+      :method => :get,
+      :expects => 200
+    )
+    
+    if res['error']
+      error res['error_message']
+    end
+    
+    data = res.body.first
+    
+    if data.nil?
+      error "metric not found"
+    end
+    
+    data = data["datapoints"]
+    
+    last_date = data.last(30).first[1]
+    formatted = []
+    i = 0
+    data.each do |tup|
+      value = tup[0]
+      date = tup[1] - last_date
+      if date > 0 
+        formatted <<  [i, value || 0]
+        i += 1
+      end
+    end
+    
+    display "#{name}:"
+    display AsciiCharts::Cartesian.new(formatted, :bar => true, :hide_zero => true).draw
+    
+    
+  end
+
+
+
+  # relations:create NAME
+  #
+  # creates a new relation
+  #
+  # --schema SCHEMA # a JSON array of column types
+  # --semantics SEMANTICS # a JSON array of semantics within the relation
+  # --public # Make the relation public
+  #
+  def create
+    
+    name = options[:name] || shift_argument
+    error "no name given" if name.nil?
+
+    schema = JSON.parse(options[:schema]) if options[:schema]
+    semantics = JSON.parse(options[:semantics]) if options[:semantics]
+    is_public = options[:public] || false
+   
+    hash = set_relation_properties(schema,semantics,is_public) 
+    res = api.data.create name, hash
+    
+    if res['error']
+      display "error: #{res['error_message']}"
+    else
+      display "relation ##{res['id']} #{res['action']}. size: #{res['size']}"
+    end
+    
+  end
+
+  #alias_command "upload", "data:upload"
+  alias_command "relations:upload", "relations:create"
+  alias_command "relation:upload", "relations:create"
+  
+  
+  
+  # relations:append ID FILE
+  #
+  # adds data to an existing relation
+  #
+  # --type TYPE # interpret file as type [csv, xlsx]
+  #
+  def append
+
+    id = options[:id] || shift_argument
+    file = options[:file] || shift_argument
+    type = options[:type]
+
+    error "no id given" if id.nil?
+    error "no id given" if file.nil?
+    type ||= File.extname(file).gsub(".", "")
+
+    dataset = self.api.data.get(id)
+    rows = sanity_check_file(file,type,dataset)
+   
+    #res = self.api.data.append(id, {:row => rows})
+    res = self.api.data.append(id, {:file => file})
+    display "relation #{id} appended"
+    
+  end  
+  
+  
+  # relations:show ID
+  #
+  # shows raw data in a relation. run 'queries' for more elaborate functionality
+  #
+  #  --no_truncation   # don't truncate long strings
+  #
+  def show
+    
+    name = options[:name] || shift_argument
+    error "no id given" if name.nil?
+    
+    res = self.api.data.get(name)
+    
+    if res["rows"].size > 0
+      
+      headings = res["rows"].first.keys
+      headings.delete("id")
+      table = Terminal::Table.new :headings => headings    
+      res["rows"].each do |obj|
+        
+        new_row = headings.map do |heading|
+          if options[:no_truncation]
+            obj[heading]
+          else
+            if obj[heading].to_s.size > 30
+              obj[heading].to_s[0..30] + "..."
+            else
+              obj[heading]
+            end
+          end
+        end
+        
+        table.add_row new_row
+        
+      end
+      display table.to_s
+      
+    else
+      
+      display "empty relation"
+      
+    end
+    
+  end
+  
+  alias_command "relations:show", "relations:show"
+  alias_command "relation:show", "relations:show"
+  
+  # relations:test NAME FILE
+  #
+  # creates test relation NAME and uploads FILE to NAME
+  #
+  def test
+    name = options[:name] || shift_argument
+    error "no name given" if name.nil?
+    
+    file = options[:file] || shift_argument
+    error "no file given" if file.nil?
+
+    schema = options[:schema]
+    relations = options[:relations]
+    is_public = options[:public] || false
+   
+    hash = set_dataset_properties(schema,relations,is_public) 
+    res = api.data.create name, hash
+
+    type = options[:type]
+    type ||= File.extname(file).gsub(".", "")
+
+    id = res['id']
+    dataset = self.api.data.get(id)
+    rows = sanity_check_file(file,type,dataset)
+
+    #res = self.api.data.append(id, {:row => rows,:for_test => 1})
+    res = self.api.data.append(id, {:file => file,:for_test => 1})
+    display "relation (for test) #{id} appended"
+  end
+
+private
+
+  def set_dataset_properties(schema,relations,is_public)
+    valid_types=["STRING", "NUMERIC", "FLOAT"]
+    if schema.nil?
+      schema = []
+      
+      # Get the column types (required)
+      index = 0
+      while true
+        
+        display "type for column v#{index} [#{valid_types.join(', ')}] (empty when done):", false
+        type = ask.upcase
+        break if type == "" || type.nil?
+        unless valid_types.member?(type)
+          warn "invalid type: #{type}"
+          next
+        end   
+        index += 1           
+        schema << type
+      end
+      
+      display "using schema: #{schema.to_json}"
+      
+    end
+    
+    if relations.nil?
+      
+      # Get the semantics (inner relations)
+      relations = []
+      display "would you like to define relations on your dataset? (yes/No): ", false
+      if ['y', 'yes'].include?(ask.downcase)
+        
+        # Display the first bit of the taple
+        display Terminal::Table.new(:rows => [schema], :headings => schema.size.times.map{|i| "v#{i}"}).to_s
+        
+        # Start collection relations 
+        while true
+          display "relation predicate (ex: \"(company v0)\", \"(coordinates v1 v2)\"): ", false
+          input = ask
+          break if input.nil? || input == ""
+          
+          pred = input.gsub("(","").gsub(")", "").split(/\s+/)
+          pred_name = pred[0]
+          pred_cols = pred[1..-1]
+          pred_cols.each_with_index do |col, pred_index|
+            m = /v(\d+)/.match(col)
+            if m.nil?
+              warn "the predicate variables must be of the form \"v[1-10]\""
+              next
+            end
+            
+            column = m[1].to_i
+            if column >= schema.size
+              warn "the variable #{col} is out of range"
+              next
+            end
+            
+            # Otherwise, successful 
+            relations[column] ||= []
+            relations[column] << "#{pred_name}.#{pred_index}"
+          end
+          
+        end
+        
+        # Show them what the relation looks like
+        display "using relation: #{relations.to_json}"
+        
+      end
+    end
+    {:schema => schema, :relations => relations, :public => is_public}
+  end
+
+  def sanity_check_file(file,type,dataset)
+    rows = []
+    
+    case type
+    when "csv"
+      CSV.foreach(file) do |row|
+        if row.size != dataset['columns'].size
+          puts "relation expect #{dataset['columns'].size} column(s). Found a row with #{row.size}::\n #{row}" 
+        else
+          rows << row
+        end
+      end
+      
+    else
+      error "unsupported type: #{type}"
+      
+    end
+    rows
+  end
+end

data/lib/zillabyte/cli/sources.rb

@@ -0,0 +1,37 @@
+# require "zillabyte/cli/base"
+# require "csv"
+# 
+# # data sources
+# #
+# class Zillabyte::Command::Sources < Zillabyte::Command::Base
+# 
+# 
+#   # sources
+#   #
+#   #
+#   def index
+#     self.list
+#   end
+# 
+# 
+#   # sources:list
+#   #
+#   # lists public and custom data sources
+#   #
+#   def list
+#     
+#     headings = []
+#     sources = api.sources.list.map do |row|
+#       headings = row.keys if headings.size == 0
+#       row.values_at *headings
+#     end
+#     display "data sources:\n" + Terminal::Table.new(:headings => headings, :rows => sources).to_s
+#     
+#   end
+#   
+#   
+#   alias_command "sources", "sources:list"
+# 
+#   
+# 
+# end

data/lib/zillabyte/cli/templates/js/simple_function.js

@@ -0,0 +1,33 @@
+var zillabyte = require('zillabyte');
+
+function prep(controller) {
+}
+
+/** 
+ * This is the heart of your algorithm.  It's processed on every
+ * web page.  This algorithm is run in parallel on possibly hundreds
+ * of machines.
+ */
+function exec(controller, tuple) {
+  if(tuple.values["html"].indexOf("hello world") !== -1) {
+    controller.emit("has_hello_world",{"url":tuple.values["url"]});
+  }
+}
+
+zillabyte.simple_function({
+/** 
+ * This directive instructs zillabyte to give your function every
+ * web page in our known universe.  Your function will have access
+ * to two fields: URL and HTML
+ */
+  matches: "select * from web_pages",
+
+/**
+ * This directive tells Zillabyte what kind of data your function
+ * produces.  In this case, we're saying we will emit a tuple that
+ * is one-column wide and contains the field 'URL' 
+ */
+  emits: [["has_hello_world", [{"url":"string"}]]],
+  prepare: prep,
+  execute: exec
+});

data/lib/zillabyte/cli/templates/js/zillabyte.conf.yaml

@@ -0,0 +1,2 @@
+language: js
+script: simple_function.js

data/lib/zillabyte/cli/templates/python/requirements.txt

@@ -0,0 +1,7 @@
+# Indicate any required libraries here
+# Default libraries installed:
+# - numpy 1.8.0
+# - scipy 0.13.0
+# For all other libraries, enter one per line in the following format: LIBRARY==VERSION, e.g. numpy==1.8.0
+# MAKE SURE LIBRARY DEPENDENCIES ARE ALSO INCLUDED, FOR EXAMPLE IF YOUR LIBRARY NEEDS MATPLOTLIB FUNCTIONS, THEN MATPLOTLIB
+# MUST ALSO BE LISTED BELOW EVEN IF YOU DO NOT CALL ANY OF ITS FUNCTIONS EXPLICITLY.

data/lib/zillabyte/cli/templates/python/simple_function.py

@@ -0,0 +1,27 @@
+import zillabyte
+
+def prep(controller):
+  return
+
+# This is the heart of your algorithm.  It's processed on every
+# web page.  This algorithm is run in parallel on possibly hundreds
+# of machines.
+def execute(controller, tup):
+  if("hello world" in tup.values["html"]):
+    controller.emit("has_hello_world",{"url":tup.values["url"]})
+  return
+
+zillabyte.simple_function(\
+  # This directive instructs zillabyte to give your function every
+  # web page in our known universe.  Your function will have access
+  # to two fields: URL and HTML
+  matches = "select * from web_pages", \
+
+  # This directive tells Zillabyte what kind of data your function
+  # produces.  In this case, we're saying we will emit a tuple that
+  # is one-column wide and contains the field 'URL' 
+  emits = [["has_hello_world", [{"url":"string"}]]], \
+
+  prepare = prep, \
+  execute = execute\
+)

data/lib/zillabyte/cli/templates/python/zillabyte.conf.yaml

@@ -0,0 +1,4 @@
+language: python
+script: simple_function.py
+ignore_files:
+  - ./vEnv

data/lib/zillabyte/cli/templates/ruby/Gemfile

@@ -0,0 +1,3 @@
+
+gem "zillabyte", :path => "~/zb1/multilang/"
+gem "zillabyte-cli", :path => "~/zb1/cli/"

data/lib/zillabyte/cli/templates/ruby/simple_function.rb

@@ -0,0 +1,36 @@
+require 'zillabyte'
+
+Zillabyte.simple_function do |fn|
+  
+  # This directive instructs zillabyte to give your function every 
+  # web page in our known universe.  Your function will have access
+  # to two fields: URL and HTML
+  fn.matches "select * from web_pages"
+  
+  # This directive tells Zillabyte what kind of data your function
+  # produces.  In this case, we're saying we will emit a tuple that 
+  # is one-column wide and contains the field 'URL'
+  fn.emits   [["has_hello_world", [{"URL"=>:string}]]]
+
+  
+  # This is the heart of your algorithm.  It's processed on every
+  # web page.  This algorithm is run in parallel on possibly hundreds
+  # of machines. 
+  fn.execute do |controller, tuple|
+    
+    # get the fields
+    url = tuple['v0']
+    html = tuple['v1']
+    
+    # does this page contains the 'hello world' phrase? If so, emit it
+    if html.include?('hello') or html.include?('world')
+      controller.emit("URL" => url)
+    end
+    
+  end
+
+end
+  
+  
+
+

data/lib/zillabyte/cli/templates/ruby/zillabyte.conf.yaml

@@ -0,0 +1,2 @@
+language: ruby
+script: simple_function.rb

data/lib/zillabyte/cli/version.rb

@@ -0,0 +1,11 @@
+require "zillabyte/cli/base"
+
+# display version number
+#
+class Zillabyte::Command::Version < Zillabyte::Command::Base
+  
+  def index(*direct_args)
+    display "Zillabyte CLI Version #{Zillabyte::CLI::VERSION}"
+  end
+
+end

data/lib/zillabyte/cli/zillalogs.rb

@@ -0,0 +1,86 @@
+# require "zillabyte/cli/base"
+# require "rest_client"
+# require "json"
+# 
+# # manage custom zillalogs
+# #
+# class Zillabyte::Command::Zillalogs < Zillabyte::Command::Base
+# 
+# 
+#   # zillalogs
+#   #
+#   #
+#   def index
+#     Zillabyte::Command::Help.new.index "zillalogs"
+#   end
+# 
+#   
+# 
+#   # zillalogs:list
+#   #
+#   # list custom zillalogs
+#   #
+#   def list
+#     
+#     rules = api.zillalogs.list.map do |row|
+#       row
+#     end
+#     display "your rules:\n" + Terminal::Table.new(:rows => rules).to_s
+#     
+#   end
+#   
+# 
+# 
+# 
+#   # zillalogs:show ID
+#   #
+#   # shows a zillalog
+#   #
+#   def show
+#   
+#     id = options[:id] || shift_argument
+#     rule = api.zillalogs.find(id)
+#     
+#     if rule['error']
+#       error "#{rule['error_message']}"
+#     else
+#       display rule["zillalog"]
+#     end
+#   
+#   end
+# 
+# 
+# 
+#   
+#   # zillalogs:add NAME [CONTENT]
+#   #
+#   # adds (or updates) a rule
+#   #
+#   def add
+#     
+#     name = options[:name] || shift_argument
+#     error "name is required" unless name
+#     
+#     content = options[:content] || shift_argument
+#     if content.nil?
+#       puts "type ^D to exit"      
+#       content = STDIN.readlines.join("")
+#       display "saving..."
+#     end
+#     
+#     error "content is required" if content.nil?
+#     res = api.zillalogs.save nil, name, content
+#     
+#     if res['error']
+#       display "error: #{res['error_message']}"
+#     else
+#       display "rule ##{res['id']} #{res['action']}"
+#     end
+#     
+#   end
+#   
+#   # alias_command "add", "zillalogs:add"
+#   
+# 
+# 
+# end

data/lib/zillabyte/cli.rb

@@ -0,0 +1,37 @@
+load('zillabyte/helpers.rb') # reload helpers after possible inject_loadpath
+# load('zillabyte/updater.rb') # reload updater after possible inject_loadpath
+
+require 'zillabyte/cli/config'
+require "zillabyte/auth"
+require "zillabyte/command"
+require 'terminal-table'
+require 'chronic'
+require 'ascii_charts'
+
+# workaround for rescue/reraise to define errors in command.rb failing in 1.8.6
+if RUBY_VERSION =~ /^1.8.6/
+  # require('zillabyte-api')
+  require('rest_client')
+end
+
+module Zillabyte::CLI
+
+  extend Zillabyte::Helpers
+
+  def self.start(*args)
+    begin
+      command = args.shift.strip rescue "help"
+      Zillabyte::Command.load
+      Zillabyte::Command.run(command, args)
+    rescue Interrupt
+      `stty icanon echo`
+      error("Command cancelled.")
+    rescue => error
+      raise
+      #display(error)
+      #exit(1)
+    end
+  end
+
+end
+