zillabyte-cli 0.1.1 → 0.1.2

data/lib/zillabyte/cli/keys.rb

@@ -15,11 +15,10 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
 
   # keys:add NAME [KEY]
   #
-  #  add a key for the current user
+  # Add a key for the current user.
+  # If no KEY is specified, will try to find ~/.ssh/id_rsa.pub.
   #
-  #  if no KEY is specified, will try to find ~/.ssh/id_rsa.pub
-  #
-  #  --output_type OUTPUT_TYPE # The output format type
+  # --output_type OUTPUT_TYPE                   # Specify an output format type i.e. json #HIDDEN
   #
   def add()
     type = options[:output_type] || nil
@@ -43,9 +42,9 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
 
   # keys:show NAME
   #
-  #  show a key for the current user
+  # Show a key for the current user.
   #
-  #  --output_type OUTPUT_TYPE # The output format type
+  # --output_type OUTPUT_TYPE                   # Specify an output format type i.e. json #HIDDEN
   #
   def show()
     type = options[:output_type] || nil
@@ -64,11 +63,11 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
 
   # keys:remove [NAME]
   #
-  #  remove a key from the current user
+  # Remove a key from the current user.
   #
-  #  --output_type OUTPUT_TYPE # The output format type
-  #  -l, --long                # display extended information for each key
-  #  -f, --force               # Force -- remove key without prompting
+  # -l, --long                                  # Display extended information for each key
+  # -f, --force                                 # Force -- remove key without prompting
+  # --output_type OUTPUT_TYPE                   # Specify an output format type i.e. json #HIDDEN
   #
   def remove()
     type = options[:output_type] || nil
@@ -127,10 +126,11 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
 
   # keys
   #
-  #  display keys for the current user
+  # Display keys for the current user.
+  #
+  # -l, --long                                  # Display extended information for each key
+  # --output_type OUTPUT_TYPE                   # Specify an output format type i.e. json #HIDDEN
   #
-  #  --output_type OUTPUT_TYPE # The output format type
-  #  -l, --long                # display extended information for each key
   def index()
     type = options[:output_type] || nil
     long = options[:long]
@@ -144,9 +144,9 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
 
   # keys:clear
   #
-  #  remove all authentication keys from the current user
+  # Remove all authentication keys from the current user.
   #
-  #  --output_type OUTPUT_TYPE # The output format type
+  # --output_type OUTPUT_TYPE                   # Specify an output format type i.e. json #HIDDEN
   #
   def clear()
     type = options[:output_type] || nil
@@ -159,7 +159,10 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
     end
   end
 
+
   private
+
+
   def format_keys(keys, long, type)
     outlong = long || type == "json"
     rows = keys.map() { |row|

data/lib/zillabyte/cli/nuke.rb

@@ -7,14 +7,12 @@ require "zillabyte/common"
 #
 class Zillabyte::Command::Nuke < Zillabyte::Command::Base
 
-
-  
   
   # apps:nuke
   #
-  # destroys all state associated with the current user
+  # Destroys all state associated with the current user.
   # 
-  # --force # this option must always be given
+  # --force                                     # This option must always be given
   #
   def index
     
@@ -34,9 +32,9 @@ class Zillabyte::Command::Nuke < Zillabyte::Command::Base
     if options[:output_type] == "json"
       display res.body.to_json
     else
-      display "Nuked #{res.body['deleted_apps']} app(s) and #{res.body['deleted_apps']} relations(s)"
+      display "Nuked #{res.body['deleted_apps']} app(s) and #{res.body['deleted_apps']} dataset(s)"
     end
     
   
   end
-end
+end

data/lib/zillabyte/cli/query.rb

@@ -9,12 +9,13 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
 
   # query:sxp EXPRESSION
   #
-  # executes queries against the zillabyte corpus
+  # Executes SXP queries against the zillabyte corpus.
   #
-  # -o, --offset OFFSET  # skips to the offset (default: 0)
-  # -l, --limit LIMIT    # sets the result limit (default: 20)
-  # --output_type OUTPUT_TYPE          # the output format type i.e json
-  # -t, --tail TAIL      # continuously watches for new results
+  # -o, --offset OFFSET                         # Skips to the offset [default: 0]
+  # -l, --limit LIMIT                           # Sets the result limit [default: 20]
+  # -t, --tail TAIL                             # Continuously watches for new results
+  # --meta                                      # Show meta columns (since, confidence, source)
+  # --output_type OUTPUT_TYPE                   # Specify an output type i.e. json #HIDDEN
   #
   #Examples:
   #
@@ -26,6 +27,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
     opts[:offset] = options[:offset] || 0
     opts[:limit] = options[:limit] || 10
     type = options[:output_type]
+    show_meta = options[:meta] || false
     tail = options[:tail] || false
     expression = options[:expression] || shift_argument 
     opts[:since] = options[:since]
@@ -79,6 +81,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
       headings = []
       if res['results'].first
         headings = res['results'].first.keys
+        META_COLUMNS.each {|c| concrete_headings.delete c} if (!show_meta)
         rows = res['results'].each do |obj|
           new_row = headings.map do |heading|
             obj[heading]
@@ -110,14 +113,15 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
 
   # query:sql EXPRESSION
   #
-  # executes queries against the zillabyte corpus
+  # Executes SQL queries against the zillabyte corpus.
   #
-  # -o, --offset OFFSET  # skips to the offset (default: 0)
-  # -l, --limit LIMIT    # sets the result limit (default: 20)
-  # -t, --tail TAIL      # continuously watches for new results
-  # -s, --since SINCE    # newer records since
-  # --no_truncation      # doesn't truncate long strings
-  # --output_type OUTPUT_TYPE          # The result display type
+  # -o, --offset OFFSET                         # Skips to the offset [default: 0]
+  # -l, --limit LIMIT                           # Sets the result limit [default: 20]
+  # -t, --tail TAIL                             # Continuously watches for new results
+  # -s, --since SINCE                           # Grab newer records since SINCE
+  # --no_truncation                             # Doesn't truncate long strings
+  # --meta                                      # Show meta columns (since, confidence, source)
+  # --output_type OUTPUT_TYPE                   # Specify an output type i.e. json #HIDDEN
   #
   #Examples:
   #
@@ -132,6 +136,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
     tail = options[:tail] || false
     expression = options[:expression] || shift_argument 
     opts[:since] = options[:since]
+    show_meta = options[:meta] || false
     type = options[:output_type]
     seen = {}
     
@@ -183,6 +188,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
       filtered = []
       if res['rows'].first
         concrete_headings = res['rows'].first.keys
+        META_COLUMNS.each {|c| concrete_headings.delete c} if (!show_meta)
         concrete_headings.each do |ch|
           has_alias = false
           col_aliases.each do |al|
@@ -242,9 +248,8 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
 
   # query:pull QUERY FILE
   #
-  # executes a queries and downloads the results to FILE
+  # Executes a query and downloads the results to FILE.
   #
-  # pulls query data to OUTPUT.gz
   def pull
 
     query = options[:query] || shift_argument
@@ -264,43 +269,12 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
     )
     res = res.body
     
-    if(res["uri"])
-      display "Waiting for download." if type.nil?
-      File.open(file, "w") do |f|
-        f.write open(res["uri"]).read
-      end
-    elsif(res["s3_credentials"])
-      display "Pulling query results." if type.nil?
-      s3 = AWS::S3.new(res["s3_credentials"])
-      bucket = s3.buckets[res["s3_bucket"]]
-      obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
-
-      while(true)
-        keys = obj.collect(&:key)
-        if keys.length > 0 and keys.include?("#{res["s3_file_key"]}/manifest")
-          display "Starting to write to file..." if type.nil?
-          File.open(file, "w") do |f|
-            obj.each do |o|
-              if o.key == "#{res["s3_file_key"]}/manifest"
-                next
-              end
-              f.write(o.read)
-            end
-          end
-          break
-        else
-          sleep(3)
-          obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
-        end
-      end
-    else
-      throw "bad response: #{res}"
-    end
+    handle_downloading_manifest(file, res, type)
 
     if type == "json"
       display "{}"
     else
-      display "results downloaded to file #{file}"
+      display "finished downloading results to file"
     end
     
   end
@@ -310,8 +284,10 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
 
   # query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
   #
-  # pulls query data to S3_BUCKET/FILE_KEY/part***.gz
-  # --output_type OUTPUT_TYPE       # specify an output format type i.e. json
+  # Executes a query and pulls data to S3_BUCKET/FILE_KEY/part***.gz.
+  #
+  # --output_type OUTPUT_TYPE                   # Specify an output type i.e. json #HIDDEN
+  #
   def pull_to_s3
 
     query = options[:query] || shift_argument
@@ -337,7 +313,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
       display {}.to_json
     else
       display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
-      display "if the relation is large, this may take a while, please check your s3 account after a few minutes"
+      display "if the dataset is large, this may take a while, please check your s3 account after a few minutes"
     end
   end
   alias_command "query:pull:s3", "query:pull_to_s3"

data/lib/zillabyte/cli/repl.rb

@@ -7,9 +7,11 @@ class Zillabyte::Command::Repl < Zillabyte::Command::Base
 
   # repl
   #
-  # start a console session for zillabyte
-  # --quiet # HIDDEN
-  # --history HISTORY# HIDDEN hack to allow history for readline
+  # Start a console session for zillabyte.
+  #
+  # --quiet                                     # HIDDEN
+  # --history HISTORY                           # HIDDEN hack to allow history for readline
+  #
   def index
     if !options[:quiet]
       Zillabyte::Command.run("version")
@@ -29,7 +31,7 @@ class Zillabyte::Command::Repl < Zillabyte::Command::Base
         Readline::HISTORY << his
       end
     end
-    # TODO: Add tab completion for basic commands, app/relation names etc.
+    # TODO: Add tab completion for basic commands, app/dataset names etc.
     buf = ""
     while cmd = Readline.readline(prompt, true)
       if cmd && cmd.length > 0

data/lib/zillabyte/cli/templates/apps/ruby/README.md

@@ -0,0 +1,123 @@
+# Getting Started with Zillabyte Apps
+
+Zillabyte Apps are chunks of code that can scalably analyze almost any type of data.  You develop & test the app locally and then push to the Cloud when you are ready to scale. 
+
+Apps are fundamentally comprised of inputs (`sources`), outputs (`sinks`), and zero or more in-between `operations`.  This style of programming is known as the [Pipe Programming Paradigm](http://blog.zillabyte.com/2014/05/14/the-pipe-programming-paradigm/) and it allows Zillabyte to scale your App in the cloud.
+
+To learn more about Zillabyte Apps, please take a few moments and visit [docs.zillabyte.com](http://docs.zillabyte.com/)
+
+## Step 1: Understand your Data Options
+
+`Sources` are how you get data into your App.  
+
+#### Source from Data Inside Zillabyte 
+
+You can store your datasets inside Zillabyte.  In this regard, Zillabyte acts like a large database, ready to feed your App.  Consider the following: 
+
+```ruby
+require "zillabyte"
+
+# Create an app
+app = Zillabyte.app("hello_world")
+
+# Source from the 'web_pages' dataset
+stream = app.source("web_pages")
+
+# ...
+```
+
+The above [example](http://docs.zillabyte.com/quickstart/hello_world/) will source data from the 'web_pages' dataset.  This particular dataset happens to be an [Open Dataset](http://zillabyte.com/data) and is open to all users.  
+
+You are of course free to create private datasets that will not be accessible to the public.  To do so, use the `zillabyte data` commands. 
+
+```bash
+$ zillabyte data --help
+```
+
+ 
+#### Other Source Options
+
+You are free to source from other datasets outside of Zillabyte.  To do so, please see our documentation on [docs.zillabyte.com](http://docs.zillabyte.com).
+
+
+## Step 2: Build & Test your App
+
+Once your app is built, you will want to test it locally.  The `zillabyte test` is intended to help you identify bugs before submitting to the cloud.  `zillabyte test` simulates the cloud environment on your local machine.  In your App's directory, run the following:
+
+```bash
+$ # in your app's directory
+$ zillabyte test
+
+inferring your app details...
+            app name: hello_world
+        app language: ruby
+====================  operation #0
+                name: source_1
+                type: source
+#... 
+```
+
+
+## Step 3: Submit your App to the Zillabyte Cloud 
+
+Once you are confident in your code, you can push to the cloud and run at scale.  If you wish to push the entire directory, run the following: 
+
+```bash
+$ # in your app's directory
+$ zillabyte push
+packaging directory... done
+uploading 31744 bytes... 
+#...
+```
+
+The `zillabyte push` command may take a few minutes to complete.  During this time, you will see log output coming from the cloud servers.  You will notice your operations are being parallelized automatically.  For example, if you had an operation named `each_1`, you will see it parallelized into `each_1.1`, `each_1.2`, `each_1.3`, and so forth.  
+
+
+## Step 4: Examine the Your App's Logs
+
+The `zillabyte push` will stop streaming logs once the push sequence is complete.  If you wish to examine your app's logs further, run the `zillabyte logs` command
+
+```bash
+$ zillabyte logs
+
+Retrieving logs for flow #hello_world...please wait...
+2014-07-01T05:02:40.923+00:00 flow_1210[app] - [STARTUP] new app created.
+2014-07-01T05:02:42.563+00:00 flow_1210[app] - [STARTUP] registering app.
+2014-07-01T05:02:46.083+00:00 flow_1210[app] - [STARTUP] Beginning app deployment
+2014-07-01T05:02:48.915+00:00 flow_1210[flow] - [STARTUP] Pulling the code from our servers...
+2014-07-01T05:02:48.949+00:00 flow_1210[app] - [STARTUP] Initializing the app's environment...this might take a while.
+2014-07-01T05:03:06.775+00:00 flow_1210[flow] - [STARTUP] Performing the initial setup...
+```
+
+The logs will stream indefinitely.  To terminate, hit `ctrl+c`.
+
+## Step 5: Optionally Kill & Resubmit your App 
+
+If you notice a problem with your app, you can make the code change and `zillabyte push` right away.  Only one instance of your app can be active at a time, so Zillabyte will automatically terminate your existing app before proceeding. 
+
+If you wish to kill your app without re-pushing, simple run the `kill` command. 
+
+```bash
+$ # in your app's directory
+$ zillabyte apps:kill
+```
+
+## Step 6: Download Results 
+
+When your app is done processing, you will most likely have data to download from the cloud.  This can be accomplished with the `zillabyte data:pull` command.  For example: 
+
+```bash
+# list my datasets
+$ zillabyte data
++------+---------------------+
+| id   | name                |
++------+---------------------+
+| 1516 | hello_worlds        |
++------+---------------------+
+```
+
+```bash
+# download the entire dataset to a local file
+$ zillabyte data:pull hello_worlds my_local_file
+```
+

data/lib/zillabyte/cli/templates/apps/ruby/zillabyte.conf.yaml

@@ -1,2 +1,5 @@
+# This file is required by Zillabyte to understand your app.  Most fields should
+# be self-explanatory.  For further information, please visit docs.zillabyte.com or
+# shoot us a note at support@zillabyte.com. 
 language: ruby
 script: app.rb

data/lib/zillabyte/cli/templates/ruby/Gemfile.lock

@@ -0,0 +1,81 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (3.2.17)
+      activemodel (= 3.2.17)
+      activesupport (= 3.2.17)
+      builder (~> 3.0.0)
+      erubis (~> 2.7.0)
+      journey (~> 1.0.4)
+      rack (~> 1.4.5)
+      rack-cache (~> 1.2)
+      rack-test (~> 0.6.1)
+      sprockets (~> 2.2.1)
+    activemodel (3.2.17)
+      activesupport (= 3.2.17)
+      builder (~> 3.0.0)
+    activesupport (3.2.17)
+      i18n (~> 0.6, >= 0.6.4)
+      multi_json (~> 1.0)
+    ascii_charts (0.9.1)
+    aws-sdk (1.33.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+      uuidtools (~> 2.1)
+    builder (3.0.4)
+    chronic (0.10.2)
+    colorize (0.7.2)
+    erubis (2.7.0)
+    excon (0.33.0)
+    hike (1.2.3)
+    i18n (0.6.9)
+    indentation (0.1.1)
+    journey (1.0.4)
+    json (1.8.1)
+    mime-types (2.2)
+    mini_portile (0.5.3)
+    multi_json (1.9.2)
+    netrc (0.7.7)
+    nokogiri (1.6.1)
+      mini_portile (~> 0.5.0)
+    rack (1.4.5)
+    rack-cache (1.2)
+      rack (>= 0.4)
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    sprockets (2.2.2)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    terminal-table (1.4.5)
+    tilt (1.4.1)
+    time_difference (0.3.2)
+      activesupport
+    uuidtools (2.1.4)
+    zillabyte (0.0.20)
+      zillabyte-cli (~> 0.0.20)
+    zillabyte-cli (0.0.20)
+      actionpack (~> 3.2.13)
+      activesupport (~> 3.2.11)
+      ascii_charts (~> 0.9.1)
+      aws-sdk (~> 1.33.0)
+      bundler (~> 1.3)
+      chronic (~> 0.10)
+      colorize (~> 0.6)
+      excon (~> 0.31)
+      indentation (~> 0.1)
+      mini_portile (~> 0.5.0)
+      multi_json (~> 1.0)
+      netrc (~> 0.7.7)
+      rest-client (~> 1.6.1)
+      terminal-table (~> 1.4)
+      time_difference
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  zillabyte

data/lib/zillabyte/helpers.rb

@@ -153,6 +153,28 @@ module Zillabyte
       return nil
     end
     
+    
+    def handle_downloading_manifest(file, res, type = nil)
+      if res["manifest"]
+        res["manifest"].each_with_index do |uri, index|
+        
+          # Do we have multiple files? then suffix with numbers... 
+          if res["manifest"].size <= 1
+            shard_file = file
+          else
+            shard_file = "#{File.basename(file, File.extname(file))}__#{'%04i' % index}#{File.extname(file)}" 
+          end
+        
+          display "downloading #{shard_file}" unless type == "json"
+          
+          File.open(shard_file, "w") do |f|
+            f.write open(uri).read
+          end
+        end
+      else
+        error "expected remote server to return file manifest"
+      end
+    end
 
   end
 end

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

data/zillabyte-cli.gemspec

@@ -37,7 +37,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "chronic", "~> 0.10"
   spec.add_dependency "ascii_charts", "~> 0.9.1"
   spec.add_dependency "indentation", "~> 0.1"
-  spec.add_dependency "aws-sdk", "~> 1.33.0"
   spec.add_dependency "time_difference"
   spec.add_dependency "mkfifo"