optica-client 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2707b665c02b7e9fab50ed54997fcd7f6deaba82
+  data.tar.gz: 6a1e0236dd17a745341c62fb12723a325ad89a78
+SHA512:
+  metadata.gz: f1011f391dd4695e3ebd5bc5833697d34e785d93b3510ff13fcd35c936731a4c9800b9a76cd4a2d7512620f2fbd8c2d9cedfe56a7bec06353b1fab1c69d5d759
+  data.tar.gz: 5be5f02624481514824262fd12fbde9d21287623996ee43d1f94d8526a60d0f3bb0f43653e516ed5e44bd90469203f367124f3f7e8314f73a0bb5f716d855522

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/README.md

@@ -0,0 +1,47 @@
+# optica-client
+
+This `optica-client` is a CLI for Airbnb's [Optica](https://github.com/airbnb/optica) service.
+It's command-line name is `optical`.
+
+This is not an official Airbnb product.
+
+## Installation
+
+Install it via Rubygems:
+
+    $ gem install optica-client
+
+You'll want to configure the command-line client to use your Optica instance:
+
+    $ optical -H https://optica.example.com
+
+## Usage
+
+```text
+Usage: optical [options] [FIELD=FILTER] [FIELD2=FILTER2...]
+
+  Fetch host information from Optica, and cache it for 15 minutes. Output the
+  fetched information as single-line JSON hashes, suitable for furhter processing with `jq`.
+
+  FIELD: any optica field; see your optica host for availible fields
+  FILTER: either a bare string, like "optica", or a regex string, like "/^(o|O)ptica?/"
+
+Options:
+    -s, --select a,b,c               Retrieve the given fields, in addition to the defaults
+    -a, --all                        Retrieve all fields (default is just role,id,hostname)
+    -v, --verbose                    Print debug information to STDERR
+    -p, --pretty[=true]              Pretty-print JSON (default true when STDOUT is a TTY)
+    -r, --refresh                    Delete cache before performing request.
+    -h, --host URI                   Optica host (default "https://optica.d.musta.ch")
+    -H, --set-default-host URI       Set the default optica host
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/justjake/optica-client.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "optical"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/optical

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'optica/client/cli'
+exit(Optica::Client::CLI.run(ARGV))

data/lib/optica/client/cli.rb

@@ -0,0 +1,290 @@
+require 'optparse'
+require 'filecache'
+require 'pathname'
+
+require_relative './config'
+require_relative './request'
+require_relative './fetch_json'
+
+module Optica
+  module Client
+    # Command-line interface
+    class CLI
+      # rate-limit when printing huge JSON to a tty
+      TTY_TIMEOUT = 0.00001
+
+      USER_PATH = Pathname.new('~/.optical').expand_path
+      CONFIG_PATH = USER_PATH.join('config.yml')
+      CACHE_ROOT = USER_PATH.join('cache').to_s
+
+      ERR_NOT_FOUND = 4
+      ERR_INVALID = 2
+
+      # 15 min
+      CACHE_MAX_AGE = 15 * 60
+
+      def self.run(argv)
+        new.run(argv)
+      end
+
+      attr_reader :fields, :verbose, :cache, :delete_cache
+
+      def initialize
+        @config = ::Optica::Client::Config.from_file(CONFIG_PATH)
+        @host = nil
+        @fields = []
+        @verbose = false
+        @pretty = data_pipe.tty?
+        @cache = ::FileCache.new(
+          "requests",
+          CACHE_ROOT,
+          CACHE_MAX_AGE,
+          1
+        )
+        @delete_cache = false
+      end
+
+      # Options for the CLI
+      #
+      # @return [OptionParser]
+      def option_parser
+        @option_parser ||= OptionParser.new do |o|
+          o.banner = "Usage: optical [options] [FIELD=FILTER] [FIELD2=FILTER2...]"
+
+          o.separator ''
+          o.separator <<-EOS
+  Fetch host information from Optica, and cache it for 15 minutes. Output the
+  fetched information as single-line JSON hashes, suitable for furhter processing with `jq`.
+
+  FIELD: any optica field; see your optica host for availible fields
+  FILTER: either a bare string, like "optica", or a regex string, like "/^(o|O)ptica?/"
+          EOS
+
+          o.separator ''
+          o.separator 'Options:'
+
+          o.on(
+            '-s',
+            '--select a,b,c',
+            ::Array,
+            'Retrieve the given fields, in addition to the defaults'
+          ) do |fs|
+            @fields.concat(fs)
+          end
+
+          o.on(
+            '-a',
+            '--all',
+            'Retrieve all fields (default is just role,id,hostname)'
+          ) do |all|
+            @fields = nil if all
+          end
+
+          o.on(
+            '-v',
+            '--verbose',
+            'Print debug information to STDERR'
+          ) do |v|
+            @verbose = v
+          end
+
+          o.on('-p', "--pretty[=#{data_pipe.tty?}]", "Pretty-print JSON (default true when STDOUT is a TTY)") do |p|
+            @pretty = p
+          end
+
+          o.on(
+            '-r',
+            '--refresh',
+            'Delete cache before performing request.'
+          ) do |r|
+            @delete_cache = r
+          end
+
+          o.on(
+            '-h',
+            '--host URI',
+            "Optica host (default #{@config.default_host.inspect})"
+          ) do |host|
+            @host = host
+          end
+
+          o.on(
+            '-H URI',
+            '--set-default-host URI',
+            'Set the default optica host'
+          ) do |h|
+            # TODO: move to #run
+            @host = h
+            @config.default_host = h
+            @config.write!
+            log "set default host to #{h}"
+          end
+
+        end
+      end
+
+      # Run the CLI
+      #
+      # @param [Array<String>] argv Command-line args
+      def run(argv)
+        args = option_parser.parse(argv)
+
+        begin
+          filters = args.map { |arg| parse_filter(arg) }.reduce({}, :merge)
+        rescue ArgumentError => err
+          ui_pipe.puts err.message
+          ui_pipe.puts ''
+          ui_pipe.puts option_parser
+          return ERR_INVALID
+        end
+
+        if host.nil?
+          ui_pipe.puts 'No host given.'
+          ui_pipe.puts 'Set the default with -H, or for the invocation with -h.'
+          ui_pipe.puts ''
+          ui_pipe.puts option_parser
+          return ERR_INVALID
+        end
+
+        manage_cache
+
+        req = ::Optica::Client::Request.new(host).where(filters)
+        if fields
+          req.select(*fields)
+        else
+          req.select_all
+        end
+
+        json = fetch(req)
+        output(json['nodes'])
+
+        if json['nodes'].any?
+          # happy!
+          return 0
+        else
+          # none found
+          return ERR_NOT_FOUND
+        end
+      end
+
+      def fetch(req)
+        log "URL:     #{req.root}"
+        log "Filters: #{req.filters}"
+        log "Fields:  #{req.select_all? ? '(all fields)' : req.fields.inspect}"
+        log "GET      #{req.to_uri}"
+
+        key = req.to_uri.to_s
+        cache.get_or_set(key) do
+          fetch_with_progress_bar(req.to_uri)
+        end
+      end
+
+      def output(json)
+        log "got #{json.size} entries"
+
+        use_sleep = false
+        if json.size >= 1000 && data_pipe.tty?
+          log 'reducing output speed to allow Ctrl-C'
+          use_sleep = true
+        end
+
+        json.each do |_ip, node|
+          string = @pretty ? JSON.pretty_generate(node) : JSON.fast_generate(node)
+          data_pipe.puts string
+          # easier Ctrl-C in Tmux
+          sleep TTY_TIMEOUT if use_sleep
+        end
+      end
+
+      def manage_cache
+        # clean up expired stuff
+        cache.purge
+
+        if delete_cache
+          log "deleting cache dir #{CACHE_ROOT}"
+          FileUtils.rm_r(CACHE_ROOT)
+        end
+      end
+
+      def fetch_with_progress_bar(uri)
+        ::Optica::Client.fetch_json(uri) do |_chunk, ratio|
+          view = "Download #{progress_bar(ratio)}"
+
+          ui_pipe.print "\r#{view}"
+          ui_pipe.print "\n" if ratio == 1.0
+        end
+      end
+
+      # Returns a progress bar, as a string.
+      #
+      # Looks kinda like this:
+      # [>>>>>            ] NN.NN%
+      #
+      # @return [String]
+      def progress_bar(ratio)
+        width = 40
+        chars = (width * ratio).to_i
+        start = '['
+        fin = ']'
+        done = '>' * chars
+        to_do = ' ' * (width - chars)
+        percent = format(' %.2f%', ratio * 100)
+        start + done + to_do + fin + percent
+      end
+
+      # Parse a command-line argument into a single Optica filter.
+      #
+      # Filter types:
+      # - exact string match: base case
+      #     attribute=string
+      #
+      # - regex match: filter begins and ends with /
+      #     attribute=/^[rR]egexp?/
+      #
+      # - array match: begins with [, ends with ]
+      #     attribute=[one,two]
+      #
+      # @param string [String]
+      # @return [Hash<String, Any>] a filter hash
+      def parse_filter(string)
+        unless string.include?('=')
+          raise ArgumentError.new("Invalid filter: #{string.inspect}")
+        end
+
+        key, *values = string.split('=')
+        value = values.join('=')
+
+        # parse regex
+        if value[0] == '/' && value[-1] == '/'
+          return { key => /#{value[1...-1]}/ }
+        end
+
+        # parse array-like
+        if value[0] == '[' && value[-1] == ']'
+          return { key => value[1...-1].split(',') }
+        end
+
+        # just a string
+        { key => value }
+      end
+
+      def host
+        @host || @config.default_host
+      end
+
+      def log(msg)
+        if verbose
+          ui_pipe.puts(msg)
+        end
+      end
+
+      def data_pipe
+        $stdout
+      end
+
+      def ui_pipe
+        $stderr
+      end
+    end
+  end
+end

data/lib/optica/client/config.rb

@@ -0,0 +1,45 @@
+require 'yaml'
+
+module Optica
+  module Client
+    # Handles storing and loading configurable settings for our CLI
+    class Config
+      attr_accessor :config_path
+      attr_accessor :default_host
+
+      def self.from_file(path)
+        new(config_path: path.to_s).reload
+      end
+
+      def initialize(hash = {})
+        set_all(hash)
+      end
+
+      def to_h
+        {
+          config_path: config_path.to_s,
+          default_host: default_host.to_s,
+        }
+      end
+
+      def write!
+        data = YAML.dump(to_h)
+        File.write(config_path, data)
+      end
+
+      def reload
+        return self unless File.exist?(config_path)
+
+        data = YAML.load(File.read(config_path)).merge(config_path: config_path)
+        set_all(data)
+        self
+      end
+
+      def set_all(data)
+        data.each do |key, val|
+          self.public_send("#{key}=", val)
+        end
+      end
+    end
+  end
+end

data/lib/optica/client/fetch_json.rb

@@ -0,0 +1,39 @@
+require 'net/http'
+require 'json'
+require 'stringio'
+
+module Optica
+  module Client
+    # Fetch the JSON data at the given URI. Blocks.
+    #
+    # As data is received, yeilds the data chunk (a string) and the completion
+    # ratio (a float).
+    #
+    # @return [Any] the parsed JSON
+    def self.fetch_json(uri)
+      io = StringIO.new
+
+      Net::HTTP.start(
+        uri.host,
+        uri.port,
+        use_ssl: uri.scheme == 'https'
+      ) do |http|
+        req = Net::HTTP::Get.new(uri)
+
+        http.request(req) do |res|
+          file_size = res['content-length'].to_i
+          downloaded = 0.0
+
+          res.read_body do |chunk|
+            io.write chunk
+            downloaded += chunk.size
+            ratio = downloaded / file_size
+            yield(chunk, ratio) if block_given?
+          end
+        end
+      end
+
+      JSON.parse(io.string)
+    end
+  end
+end

data/lib/optica/client/request.rb

@@ -0,0 +1,120 @@
+require 'uri'
+
+module Optica
+  module Client
+    # Request builds URIs to fetch data from Optica. There is no planned support
+    # for posting data to Optica.
+    #
+    # By default a Request will request the minimum number of fields from Optica
+    # to save on response size, using the /roles endpoint. Use the {#fields} or
+    # {#all_fields!} methods to fetch more fields.
+    #
+    # Build your request, then submit it using your HTTP client of choice using
+    # the {#to_uri} method.
+    class Request
+      # @param root_url [String] HTTP(S) URI root of your Optica instance.
+      # @param given_opts [Hash] options
+      # @option given_opts [Array<String>] :fields Additional fields to request.
+      #   see also {#fields} and {#all_fields!}
+      # @option given_opts [Hash] :filters Filters. See {#where}.
+      # @option given_opts [Boolean] :all_fields (false) See {#all_fields!}
+      def initialize(root_url, given_opts = {})
+        opts = {
+            fields: [],
+            filters: {},
+            all_fields: false,
+        }.merge(given_opts)
+
+        @uri = URI(root_url)
+        @fields = opts[:fields]
+        @filters = opts[:filters]
+        @all_fields = opts[:all_fields]
+      end
+
+      def filters
+        @filters.dup
+      end
+
+      def fields
+        @fields.dup
+      end
+
+      # returns the fields that will be fetched using this request, via the
+      # /roles endpoint.
+      #
+      # @see https://github.com/airbnb/optica/blob/master/optica.rb#L15-L24
+
+      # Add more requested fields.
+      #
+      # @param fields [Array<String>]
+      # @return self
+      def select(*fields)
+        @fields.concat(fields).uniq!
+        self
+      end
+
+      # Request all fields. Overwrites any previously-requested fields.
+      #
+      # @return self
+      def select_all
+        @all_fields = true
+        self
+      end
+
+      # Add filters to this optica request
+      #
+      # Filters should be a hash of String => Constraint, where constraint is a
+      # - regex (regex match)
+      # - boolean (matches 1, true, or True)
+      # - string (exact match)
+      #
+      # @param filters [Hash]
+      # @return self
+      def where(filters)
+        @filters.merge!(filters)
+        self
+      end
+
+      # @return [URI]
+      def to_uri
+        uri = @uri.dup
+        uri.query = URI.encode_www_form(query_params)
+        uri.path = '/roles' unless @all_fields
+        uri
+      end
+
+      def root
+        @uri
+      end
+
+      def select_all?
+        @all_fields
+      end
+
+      private
+
+      # @return [String]
+      def query_params
+        params = {}
+        @filters.each do |key, value|
+          params[key.to_s] = filter_to_param(value)
+        end
+        if @fields.any? && !@all_fields
+          params['_extra_fields'] = @fields.join(',')
+        end
+        params
+      end
+
+      # @return [String]
+      def filter_to_param(value)
+        case value
+          when ::Regexp
+            value.source
+          else
+            literal = ::Regexp.escape(value.to_s)
+            "^#{literal}$"
+        end
+      end
+    end
+  end
+end

data/lib/optica/client/version.rb

@@ -0,0 +1,5 @@
+module Optica
+  module Client
+    VERSION = "0.1.0"
+  end
+end

data/lib/optica/client.rb

@@ -0,0 +1,12 @@
+##
+# Optica::Client helps you efficiently query Optica, Airbnb's open-source host
+# registration service.
+#
+# @see Optica::Client::Request
+
+module Optica
+  module Client
+    require_relative './client/version'
+    require_relative './client/request'
+  end
+end

data/optica-client.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'optica/client/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "optica-client"
+  spec.version       = Optica::Client::VERSION
+  spec.authors       = ["Jake Teton-Landis"]
+  spec.email         = ["jake.tl@airbnb.com"]
+
+  spec.summary       = %q{An Optica client}
+  spec.description   = %q{An efficient Optica CLI client and accompanying Ruby library}
+  spec.homepage      = "https://github.com/justjake/optica-client"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "filecache", "~> 1.0.2"
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-byebug"
+  spec.add_development_dependency "pry-doc"
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: optica-client
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jake Teton-Landis
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-04-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: filecache
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-doc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: An efficient Optica CLI client and accompanying Ruby library
+email:
+- jake.tl@airbnb.com
+executables:
+- optical
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/optical
+- lib/optica/client.rb
+- lib/optica/client/cli.rb
+- lib/optica/client/config.rb
+- lib/optica/client/fetch_json.rb
+- lib/optica/client/request.rb
+- lib/optica/client/version.rb
+- optica-client.gemspec
+homepage: https://github.com/justjake/optica-client
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.10
+signing_key: 
+specification_version: 4
+summary: An Optica client
+test_files: []