sis_ruby 1.0.0

data/lib/sis_ruby/params.rb

@@ -0,0 +1,117 @@
+module SisRuby
+class Params
+
+  # These methods behave as simple accessors when called with no arguments.
+  # When called with an argument, that argument is used to set the value corresponding to the method name.
+
+  # The conventional 'attribute=' method forms for this class are supported
+  # in addition to the methods below, as an alternate way to set values.
+
+  attr_writer :fields, :filter, :limit, :offset, :sort
+
+  def fields(*args)
+    if args.none?
+      return @fields
+    elsif args.one?
+      @fields = Array(args.first)
+    else
+      @fields = args
+    end
+    @fields = @fields.to_set  # order shouldn't matter, and no dups
+    self
+  end
+
+
+  def filter(*args)
+    if args.any?
+      @filter = args.first
+      self
+    else
+      @filter
+    end
+  end
+
+
+  def limit(*args)
+    if args.any?
+      @limit = args.first
+      self
+    else
+      @limit
+    end
+  end
+
+
+  def offset(*args)
+    if args.any?
+      @offset = args.first
+      self
+    else
+      @offset
+    end
+  end
+
+
+  def sort(*args)
+    if args.any?
+      @sort = args.first
+      self
+    else
+      @sort
+    end
+  end
+
+
+  def to_hash
+    h = {}
+    h['limit'] = limit if limit
+    h['offset'] = offset if offset
+    h['fields'] = fields.to_a.join(',') if fields
+    h['q'] = filter if filter
+    h['sort'] = sort if sort
+    h
+  end
+
+
+  def self.from_hash(other)
+    instance = self.new
+    instance.limit(other['limit']) if other['limit']
+    instance.fields(other['fields'].split(',').to_set) if other['fields']
+    instance.offset(other['offset']) if other['offset']
+    instance.filter(other['q']) if other['q']
+    instance.sort(other['sort']) if other['sort']
+    instance
+  end
+
+
+  def clone
+    other = Params.new
+    other.limit(self.limit)         if self.limit
+    other.fields(self.fields.clone) if self.fields
+    other.offset(self.offset)       if self.offset
+    other.filter(self.filter.clone) if self.filter
+    other.sort(self.sort)           if self.sort
+    other
+  end
+
+
+  def to_h
+    to_hash
+  end
+
+
+  def hash
+    to_h.hash
+  end
+
+
+  def ==(other)
+    other.is_a?(self.class) && other.to_h == self.to_h
+  end
+
+
+  def <=>(other)
+    self.to_h <=> other.to_h
+  end
+end
+end

data/lib/sis_ruby/result_enumerable.rb

@@ -0,0 +1,86 @@
+require 'trick_bag'
+require 'ostruct'
+require 'typhoeus'
+require_relative 'get_helper'
+require_relative 'params'
+
+
+module SisRuby
+
+
+# This class is an enumerator over a result set of SIS objects, as specified
+# in a hash-like object (that is, any object that implements to_hash,
+# such as a SisParams instance).  It can be used as any Enumerable;
+# that is, methods such as each, map, select, etc. can be called;
+# an array of all results can be produced by calling to_a,
+# and an Enumerator can be gotten by calling each without a chunk.
+#
+# Internally it fetches chunks of records only when needed to serve the values
+# to the caller.  The chunk size can be specified by the caller.
+#
+# By calling each or each_slice you can access some of the data before fetching
+# all of it. This can be handy if:
+#
+# 1) there may not be enough available memory to process the entire result set
+#    at once
+# 2) you want to process some records while others are being fetched, by
+#    using multiple threads, for example.
+#
+class ResultEnumerable < TrickBag::Enumerables::BufferedEnumerable
+
+  include GetHelper
+  include Enumerable
+
+  DEFAULT_CHUNK_RECORD_COUNT = 5_000
+
+
+  # @param endpoint_url the SIS endpoint URL string or an Endpoint instance
+  # @param params any object that implements to_hash, e.g. a Params instance
+  # @param chunk_size the maximum number of records to fetch from the server in a request
+  def initialize(endpoint_url, params = {}, chunk_size = DEFAULT_CHUNK_RECORD_COUNT)
+    super(chunk_size)
+    @endpoint_url = endpoint_url.is_a?(Endpoint) ? endpoint_url.url : endpoint_url
+    @outer_params = params.is_a?(Params) ? params : Params.from_hash(params.to_hash)
+
+    inner_limit = @outer_params.limit ? [chunk_size, @outer_params.limit].min : chunk_size
+    @inner_params = @outer_params.clone.limit(inner_limit)
+
+    @inner_params.offset ||= 0
+    @position     = 0
+    @total_count  = nil
+  end
+
+
+  def fetch
+    # This exits the fetching when the desired number of records has been fetched.
+    if @outer_params.limit && @yield_count >= @outer_params.limit
+      self.data = []
+      return
+    end
+
+    request = Typhoeus::Request.new(@endpoint_url, params: @inner_params.to_hash, headers: create_headers(true))
+    response = request.run
+    validate_response_success(response)
+    self.data = JSON.parse(response.body)
+    @total_count = response.headers['x-total-count'].to_i
+    @inner_params.offset(@inner_params.offset + chunk_size)
+
+    # TODO: Deal with this
+    # if @total_count > chunk_size && @inner_params.sort.nil?
+    #   raise "Total count (#{@total_count}) exceeds chunk size (#{chunk_size})." +
+    #       "When this is the case, a sort order must be specified, or chunk size increased."
+    # end
+  end
+
+
+  def total_count
+    fetch unless @total_count
+    @total_count
+  end
+
+
+  def fetch_notify
+    # puts "Fetch at #{Time.now}: chunk size: #{chunk_size}, yield count: #{yield_count}, total count = #{@total_count}"
+  end
+end
+end

data/lib/sis_ruby/version.rb

@@ -0,0 +1,3 @@
+module SisRuby
+  VERSION = '1.0.0'
+end

data/lib/sis_ruby.rb

@@ -0,0 +1,8 @@
+# Load all *.rb files in lib/sis_ruby and below.
+# Use a lambda so that the intermediate variables do not survive this file.
+->() {
+  start_dir = File.join(File.dirname(__FILE__), 'sis_ruby') # the lib directory
+  file_mask = "#{start_dir}/**/*.rb"
+  Dir[file_mask].each { |file| require file }
+}.()
+

data/samples/sample-1.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+require 'sis_ruby'
+require 'awesome_print'
+
+include SisRuby
+
+raise "Must specify SIS server URL in SIS_URL environment variable." unless ENV['SIS_URL']
+client = Client.new(ENV['SIS_URL'])
+host_endpoint = client.entities('host')
+
+records = host_endpoint.list(Params.new.fields('hostname').limit(3))
+puts "Fetched #{records.size} host records."
+puts "Last one is: #{records.last}, host name is #{records.last['hostname']}"
+
+
+full_record = host_endpoint.list(Params.new.limit(1)).first
+puts 'Host record keys:'
+ap full_record.keys
+
+puts "\n\n"
+query = Params.new.limit(1).fields('hostname', 'fqdn', 'status')
+result = host_endpoint.list_as_openstructs(query).first
+puts "Host: #{result.hostname}, FQDN: #{result.fqdn}, Status: #{result.status}"
+puts "\nEntire record:"
+ap result

data/samples/sample-2.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'awesome_print'
+require 'pry'
+require 'sis_ruby'
+
+raise "Must specify SIS server URL in SIS_URL environment variable." unless ENV['SIS_URL']
+client = Client.new(ENV['SIS_URL'])
+HOSTS = client.entities('host')
+# puts "Total host count is #{hosts.count}"
+# puts "Total qa host count is  #{hosts.count('environment' => 'qa')}"
+
+# p1 = SisRuby::Params.new.sort('host').offset(9000).limit(2)
+# p1 = SisRuby::Params.new.sort('host').offset(9000).limit(2).fields('hostname', 'environment')
+# records = hosts.list(p1)
+# ap records.first
+
+
+def fetch(count, offset)
+  HOSTS.list(SisRuby::Params.new.fields('hostname').limit(count).offset(offset))
+end
+
+arrays = (0..10).to_a.map { |i| fetch(1000, i * 1000) }
+a_combined = arrays.flatten
+a_uniq = a_combined.uniq
+host_names = a_combined.map { |h| h['hostname'] }
+
+puts "Combined count is #{a_combined.size}"
+puts "Unique count is #{a_uniq.size}"
+puts "Host name count is #{host_names.size}"
+puts "Unique host name count is #{host_names.uniq.size}"
+binding.pry
+

data/sis_ruby.gemspec

@@ -0,0 +1,22 @@
+require File.join(File.dirname(__FILE__), 'lib', 'sis_ruby', 'version')
+
+Gem::Specification.new do |s|
+  s.name              = 'sis_ruby'
+  s.version           = SisRuby::VERSION
+  s.platform          = Gem::Platform::RUBY
+  s.authors           = ['Keith Bennett', 'Neel Goyal']
+  s.email             = ['keithrbennett@gmail.com', 'sis-dev@verisign.com']
+  s.homepage          = 'https://github.com/keithrbennett/sis_ruby'
+  s.summary           = 'SIS Ruby Client'
+  s.description       = 'Ruby client for SIS web service (https://github.com/sis-cmdb/sis-api)'
+  s.files             = `git ls-files`.split($/)
+  s.require_path      = 'lib'
+  s.license           = 'BSD 3-Clause'
+
+  s.add_dependency 'awesome_print', '>= 1.6.0'
+  s.add_dependency 'rake'
+  s.add_dependency 'trick_bag', '>= 0.63.1'
+  s.add_dependency 'typhoeus', '>= 0.8.0'
+
+  s.add_development_dependency 'rspec', '>= 3.0'
+end

data/spec/client_spec.rb

@@ -0,0 +1,57 @@
+require_relative 'spec_helper'
+require_relative '../lib/sis_ruby/exceptions/missing_id_error'
+
+# These tests test client behavior that does not require a server.
+
+module SisRuby
+
+  describe Client do
+
+    let(:server_url)     { 'https://fictitious-domain.com' }
+    let(:id_fieldname)   { '_id_' }
+    let(:client)         { Client.new(server_url) }
+    let(:host_endpoint)  { client.entities('host', id_fieldname) }
+
+    context 'constructor parameters' do
+
+      specify 'omitting version gets default version' do
+        expect(client.api_version).to eq(Client::DEFAULT_API_VERSION)
+      end
+
+      specify 'overriding default version works' do
+        api_v = 'v_not_default'
+        expect(Client.new(server_url, api_version: api_v).api_version).to eq(api_v)
+      end
+
+      specify 'setting auth_token works' do
+        auth_token = 'abcdefgxyz'
+        expect(Client.new(server_url, auth_token: auth_token).auth_token).to eq(auth_token)
+      end
+
+    end
+
+    context 'precreated endpoints' do
+
+      specify 'are precreated' do
+        endpoints = [client.hooks, client.schemas, client.hiera]
+        expect(endpoints.all? { |ep| ep.is_a?(Endpoint) }).to eq(true)
+      end
+
+      specify 'have URLs that end in the right name' do
+        endpoints = [client.hooks, client.schemas, client.hiera]
+        endpoints_and_names = endpoints.zip(%w(hooks  schemas  hiera))
+        expect(endpoints_and_names.all? { |ep, name| ep.url.end_with?(name) }).to eq(true)
+      end
+    end
+
+
+    context 'tokens' do
+
+      specify 'the URL is correct' do
+        username = 'david_bisbal'
+        token = client.tokens(username)
+        expect(token.url.end_with?("users/#{username}/tokens")).to eq(true)
+      end
+    end
+  end
+end

data/spec/endpoint_spec.rb

@@ -0,0 +1,54 @@
+require_relative 'spec_helper'
+require_relative '../lib/sis_ruby/exceptions/missing_id_error'
+
+module SisRuby
+
+describe Endpoint do
+
+  let(:id_fieldname)   { '_id_' }
+  let(:server_url)     { 'https://fictitious-domain.com' }
+  let(:client)         { Client.new(server_url) }
+  let(:host_endpoint)  { client.entities('host', id_fieldname) }
+
+  context '#id_from_param' do
+
+    it 'works with an object other than hash' do
+      expect(host_endpoint.id_from_param(3)).to eq(3)
+    end
+
+    it 'works with a hash' do
+      expect(host_endpoint.id_from_param({ id_fieldname => 3 } )).to eq(3)
+    end
+
+    it 'raises an error on nil' do
+      expect { host_endpoint.id_from_param(nil) }.to raise_error(MissingIdError)
+      expect { host_endpoint.id_from_param( { id_fieldname => nil } ) }.to raise_error((MissingIdError))
+    end
+  end
+
+
+  context '#==' do
+
+    specify '2 endpoints created with the same parameters are ==' do
+      create_endpoint = -> { Endpoint.new(client, 'abcxyz', 'foo') }
+      endpoint_0 = create_endpoint.()
+      endpoint_1 = create_endpoint.()
+      expect(endpoint_1).to eq(endpoint_0)
+    end
+
+    specify '2 endpoints created with different clients are not ==' do
+      client1 = Client.new('url1')
+      client2 = Client.new('url2')
+      expect(Endpoint.new(client1, 'foo')).not_to eq(Endpoint.new(client2, 'foo'))
+    end
+
+    specify '2 endpoints created with different endpoint names are not ==' do
+      expect(Endpoint.new(client, 'foo')).not_to eq(Endpoint.new(client, 'bar'))
+    end
+
+    specify '2 endpoints created with different id fields are not ==' do
+      expect(Endpoint.new(client, 'foo', 'x')).not_to eq(Endpoint.new(client, 'foo', 'y'))
+    end
+  end
+end
+end

data/spec/hash_builder_spec.rb

@@ -0,0 +1,83 @@
+require_relative 'spec_helper'
+
+describe HashBuilder do
+
+  PERMITTED_ENTITY_ENDPOINT_TOP_LEVEL_KEYS = %w(
+      fields
+      id
+      limit
+      offset
+      q
+  )
+
+  QUERY_HASH = {
+      'environment'  => 'prod',
+      'status'       => 'live',  # not maintenance
+      'role.product.name' => 'mdns',
+  }
+
+  specify 'will create a limit key/value pair correctly' do
+    builder = HashBuilder.new
+    expect(builder.limit(10).to_h).to eq({ 'limit' => 10 })
+  end
+
+  specify 'subsequent calls will replace an existing value' do
+    builder = HashBuilder.new
+    builder.limit(10)
+    expect(builder.limit(11).to_h).to eq({ 'limit' => 11 })
+  end
+
+  specify 'a query hash will be correctly inserted' do
+    builder = HashBuilder.new
+    builder.q(QUERY_HASH)
+    expect(builder.to_h).to eq({ 'q' => QUERY_HASH})
+  end
+
+  specify '2 key/value pairs are no problem' do
+    builder = HashBuilder.new
+    builder.q(QUERY_HASH).limit(20)
+    expect(builder.to_h).to eq({ 'q' => QUERY_HASH, 'limit' => 20 })
+  end
+
+  specify 'keys are all Strings when key type of String is specified' do
+    builder = HashBuilder.new(String)
+    expect(builder.foo(1).bar(2).to_h.keys.all? { |key| key.is_a?(String) }).to eq(true)
+  end
+
+  specify 'keys are all Symbols when key type of Symbol is specified' do
+    builder = HashBuilder.new(Symbol)
+    expect(builder.foo(1).bar(2).to_h.keys.all? { |key| key.is_a?(Symbol) }).to eq(true)
+  end
+
+  specify 'respond_to_missing? always returns true' do
+    builder = HashBuilder.new
+    random_name = 'x' + Random.rand(1_000_000).to_s
+    expect(builder.respond_to?(random_name, false)).to eq(true)
+  end
+
+  # Disable permitted methods feature for now; it was not fully implemented,
+  # and should be a predicate lambda rather than an array anyway.
+
+  # context 'permitted methods' do
+  #
+  #   specify 'when not specified, all are allowed' do
+  #     builder = HashBuilder.new
+  #     expect(builder.respond_to?('foo')).to eq(true)
+  #   end
+  #
+  #   specify 'when empty, none are allowed' do
+  #     builder = HashBuilder.new(String, [])
+  #     expect(builder.new(String).respond_to?('foo')).to eq(false)
+  #     builder.something(123)
+  #     expect { builder.something(123) }.to raise_error
+  #   end
+  #
+  #   specify 'when non-empty, only permitted are permitted' do
+  #     builder = HashBuilder.new(String, [:foo])
+  #     expect(builder.new(String).respond_to?('foo')).to eq(true)
+  #     expect(builder.new(String).respond_to?('bar')).to eq(false)
+  #
+  #   end
+  # end
+end
+

data/spec/params_spec.rb

@@ -0,0 +1,75 @@
+require_relative 'spec_helper'
+
+module SisRuby
+
+describe Params do
+
+  it 'results in an empty hash when nothing is set' do
+    expect(Params.new.to_hash).to eq({})
+  end
+
+  it 'correctly concatenate fields' do
+    fields = %w(foo bar).to_set
+    params = Params.new.fields(fields)
+    expect(params.fields).to eq(fields)
+    expect(params.to_hash).to eq({ 'fields' => 'foo,bar'})
+  end
+
+  it 'sets limit correctly' do
+    params = Params.new.limit(3)
+    expect(params.limit).to eq(3)
+    expect(params.to_hash).to eq({ 'limit' => 3 })
+  end
+
+  it 'sets offset correctly' do
+    params = Params.new.offset(4)
+    expect(params.offset).to eq(4)
+    expect(params.to_hash).to eq({ 'offset' => 4 })
+  end
+
+  it 'sets filter correctly' do
+    filter = { 'my_numeric_field' => { '$gt' => 10 }}
+    params = Params.new.filter(filter)
+    expect(params.filter).to eq(filter)
+    expect(params.to_hash).to eq({ 'q' => filter })
+  end
+
+  it 'sets sort correctly' do
+    params = Params.new.sort('-count')
+    expect(params.sort).to eq('-count')
+    expect(params.to_hash).to eq({ 'sort' => '-count' })
+  end
+
+  specify 'to_h and to_hash return equal hashes' do
+    params = Params.new.sort('-count')
+    expect(params.to_h).to eq(params.to_hash)
+  end
+
+  specify '2 identical objects are ==, and <=> returns 0' do
+    params1 = Params.new.sort('-count')
+    params2 = Params.new.sort('-count')
+    expect(params1).to eq(params2)
+    expect(params1 <=> params2).to eq(0)
+  end
+
+  specify '2 different objects are not ==, and <=> returns non-0' do
+    params1 = Params.new.sort('-count')
+    params2 = Params.new.sort('-date')
+    expect(params1).not_to eq(params2)
+    expect(params1 <=> params2).not_to eq(0)
+  end
+
+
+  specify 'clone returns an equal copy' do
+    params_orig = Params.new.sort('-count').limit(300).filter({key: 'value'}).offset(100).fields('a', 'b')
+    params_clone = params_orig.clone
+    expect(params_clone).to eq(params_orig)
+  end
+
+  specify 'from_hash returns an equal copy' do
+    params_orig = Params.new.sort('-count').limit(300).filter({key: 'value'}).offset(100).fields('a', 'b')
+    params_copy = Params.from_hash(params_orig.to_hash)
+    expect(params_copy).to eq(params_orig)
+  end
+end
+end

data/spec/setup/drop-db.js

@@ -0,0 +1,4 @@
+/* Drops the 'sis' data base. */
+conn = new Mongo();
+db = conn.getDB('sis')
+db.dropDatabase();

data/spec/setup/recreate-db.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# This script drops the Mongo DB and adds the test user.
+# It assumes the existence of a ~/opt/sis-api directory
+# that is a clone of the sis-api git repo, and a 'test-user.json'
+# data file that is a copy of the file by that name in this directory.
+#
+# The script should be run from the directory in which it exists.
+
+mongo drop-db.js
+#echo  "use sis;\n db.dropDatabase();\n exit" | mongo
+cd ~/opt/sis-api
+node tools/useradmin.js update test-user.json
+cd -

data/spec/setup/test-user.json

@@ -0,0 +1,6 @@
+{
+    "name" : "test",
+    "pw" : "abc123",
+    "email" : "nobody@bbs-software.com",
+    "super_user" : true
+}

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require_relative '../lib/sis_ruby'

data/spec-writing/entity.json

@@ -0,0 +1,50 @@
+{
+  "entity_type": "entity_test",
+  "id_field": "_id",
+  "required_schema": {
+    "name": "entity_test",
+    "owner": "schema_owner",
+    "definition": {
+      "name": {
+        "type": "String",
+        "required": true,
+        "unique": true
+      },
+      "number": {
+        "type": "Number",
+        "unique": true,
+        "required": true
+      }
+    },
+    "_sis": {
+      "owner": "test"
+    }
+  },
+  "valid_items": [
+    {
+      "name": "entity1",
+      "number": 0
+    },
+    {
+      "name": "entity2",
+      "number": 1
+    },
+    {
+      "name": "entity3",
+      "number": 2
+    }
+  ],
+  "invalid_items": [
+    {
+      "foo": "bar"
+    },
+    {
+      "name": "entity1",
+      "number": 4
+    },
+    {
+      "name": "non_unique_num",
+      "number": 0
+    }
+  ]
+}