reso_transport 1.5.4 → 1.5.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d8d276bf6165281a38be1f3eecda4f50c530bb7a2a2d40b2f8382ae44211bf9
-  data.tar.gz: 2ecab6485d270956e91e5e3278fd46a9c1cd65a8960914dfccf86fa530d4c561
+  metadata.gz: 49e97dc43976dfe814ab0bdc7ce7aea16ae5f03e00c94d10d60de3319cb4d388
+  data.tar.gz: f15dd31a948a33dbd8008447be68828ba953114214b7633ff8a4628afa2bdfeb
 SHA512:
-  metadata.gz: 55b29a0d927e87fa2aef9736bed074b72dfdce04d866ec6b4f1b9f1e4a237d4d7ef44199e19956b45bdf879954f5575e055049adb3b550f80eb10ec2ad42ca10
-  data.tar.gz: 91366382060311726be58cf26c1439aad63e6fcfb96f1b6044465fdc630bc499d8ef0edcd6c1443d4ce06b0acc881009977584faed9b2bc6ebe3efbcc121bcb4
+  metadata.gz: 375f0962243d54dd27b80da550281fbc1a3b2e9a92affc6cb3a00e94104f5be0fddf6b3e1e5e636fc77a557d2a21438427867af8a66b1b18ae4a897c1027d5f1
+  data.tar.gz: 272dedeb65d29749ff3abeacb955c900a9e7e7dcf2449b53ab1bf9c6f7b996e84b78773f635868e603cc86eb34812a7e3b88f7e981ae41ce671055ddc9fd02a2

data/.gitignore

@@ -13,5 +13,7 @@
 /log/**
 /test/vcr_cassettes/**
 md_cache/**
+ds_cache/**
 .byebug_history
 secrets.yml
+/.history/

data/.gitpod.yml

@@ -0,0 +1,2 @@
+tasks:
+  - init: bundle install

data/.rubocop.yml

@@ -0,0 +1,33 @@
+AllCops:
+  TargetRubyVersion: 2.6
+
+Layout/FirstHashElementIndentation:
+  Enabled: false
+
+Layout/MultilineMethodCallIndentation:
+  Enabled: false
+
+Metrics/AbcSize:
+  Max: 20
+
+Metrics/BlockLength:
+  Exclude:
+    - test/**/*
+
+Metrics/ClassLength:
+  Max: 150
+
+Metrics/MethodLength:
+  Max: 15
+
+Metrics/ModuleLength:
+  Max: 150
+
+Style/ColonMethodCall:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    reso_transport (1.5.2)
+    reso_transport (1.5.5)
       faraday (~> 1.0.1)
 
 GEM

data/README.md

@@ -1,3 +1,6 @@
+[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/wrstudios/reso_transport)
+[![Gem Version](https://badge.fury.io/rb/reso_transport.svg)](https://badge.fury.io/rb/reso_transport)
+
 # ResoTransport
 
 A Ruby gem for connecting to and interacting with RESO WebAPI services.  Learn more about what that is by checking out the [RESO WebAPI](https://www.reso.org/reso-web-api/) Documentation.
@@ -51,7 +54,7 @@ Or you can set a logger for each specific instance of a client which can be usef
 
 ### Getting Connected
 
-There are 2 strategies for authentication. 
+There are 2 strategies for authentication.
 
 **Bearer Token**
 
@@ -60,7 +63,8 @@ It's simple to use a static access token if your token never expires:
 ```ruby
   @client = ResoTransport::Client.new({
     md_file: METADATA_CACHE,
-    endpoint: ENDPOINT_URL
+    endpoint: ENDPOINT_URL,
+    use_replication_endpoint: false # this is the default and can be ommitted
     authentication: {
       access_token: TOKEN,
       token_type: "Bearer" # this is the default and can be ommitted
@@ -89,7 +93,13 @@ If the connection requires requesting a new token periodically, it's easy to pro
 
 This will pre-fetch a token from the provided endpoint when the current token is either non-existent or has expired.
 
+The `use_replication_endpoint` flag will append `/replication` to all resource queries if set to `true`. This is required by some data sources to query resources beyond 10,000 records.
 
+When using this feature, you can retrieve the next link by accessing `next_link` after gettings results:
+```
+  results = @client.resources["Property"].query.results
+  next_link = @client.resources["Property"].query.next_link
+```
 
 ### Caching Metadata
 
@@ -108,9 +118,9 @@ If you don't have access to the file system, like on Heroku, or you just don't w
 
 ```ruby
 class MyCacheStore < ResoTransport::MetadataCache
-  
+
   def read
-    # read `name` from somewhere    
+    # read `name` from somewhere
   end
 
   def write(data)
@@ -158,12 +168,25 @@ Once you have a successful connection you can explore what resources are availab
   #=> {"Property"=>#<ResoTransport::Resource entity_set="Property", schema="ODataService">, "Office"=>#<ResoTransport::Resource entity_set="Office", schema="ODataService">, "Member"=>#<ResoTransport::Resource entity_set="Member", schema="ODataService">}
 
   @client.resources["Property"]
-  #=> #<ResoTransport::Resource entity_set="Property", schema="ODataService"> 
+  #=> #<ResoTransport::Resource entity_set="Property", schema="ODataService">
 
   @client.resources["Property"].query.limit(1).results
   #=> Results Array
 ```
 
+If the resource contains localizations you can access those as well.
+
+```ruby
+@client.resources["Property"].localizations
+#=> {"CommercialSale"=>{"Name"=>"CommercialSale", "ResourcePath"=>"/Property?Class=CommercialSale", "Description"=>"Contains data for Commercial searches.", "DateTimeStamp"=>"2021-05-03T18:13:20.643-07:00"}, "Residential"=>{"Name"=>"Residential", "ResourcePath"=>"/Property?Class=Residential", "Description"=>"Contains data for Residential searches.", "DateTimeStamp"=>"2021-05-03T18:13:20.643-07:00"}}
+```
+
+If a resource contains localizations you must select one by name, before querying, like so:
+
+```ruby
+@client.resources["Property"].localization('Residential').query.limit(1).results
+```
+
 #### Querying
 
 ResoTransport provides powerful querying capabilities:
@@ -196,7 +219,7 @@ To see what child records can be expanded look at `expandable`:
 
 ```ruby
   @resource.expandable
-  #=> [#<struct ResoTransport::Property name="Media", data_type="Collection(RESO.Media)", attrs={"Name"=>"Media", "Type"=>"Collection(RESO.Media)"}, multi=true, enum=nil, complex_type=nil, entity_type=#<struct ResoTransport::EntityType name="Media", base_type=nil, primary_key="MediaKey", schema="CoreLogic.DataStandard.RESO.DD">> ...] 
+  #=> [#<struct ResoTransport::Property name="Media", data_type="Collection(RESO.Media)", attrs={"Name"=>"Media", "Type"=>"Collection(RESO.Media)"}, multi=true, enum=nil, complex_type=nil, entity_type=#<struct ResoTransport::EntityType name="Media", base_type=nil, primary_key="MediaKey", schema="CoreLogic.DataStandard.RESO.DD">> ...]
 ```
 
 Use `expand` to expand child records with the top level results.
@@ -210,7 +233,7 @@ You have several options to expand multiple child record sets. Each of these wil
 
 ```ruby
   @resource.query.expand("Media", "Office").limit(10).results
-  
+
   @resource.query.expand(["Media", "Office"]).limit(10).results
 
   @resource.query.expand("Media").expand("Office").limit(10).results
@@ -257,9 +280,22 @@ When querying for an enumeration value, you can provide either the system name,
 
 ```ruby
   @resource.query.eq(StandardStatus: "Active Under Contract").limit(1).compile_params
-  #=> {"$top"=>1, "$filter"=>"StandardStatus eq 'ActiveUnderContract'"} 
+  #=> {"$top"=>1, "$filter"=>"StandardStatus eq 'ActiveUnderContract'"}
 ```
 
+### Troubleshooting
+
+In the event there are connection issues, the following errors are raised:
+
+* `ResoTransport::NoResponse` - The server did not respond to the request
+* `ResoTransport::RequestError` - The server responded with a status code outside the 200 range
+* `ResoTransport::ResponseError` - The server responded with errors in the body
+* `ResoTransport::AccessDenied` - Check your authentication details
+* `ResoTransport::LocalizationRequired` - Provide one of the required localizations through the `localization` method
+* `ResoTransport::EncodeError` - No match was found for one or more of the properties
+
+The Faraday Request hash is attached to the error for `NoResponse`, `RequestError`, `ResponseError`, and `AccessDenied`.  A Faraday Response is attached on `RequestError`, `ResponseError`, and `AccessDenied`.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "reso_transport"
+require 'bundler/setup'
+require 'reso_transport'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -12,19 +12,19 @@ require "reso_transport"
 #
 
 ResoTransport.configure do |c|
-  c.logger = Logger.new("log/console.log")
+  c.logger = Logger.new('log/console.log')
 end
 
-
-require "irb"
+require 'irb'
 require 'yaml'
 require 'byebug'
 
-SECRETS = YAML.load_file("secrets.yml")
+SECRETS = YAML.load_file('secrets.yml')
 
-@trestle = ResoTransport::Client.new(SECRETS[:trestle].merge(logger: Logger.new($stdout)))
-#@bridge = ResoTransport::Client.new(SECRETS[:bridge].merge(logger: Logger.new($stdout)))
-# @spark = ResoTransport::Client.new(SECRETS[:spark])
-#@crmls = ResoTransport::Client.new(SECRETS[:crmls])
+SECRETS.each do |name, data|
+  data[:logger] = Logger.new($stdout)
+  client = ResoTransport::Client.new(data)
+  instance_variable_set("@#{name}", client)
+end
 
 IRB.start(__FILE__)

data/bin/rake

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")

data/lib/reso_transport/authentication/fetch_token_auth.rb

@@ -1,48 +1,71 @@
 module ResoTransport
   module Authentication
     class FetchTokenAuth < AuthStrategy
-      attr_reader :connection, :endpoint, :client_id, :client_secret, :grant_type, :scope
-      
+      attr_reader :endpoint,
+                  :client_id,
+                  :client_secret,
+                  :grant_type,
+                  :scope,
+                  :username,
+                  :password
+
       def initialize(options)
-        @grant_type    = options.fetch(:grant_type, "client_credentials")
-        @scope         = options.fetch(:scope, "api")
+        super()
+
+        @grant_type    = options.fetch(:grant_type, 'client_credentials')
+        @scope         = options.fetch(:scope, 'api')
         @client_id     = options.fetch(:client_id)
         @client_secret = options.fetch(:client_secret)
         @endpoint      = options.fetch(:endpoint)
+        @username      = options.fetch(:username, nil)
+        @password      = options.fetch(:password, nil)
+        @request       = nil
+      end
 
-        @connection = Faraday.new(@endpoint) do |faraday|
+      def connection
+        @connection ||= Faraday.new(@endpoint) do |faraday|
           faraday.request  :url_encoded
           faraday.response :logger, ResoTransport.configuration.logger if ResoTransport.configuration.logger
           faraday.adapter Faraday.default_adapter
-          faraday.basic_auth @client_id, @client_secret
+          faraday.basic_auth client_id, client_secret
         end
       end
 
       def authenticate
-        response = connection.post nil, auth_params
+        response = connection.post(nil, auth_params { |req| @request = req })
         json = JSON.parse response.body
 
-        unless response.success?
-          message = "#{response.reason_phrase}: #{json['error'] || response.body}"
-          raise ResoTransport::AccessDenied, response: response, message: message
-        end
+        raise AccessDenied.new(request, response, 'token') unless response.success?
 
         Access.new({
           access_token: json.fetch('access_token'),
           expires_in: json.fetch('expires_in', 1 << (1.size * 8 - 2) - 1),
-          token_type: json.fetch('token_type', "Bearer")
+          token_type: json.fetch('token_type', 'Bearer')
         })
       end
 
+      def request
+        return @request.to_h if @request.respond_to? :to_h
+
+        {}
+      end
+
       private
 
       def auth_params
-        {
-          client_id:     client_id,
+        params = {
+          client_id: client_id,
           client_secret: client_secret,
-          grant_type:    grant_type,
-          scope:         scope
+          grant_type: grant_type,
+          scope: scope
         }
+
+        if grant_type == 'password'
+          params[:username] = username
+          params[:password] = password
+        end
+
+        params
       end
     end
   end

data/lib/reso_transport/authentication/middleware.rb

@@ -18,10 +18,11 @@ module ResoTransport
           authorize_request(request_env)
 
           @app.call(request_env).on_complete do |response_env|
-            raise_if_unauthorized(response_env)
+            raise_if_unauthorized(request_env, response_env)
           end
         rescue ResoTransport::AccessDenied
-          raise if retries == 0
+          raise if retries.zero?
+
           @auth.reset
           retries -= 1
           retry
@@ -38,8 +39,8 @@ module ResoTransport
         )
       end
 
-      def raise_if_unauthorized(response_env)
-        raise ResoTransport::AccessDenied if response_env[:status] == 401
+      def raise_if_unauthorized(request_env, response_env)
+        raise ResoTransport::AccessDenied.new(request_env.to_h, response_env) if response_env[:status] == 401
       end
     end
   end

data/lib/reso_transport/base_metadata.rb

@@ -0,0 +1,64 @@
+module ResoTransport
+  class BaseMetadata
+    MIME_TYPES = {
+      xml: 'application/xml',
+      json: 'application/json'
+    }.freeze
+
+    attr_reader :client
+
+    def initialize(client)
+      @client = client
+      @prefix = nil
+      @classname = nil
+    end
+
+    def prefix
+      raise 'prefix not set' unless @prefix
+
+      @prefix
+    end
+
+    def classname
+      raise 'classname not set' unless @classname
+
+      @classname
+    end
+
+    def parser
+      @parser ||= Object::const_get("#{classname}Parser").new.parse(data)
+    end
+
+    def data
+      if cache_file
+        cache.read || cache.write(raw)
+      else
+        raw
+      end
+    end
+
+    def cache
+      @cache ||= client.send("#{prefix}_cache").new(cache_file)
+    end
+
+    def cache_file
+      @cache_file ||= client.send "#{prefix}_file"
+    end
+
+    def raw
+      return response.body if response.success?
+
+      raise RequestError.new(request, response, classname)
+    end
+
+    def response
+      raise 'Must implement response method'
+    end
+
+    def request
+      return @request.to_h if @request.respond_to? :to_h
+
+      {}
+    end
+  end
+end

data/lib/reso_transport/client.rb

@@ -1,35 +1,52 @@
 module ResoTransport
   class Client
-    attr_reader :connection, :uid, :vendor, :endpoint, :auth, :md_file, :md_cache
-    
+    attr_reader :connection, :uid, :vendor, :endpoint, :authentication, :md_file, :md_cache, :ds_file, :ds_cache,
+                :use_replication_endpoint
+
     def initialize(options)
-      @endpoint        = options.fetch(:endpoint)
-      @md_file         = options.fetch(:md_file, nil)
-      @authentication  = ensure_valid_auth_strategy(options.fetch(:authentication))
-      @vendor          = options.fetch(:vendor, {})
-      @faraday_options = options.fetch(:faraday_options, {})
-      @logger          = options.fetch(:logger, nil)
-      @md_cache        = options.fetch(:md_cache, ResoTransport::MetadataCache)
-
-      @connection = Faraday.new(@endpoint, @faraday_options) do |faraday|
+      @use_replication_endpoint = options.fetch(:use_replication_endpoint, false)
+      @endpoint                 = options.fetch(:endpoint)
+      @md_file                  = options.fetch(:md_file, nil)
+      @ds_file                  = options.fetch(:ds_file, nil)
+      @authentication           = ensure_valid_auth_strategy(options.fetch(:authentication))
+      @vendor                   = options.fetch(:vendor, {})
+      @faraday_options          = options.fetch(:faraday_options, {})
+      @logger                   = options.fetch(:logger, nil)
+      @md_cache                 = options.fetch(:md_cache, ResoTransport::MetadataCache)
+      @ds_cache                 = options.fetch(:ds_cache, ResoTransport::MetadataCache)
+      @connection               = establish_connection
+    end
+
+    def establish_connection
+      Faraday.new(@endpoint, @faraday_options) do |faraday|
         faraday.request  :url_encoded
         faraday.response :logger, @logger || ResoTransport.configuration.logger
-        #yield faraday if block_given? 
         faraday.use Authentication::Middleware, @authentication
-        faraday.adapter Faraday.default_adapter #unless faraday.builder.send(:adapter_set?)
+        faraday.adapter Faraday.default_adapter # unless faraday.builder.send(:adapter_set?)
       end
     end
 
     def resources
-      @resources ||= metadata.entity_sets.map {|es| {es.name => Resource.new(self, es)} }.reduce(:merge!)
+      @resources ||= metadata.entity_sets.map { |es| { es.name => resource_for(es) } }.reduce(:merge!)
+    end
+
+    def resource_for(entity_set)
+      localizations = {}
+      localizations = datasystem.localizations_for(entity_set.entity_type) if metadata.datasystem?
+
+      Resource.new(self, entity_set, localizations)
     end
 
     def metadata
       @metadata ||= Metadata.new(self)
     end
 
+    def datasystem
+      @datasystem ||= Datasystem.new(self)
+    end
+
     def to_s
-      %(#<ResoTransport::Client endpoint="#{endpoint}", md_file="#{md_file}">)
+      %(#<ResoTransport::Client endpoint="#{endpoint}", md_file="#{md_file}", ds_file="#{ds_file}">)
     end
 
     def inspect
@@ -41,7 +58,7 @@ module ResoTransport
     def ensure_valid_auth_strategy(options)
       case options
       when Hash
-        if options.has_key?(:endpoint)
+        if options.key?(:endpoint)
           Authentication::FetchTokenAuth.new(options)
         else
           Authentication::StaticTokenAuth.new(options)
@@ -50,6 +67,5 @@ module ResoTransport
         raise ArgumentError, "#{options.inspect} invalid:  cannot determine strategy"
       end
     end
-
   end
 end

data/lib/reso_transport/datasystem.rb

@@ -0,0 +1,25 @@
+require_relative 'base_metadata'
+
+module ResoTransport
+  class Datasystem < BaseMetadata
+    def initialize(client)
+      super client
+      @prefix = 'ds'
+      @classname = self.class.name
+    end
+
+    def localizations_for(resource_name)
+      localizations = parser.resources.dig(resource_name, 'Localizations') || []
+      localizations.map { |l| [l['Name'], l] }.to_h
+    end
+
+    def response
+      @response ||= client.connection.get('DataSystem') do |req|
+        req.headers['Accept'] = 'application/json'
+        @request = req
+      end
+    rescue Faraday::ConnectionFailed
+      raise NoResponse.new(request, nil, 'DataSystem')
+    end
+  end
+end

data/lib/reso_transport/datasystem_parser.rb

@@ -0,0 +1,26 @@
+module ResoTransport
+  class DatasystemParser
+    def parse(doc)
+      begin
+        data = doc.is_a?(File) ? doc.read : doc
+        @json = JSON.parse data
+      rescue JSON::ParserError => e
+        @json = {}
+        puts e.message
+      end
+      self
+    end
+
+    # value ->
+    #   Resources ->
+    #     Name ->
+    #     ResourcePath ->
+    #     Localizations ->
+    #       Name ->
+    #       ResourcePath ->
+
+    def resources
+      @resources ||= @json['value'].map { |v| v['Resources'] }.flatten.compact.map { |r| [r['Name'], r] }.to_h
+    end
+  end
+end

data/lib/reso_transport/entity_set.rb

@@ -1,11 +1,9 @@
 module ResoTransport
   EntitySet = Struct.new(:name, :schema, :entity_type) do
-
     def self.from_stream(args)
-      schema, entity_type = ResoTransport.split_schema_and_class_name(args["EntityType"])
+      schema, entity_type = ResoTransport.split_schema_and_class_name(args['EntityType'])
 
-      new(args["Name"], schema, entity_type)
+      new(args['Name'], schema, entity_type)
     end
-
   end
 end

data/lib/reso_transport/entity_type.rb

@@ -1,30 +1,29 @@
 module ResoTransport
   EntityType = Struct.new(:name, :base_type, :primary_key, :schema) do
-
     def self.from_stream(args)
-      new(args["Name"], args["BaseType"])
+      new(args['Name'], args['BaseType'])
     end
 
     def parse(record)
-      record.each_pair do |k,v|
+      record.each_pair do |k, v|
         next if v.nil?
-        if property = (property_map[k] || navigation_property_map[k])
-          record[k] = property.parse(v)
-        end
+
+        property = property_map[k] || navigation_property_map[k]
+        record[k] = property.parse(v) if property
       end
     end
 
     def parse_value(record)
-      record.each_pair do |k,v|
+      record.each_pair do |k, v|
         next if v.nil?
-        if property = (property_map[k] || navigation_property_map[k])
-          record[k] = property.parse(v)
-        end
+
+        property = property_map[k] || navigation_property_map[k]
+        record[k] = property.parse(v) if property
       end
     end
 
     def property_map
-      @property_map ||= properties.inject({}) {|hsh, p| hsh[p.name] = p; hsh }
+      @property_map ||= properties.each_with_object({}) { |p, hsh| hsh[p.name] = p; }
     end
 
     def properties
@@ -32,7 +31,7 @@ module ResoTransport
     end
 
     def navigation_property_map
-      @navigation_property_map ||= navigation_properties.inject({}) {|hsh, p| hsh[p.name] = p; hsh }
+      @navigation_property_map ||= navigation_properties.each_with_object({}) { |p, hsh| hsh[p.name] = p; }
     end
 
     def navigation_properties
@@ -42,6 +41,5 @@ module ResoTransport
     def enumerations
       @enumerations ||= []
     end
-
   end
 end