uri_resolver 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0b5218486467a58c9767f9416999e04e3a9c3a7a
+  data.tar.gz: 0a2747914812689af0cf8d49f3a8ab3d1e8dc1fe
+SHA512:
+  metadata.gz: 4744dfb882f5cf5dd1603922bdd786d5ec6424bd55ffb2f5abf4c66af2278bb02f897dd7a64d721187c67cf45f36b4ff3d0fcff3760685f79891c23d30f0875d
+  data.tar.gz: 65c1e5fc818c305d4f243144cd99ba3506aa553be60345caa1312e85671212efef99f5a5ff31d1207aa283308c9d0a076bbba86f63475b689da0cfed36fe77a2

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.11.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in uri_resolver.gemspec
+gemspec

data/README.md

@@ -0,0 +1,74 @@
+# UriResolver
+
+Checking whether a URI resolves manually is easy: You simply enter it into your
+browser and wait to see what happens!
+
+Scripting this is normally quite easy, too. Some things you might try are:
+
+* Check if the URI is a valid format, i.e. does it match `URI::regexp`?
+* Ping the URI, and check for a response.
+* Try using `URI.parse` or `Net::HTTP.get`, and rescue if `Errno::ECONNREFUSED`?
+
+For *most* URIs a simple method like the above works fine.
+
+However, for many "obscure" websites, such as those registered in new GTLDs, you
+run into all sorts of trouble attempting this - especially when checking a very
+long list! For example:
+
+* Even a simple `ping` (i.e. opening a TCP connection) can *freeze* while
+connecting to the DNS server  - which causes it to take ~20 seconds to check
+just one URI!!
+* Some URIs resolve via many layers of redirections.
+* Some URIs resolve, but to "invalid" URIs which contain escape sequences.
+* The URI may resolve to a HTTPS connection with no/a misconfigured SSL certificate.
+* The URI may connect, but fails/times out when attempting to *read* any data.
+
+This ruby gem attempts to gracefully account for all of these edge cases (and more),
+by use of intelligent error handling. It also uses a multi threaded approach to prevent
+system-blocking timeouts.
+
+It's a pretty simplistic solution, but will hopefully be useful to others.
+
+## Installation
+
+Feel free to use this as you like, but I'm currently experimenting with it;
+implementation may change significantly, until `v1.0.0` is published.
+
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'uri_resolver'
+```
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+    UriResolver.resolve_status "google.com"        # => :resoves
+    UriResolver.resolve_status "sakjflkdjsfh.com"  # => :does_not_resolve
+
+    # If the connection times out, then the gem returns :maybe_resolves
+    UriResolver.resolve_status "getmintedpoker.tv" # => :maybe_resolves
+    # Such URIs *probably* don't resolve, but a manual check may be a good idea
+
+Warning: This is *not perfect*; you can still get some false negatives. For example:
+
+    # Intermittant and very slow... This often times out, but sometimes does resolve!
+    UriResolver.resolve_status "bet-and-win.gr"  # => :maybe_resolves
+
+    # This IS a real website, but (currently) has "Bandwidth Limit Exceeded" error:
+    UriResolver.resolve_status "notarealwebsite.com" # => :does_not_resolve
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/tom-lord/uri_resolver.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "uri_resolver"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/uri_resolver.rb

@@ -0,0 +1,84 @@
+require "uri_resolver/version"
+require 'httparty'
+
+module UriResolver
+  module Status
+    # Closest thing ruby has to an Enum....
+    Resolves = :resolves
+    MaybeResolves = :maybe_resolves
+    DoesNotResolve = :does_not_resolve
+  end
+
+  ConnectionFailed = Class.new(StandardError)
+  include HTTParty
+  default_timeout 1 # HTTParty
+
+  def self.resolve_status(uri)
+    # TODO: I used to use code like this, when the below was inside a class:
+    #@@responses[uri] ||= try_get_uri(uri) # Might raise error, if DNS error/timeout
+    # Is it safe/sensible to memoize the reponses here, for performance?
+
+    response = try_get_uri(uri) # Might raise error, if DNS error/timeout
+    case
+      # TODO: .dev domains behave differently on Ubuntu vs Windows!!
+    when response.code != 200 || response.headers['server'] == "Apache/2.4.7 (Ubuntu)"
+      Status::DoesNotResolve
+    else
+      # TODO: Is it posible to analyse anything further?
+      # E.g. does the web page redirect? Does it contain certain strings? Is it at least a minimum size? ....
+      Status::Resolves
+    end
+
+  rescue SocketError # from TCPSocket#new
+    Status::DoesNotResolve
+  rescue Net::OpenTimeout # from HTTParty#get
+    Status::DoesNotResolve
+  rescue Net::ReadTimeout # from HTTParty#get
+    Status::DoesNotResolve
+  rescue Errno::ECONNREFUSED # From HTTParty#get
+    Status::DoesNotResolve
+  rescue Errno::ECONNRESET # From HTTParty#get
+    Status::DoesNotResolve
+  rescue ConnectionFailed # From TCPSocket#new or HTTParty#get -- something funny happened, check manually!
+    Status::MaybeResolves
+  rescue OpenSSL::SSL::SSLError # Resolves, but SSL certificate is not set up correctly
+    Status::Resolves
+  rescue URI::InvalidURIError # Resolves, but to weird URL (contains escape sequences??)
+    Status::Resolves
+  rescue HTTParty::RedirectionTooDeep # Resolves, but with lots of redirection!
+    Status::Resolves
+  rescue StandardError => e # Something else happened??!!
+    warn "URI #{uri} did not resolve, for unknown reason:"
+    warn e.message
+    Status::MaybeResolves
+  end
+
+  private
+
+  def self.try_get_uri(uri, timeout = 1)
+    thread_with_timeout(timeout) { TCPSocket.new(uri, 80) }
+    # TODO: Do not prepend with "http://" if inappropriate
+    thread_with_timeout(timeout*5) { HTTParty.get("http://#{uri}") } # This can take longer...
+  end
+
+  def self.thread_with_timeout(timeout)
+    th = Thread.new { yield }
+    th.priority = 2 # Make sure this runs immediately
+
+    # Do not use Timeout::timeout, as this does not play nicely with multi-threading
+    start = Time.now
+    loop do
+      if th.alive?
+        if (Time.now - start > timeout)
+          Thread.kill(th)
+          raise ConnectionFailed
+        end
+      else
+        return th.value
+      end
+      sleep 0.1 # Prevent system hammering
+    end
+  end
+
+end
+

data/lib/uri_resolver/version.rb

@@ -0,0 +1,3 @@
+module UriResolver
+  VERSION = "0.1.0"
+end

data/uri_resolver.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'uri_resolver/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "uri_resolver"
+  spec.version       = UriResolver::VERSION
+  spec.authors       = ["Tom Lord"]
+  spec.email         = ["tom.lord@comlaude.com"]
+
+  spec.summary       = %q{Like "ping", but better -- especially for new GTLDs}
+  spec.homepage      = "https://github.com/tom-lord/uri_resolver"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_runtime_dependency 'httparty'
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: uri_resolver
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Tom Lord
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-04-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- tom.lord@comlaude.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/uri_resolver.rb
+- lib/uri_resolver/version.rb
+- uri_resolver.gemspec
+homepage: https://github.com/tom-lord/uri_resolver
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Like "ping", but better -- especially for new GTLDs
+test_files: []