url_grey 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e95f9c5eda3bb27c6c30f0cdd08aa46969030b22
+  data.tar.gz: cf5c2c804e16c7463274f117f1ef5d2c2222bbc7
+SHA512:
+  metadata.gz: c658e0642443497b7c33cb90f9c359deedf4b30dabf6d63fd252577ca77ea08b0eede2fa454fa5898cfb752c93de7cece4f12d93e06db2346837e44771faa7bf
+  data.tar.gz: d0e08cb022a5d7b23cfa54fde82bd99cdd6a74cb361d1d140166f211f5601ac41dcf35b8e795be61c3b0a1c8d98baac53eb903ef67b2e791a67d22c503841512

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.2
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in url_grey.gemspec
+gemspec

data/README.md

@@ -0,0 +1,39 @@
+# URLGrey
+
+This attempts to copy chomium's algorithm for making sense of things
+typed into the url bar. You can download the [chromium source] to play
+along, but note that it is currently 2.1 GB.
+
+The ported code is very similar to how it is written in the original
+C++. It is a great example of the imperative style of programming by
+state mutation. I'm not gonna lie, it's pretty gross. But hey, it passes
+the tests.
+
+## Usage
+
+Some examples from the tests:
+
+```ruby
+URLGrey.new("google.com").fixed
+#=> "http://google.com/"
+
+URLGrey.new("www.google.com#foo").fixed
+#=> "http://www.google.com/#foo"
+
+URLGrey.new("\u6C34.com").fixed
+#=> "http://xn--1rw.com/"
+
+URLGrey.new("http://foo.com/s?q=\uC5C5").fixed
+#=> "http://foo.com/s?q=%EC%97%85"
+
+URLGrey.new("http;/www.google.com/").fixed
+#=> "http://www.google.com/"
+
+URLGrey.new(" foo.com/asdf  bar").fixed
+#=> "http://foo.com/asdf%20%20bar"
+
+URLGrey.new("[::]:80/path").fixed
+#=> "http://[::]/path"
+```
+
+[chromium source]: https://chromium.googlesource.com/chromium/chromium/

data/Rakefile

@@ -0,0 +1,7 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = "test/*_test.rb"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "url_grey"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/url_grey.rb

@@ -0,0 +1,256 @@
+require "simpleidn"
+
+require "url_grey/version"
+
+class URLGrey
+  AUTHORITY_TERMINATORS = "/\\?#"
+  ABOUT_BLANK_URL = "about:blank"
+  PATH_PASS_CHARS = "!$&'()*+,/:;=@[]"
+  PATH_UNESCAPE_CHARS = "-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~"
+  HOST_ESCAPE_CHARS = " !\"\#$&'()*,<=>@`{|}"
+  HOST_NORMAL_CHARS = "+-.0123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZ[]_abcdefghijklmnopqrstuvwxyz"
+  HOST_CHROME_DEFAULT = "version"
+  QUERY_NORMAL_CHARS = "!$%&()*+,-./0123456789:;=?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
+  DEFAULT_PORTS = {
+    ftp:    21,
+    gopher: 70,
+    http:   80,
+    https:  443,
+    ws:     80,
+    wss:    443,
+  }
+  STANDARD_SCHEMES = ['http', 'https', 'file', 'ftp', 'gopher', 'ws', 'wss', 'filesystem']
+
+  attr_accessor :original, :coerced
+  attr_accessor :scheme, :username, :password, :host, :port, :path, :query, :ref
+  attr_accessor :slashes
+
+  def initialize(_original)
+    self.original = _original.sub(%r{^\s*}, '')
+
+    parse!
+  end
+
+  def parts
+    {
+      scheme:   self.scheme,
+      username: self.username,
+      password: self.password,
+      host:     self.host,
+      port:     self.port,
+      path:     self.path,
+      query:    self.query,
+      ref:      self.ref
+    }
+  end
+
+  def fixed
+    return ABOUT_BLANK_URL if self.original == ABOUT_BLANK_URL
+
+    "#{fixed_scheme}#{fixed_credentials}#{fixed_host}#{fixed_port}#{fixed_path}#{fixed_query}#{fixed_ref}"
+  end
+
+  def fixed_credentials
+    return "" unless (!self.username.empty? || !self.password.empty?)
+    return "#{self.username}@" if self.password.empty?
+    "#{self.username}:#{self.password}@"
+  end
+
+  # from components/url_formatter/url_fixer.cc FixupHost
+  def fixed_host
+    fixed = self.host.gsub(%r{\s}, '').downcase
+    unless fixed.match(%r{^\.*$})
+      fixed = fixed.sub(%r{^\.*}, '')
+      fixed = fixed.sub(%r{(?<=\.)\.*$}, '')
+    end
+    if fixed.empty? && ["about", "chrome"].include?(self.scheme)
+      fixed = HOST_CHROME_DEFAULT
+    end
+
+    if fixed.match(%r{^[[:ascii:]]*$})
+      fixed = fixed.chars.map do |char|
+        if HOST_NORMAL_CHARS.include?(char)
+          char
+        else
+          "%#{char.codepoints.first.to_s(16).upcase}"
+        end
+      end.join("")
+    else
+      fixed = SimpleIDN.to_ascii(fixed)
+    end
+    fixed
+  end
+
+  # from url/url_canon_path.cc CanonicalizePath
+  def fixed_path
+    fixed = self.path
+    if (fixed[0] != '/') && ((STANDARD_SCHEMES + ["about", "chrome"]).include?(self.scheme))
+      fixed = '/' + fixed
+    end
+
+    fixed.chars.map do |char|
+      if PATH_PASS_CHARS.include?(char)
+        char
+      elsif PATH_UNESCAPE_CHARS.include?(char)
+        char
+      elsif char == "."
+        # TODO: if the dot is preceded by a slash, do directory stuff:
+        # google.com/abc/.././def -> google.com/def
+        char
+      else
+        "%#{char.codepoints.first.to_s(16).upcase}"
+      end
+    end.join("")
+  end
+
+  def fixed_port
+    return "" if (self.port.empty? || self.port.to_i == DEFAULT_PORTS[self.scheme.to_sym])
+    ":#{self.port}"
+  end
+
+  def fixed_query
+    fixed = self.query
+    return "" if fixed.nil?
+    fixed = fixed.bytes.map do |byte|
+      if QUERY_NORMAL_CHARS.unpack("U*").include?(byte)
+        [byte].pack("U")
+      else
+        "%#{byte.to_s(16).upcase}"
+      end
+    end.join('')
+    "?#{fixed}"
+  end
+
+  def fixed_ref
+    return "" if self.ref.nil?
+    "\##{self.ref}"
+  end
+
+  def fixed_scheme
+    fixed = self.scheme
+    if fixed == "about"
+      fixed = "chrome"
+    end
+
+    if (STANDARD_SCHEMES + ["about", "chrome"]).include?(fixed)
+      "#{fixed}://"
+    else
+      "#{fixed}:#{self.slashes}"
+    end
+  end
+
+  private
+
+  def parse!
+    parse_scheme!
+    after_scheme = self.coerced.match(%r{:(.*)})[1]
+    self.slashes, after_slashes = after_scheme.match(%r{^([\\\/]*)(.*)$})[1..2]
+
+    # authority terminators: '/', '\', '?', '#'
+    if (after_slashes.chars & ['/', '\\', '?', '#']).any?
+      authority, full_path = after_slashes.match(%r{^(.*?)([\\\/?#].*)$})[1..2]
+    else
+      authority = after_slashes
+      full_path = ""
+    end
+
+    if authority.include?("@")
+      user_info, server_info = authority.match(%r{^(.*)@(.*)$})[1..2]
+    else
+      user_info   = ""
+      server_info = authority
+    end
+
+    # parse user_info
+    if user_info.empty?
+      self.username = ""
+      self.password = ""
+    else
+      if user_info.include?(":")
+        self.username, self.password = user_info.match(%r{^(.*?):(.*)$})[1..2]
+      else
+        self.username = user_info
+        self.password = ""
+      end
+    end
+
+    # parse server_info
+    if !server_info.include?(":")
+      self.host = server_info
+      self.port = ""
+    elsif server_info.include?("]")
+      if server_info.reverse.index(":") < server_info.reverse.index("]")
+        self.host, self.port = server_info.match(%r{^(.*):(.*)$})[1..2]
+      else
+        self.host = server_info
+        self.port = ""
+      end
+    elsif server_info.chars.first == "["
+      self.host = server_info
+      self.port = ""
+    else
+      self.host, self.port = server_info.match(%r{^(.*):(.*)$})[1..2]
+    end
+
+    # parse full_path
+    if full_path.include?("#")
+      before_ref, self.ref = full_path.match(%r{^(.*?)#(.*)$})[1..2]
+    else
+      before_ref = full_path
+      self.ref = nil
+    end
+
+    if before_ref.include?("?")
+      self.path, self.query = before_ref.match(%r{^(.*?)\?(.*)$})[1..2]
+    else
+      self.path = before_ref
+      self.query = nil
+    end
+  end
+
+  def parse_scheme!
+    self.coerced = self.original
+
+    if !find_scheme(self.original) && (self.original[0]!= ";")
+      if find_scheme(self.original.sub(";", ":"))
+        self.coerced = self.original.sub(";", ":")
+      end
+    end
+
+    if !find_scheme(self.coerced)
+      if self.coerced.match(%r{^ftp\.}i)
+        self.coerced = "ftp://" + self.coerced
+      else
+        self.coerced = "http://" + self.coerced
+      end
+    end
+
+    self.scheme = find_scheme(self.coerced) || ""
+  end
+
+  def find_scheme(text)
+    # extract scheme
+    return false unless match = text.match(%r{^(.*?):})
+
+    component = match[1].downcase
+
+    return "" if component.empty?
+
+    # first character must be a letter
+    return false unless component.match(%r{^[a-z]})
+
+    # reject anything with invalid characters
+    return false unless component.match(%r{^[+\-0-9a-z]*$})
+
+    # fix up segmentation for "www:123/"
+    return false if has_port(text)
+
+    component
+  end
+
+  def has_port(text)
+    return false unless text.include?(":")
+    match = text.match(%r{:(.*?)[\\/\?#]}) || text.match(%r{:(.*)$})
+    match[1].match(%r{^\d+$})
+  end
+end

data/lib/url_grey/version.rb

@@ -0,0 +1,3 @@
+class URLGrey
+  VERSION = "1.0.0"
+end

data/url_grey.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'url_grey/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "url_grey"
+  spec.version       = URLGrey::VERSION
+  spec.authors       = ["Stacey Touset"]
+  spec.email         = ["capicue@gmail.com"]
+
+  spec.summary       = "Coerce and normalize user inputted URLs"
+  spec.homepage      = "https://github.com/capicue/url_grey"
+  spec.licenses      = ["MIT"]
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "simpleidn", "~> 0.0.7"
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.8"
+  spec.add_development_dependency "pry", "~> 0.10"
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: url_grey
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Stacey Touset
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-02-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: simpleidn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.7
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.8'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+description: 
+email:
+- capicue@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/url_grey.rb
+- lib/url_grey/version.rb
+- url_grey.gemspec
+homepage: https://github.com/capicue/url_grey
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Coerce and normalize user inputted URLs
+test_files: []