twingly-url 1.3.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6748ed6715a818455973b7330c774c4d4e66776a
+  data.tar.gz: bb17c4b3eb99af036257f3668d7048ccb94f3466
+SHA512:
+  metadata.gz: 55194c78c7a3af312d70e88730586b806868b2b9184853bca25cbdcc99e29622113c2fe333512d9f785ed4adefb8c89fe5c4472f3eb6d1e41d28905fcb020257
+  data.tar.gz: 8b9f59dfb2566c42393edc23535d5487e9bda3e0e1342c717dc4ba4a9b74e17f327b0324ee767608af1e54bdb3cc301416f153b705c8c1a253403759eb629914

data/README.md

@@ -0,0 +1,47 @@
+# Twingly::URL
+
+[![Build Status](https://travis-ci.org/twingly/twingly-url.svg?branch=master)](https://travis-ci.org/twingly/twingly-url)
+
+Twingly URL tools.
+
+* `twingly/url` - Parse and validate URLs
+    * `Twingly::URL.parse` - Returns a Struct with `#url` and `#domain` accessors
+    * `Twingly::URL.validate` - Validates a URL
+* `twingly/url/normalizer` - Normalize URLs
+    * `Twingly::URL::Normalizer.normalize(string)` - Extracts URLs from string (Array)
+* `twingly/url/hasher` - Generate URL hashes suitable for primary keys
+    * `Twingly::URL::Hasher.documentdb_hash(url)` - MD5 hexdigest
+    * `Twingly::URL::Hasher.blogstream_hash(url)` - SHA256 unsigned long, native endian digest
+    * `Twingly::URL::Hasher.autopingdb_hash(url)` - SHA256 64-bit signed, native endian digest
+    * `Twingly::URL::Hasher.pingloggerdb_hash(url)` - SHA256 64-bit unsigned, native endian digest
+
+## Normalization example
+
+```ruby
+require 'twingly/url/normalizer'
+
+Twingly::URL::Normalizer.normalize('http://duh.se')
+# => ["http://www.duh.se/"]
+
+Twingly::URL::Normalizer.normalize('http://duh.se http://blog.twingly.com/')
+# => ["http://www.duh.se/", "http://blog.twingly.com/"]
+
+Twingly::URL::Normalizer.normalize('no URL')
+# => []
+```
+
+## Tests
+
+Run tests with
+
+    bundle exec rake
+
+### Profiling
+
+You can get some profiling by running
+
+    bundle exec rake test:profile
+
+Note that this isn't a benchmark, we're using [ruby-prof] which will slow things down.
+
+[ruby-prof]: http://ruby-prof.rubyforge.org/

data/lib/twingly-url-normalizer.rb

@@ -0,0 +1,2 @@
+warn "twingly-url-normalizer will be removed, use twingly/url/normalizer"
+require 'twingly/url/normalizer'

data/lib/twingly/url.rb

@@ -0,0 +1,36 @@
+require 'addressable/uri'
+require 'public_suffix'
+
+PublicSuffix::List.private_domains = false
+
+SCHEMES = %w(http https)
+
+module Twingly
+  module URL
+    module_function
+
+    UrlObject = Struct.new(:url, :domain) do
+      def valid?
+        url && domain && SCHEMES.include?(url.normalized_scheme)
+      end
+    end
+
+    def parse(potential_url)
+      url, domain = extract_url_and_domain(potential_url)
+      UrlObject.new(url, domain)
+    end
+
+    def extract_url_and_domain(potential_url)
+      url    = Addressable::URI.heuristic_parse(potential_url)
+      domain = PublicSuffix.parse(url.host) if url
+
+      [url, domain]
+    rescue PublicSuffix::DomainInvalid, Addressable::URI::InvalidURIError
+      []
+    end
+
+    def validate(potential_url)
+      parse(potential_url).valid?
+    end
+  end
+end

data/lib/twingly/url/hasher.rb

@@ -0,0 +1,30 @@
+require 'digest/md5'
+require 'digest/sha2'
+
+module Twingly
+  module URL
+    module Hasher
+      module_function
+
+      def taskdb_hash(url)
+        Digest::MD5.hexdigest(url)[0..29].upcase
+      end
+
+      def blogstream_hash(url)
+        Digest::MD5.hexdigest(url)[0..29].upcase
+      end
+
+      def documentdb_hash(url)
+        Digest::SHA256.digest(url).unpack("L!")[0]
+      end
+
+      def autopingdb_hash(url)
+        Digest::SHA256.digest(url).unpack("q")[0]
+      end
+
+      def pingloggerdb_hash(url)
+        Digest::SHA256.digest(url).unpack("Q")[0]
+      end
+    end
+  end
+end

data/lib/twingly/url/normalizer.rb

@@ -0,0 +1,36 @@
+require 'twingly/url'
+
+module Twingly
+  module URL
+    module Normalizer
+      module_function
+
+      def normalize(potential_urls)
+        extract_urls(potential_urls).map do |potential_url|
+          normalize_url(potential_url)
+        end.compact
+      end
+
+      def extract_urls(potential_urls)
+        Array(potential_urls).map(&:split).flatten
+      end
+
+      def normalize_url(potential_url)
+        result = Twingly::URL.parse(potential_url)
+
+        return nil unless result.valid?
+
+        unless result.domain.subdomain?
+          result.url.host = "www.#{result.domain}"
+        end
+
+        if result.url.path.empty?
+          result.url.path = "/"
+        end
+
+        result.url.to_s
+      end
+    end
+  end
+end
+

data/lib/version.rb

@@ -0,0 +1,5 @@
+module Twingly
+  module URL
+    VERSION = '1.3.1'
+  end
+end

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: twingly-url
+version: !ruby/object:Gem::Version
+  version: 1.3.1
+platform: ruby
+authors:
+- Johan Eckerström
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: public_suffix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+- !ruby/object:Gem::Dependency
+  name: shoulda-context
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Twingly URL tools
+email:
+- johan.eckerstrom@twingly.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/twingly-url-normalizer.rb
+- lib/twingly/url.rb
+- lib/twingly/url/hasher.rb
+- lib/twingly/url/normalizer.rb
+- lib/version.rb
+homepage: http://github.com/twingly/twingly-url
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Ruby library for URL handling
+test_files: []
+has_rdoc: