crawl 1.1.4 → 1.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 103da22dba2987ec6ec5bbf700473cb243170fc5
-  data.tar.gz: 8197231448106e4fe0b198b53e734110eb1748a9
+  metadata.gz: cd091ac6b2679c9dbe5336a73dfd7acab9e9ee65
+  data.tar.gz: 6457eced0c6e538809ad22fc5045615d37ce8a78
 SHA512:
-  metadata.gz: 206ff34e285a074b7dad8ebb68453295df14d67b95b3a4755d823a0d4bd2a3bae92859ff02fe52dee6b134c02966c7f4d9c61e52e1938c3815d56fc7debf7387
-  data.tar.gz: 2f8f655862a28cd4772317866b72ca31c392c7c63997226bac8c2ceccfe15fca203501189a4a0b9db0ebe002555fc9932265a016cd63c247b85b355398434592
+  metadata.gz: 5538098c9d5b5f6785524e1378564a7121443dca24c63eeb4ffb9f12721afbd9d09db08e3bbea22be560917468405bb519e1d09f24c683316e085d2ddf3e17a2
+  data.tar.gz: 3f53020b8432926125b338f5114bdcdeb35560b7060d6d123855f5d69671a8398710d0b9a6a3d1e9ae333ba6635d03570218ac0c2339804c0d1109063bc48b1b

data/lib/crawl/engine.rb

@@ -21,7 +21,7 @@ class Crawl::Engine
     @authorization = Base64.encode64("#{options[:username]}:#{options[:password]}")
     @register = Crawl::Register.new
 
-    start_pages = options[:start].to_a.map{|page| Page.new(@register, page, 'the command line')}
+    start_pages = options[:start].to_a.map{|page| Page.new(@register, page, '/')}
 
     @register.add(start_pages)
   end
@@ -109,4 +109,4 @@ private
     raw_links.delete_if{|link| IGNORE.any?{|pattern| link =~ pattern}}
     raw_links.map{ |url| Page.new(@register, url, page.url) }
   end
-end
+end

data/lib/crawl/page.rb

@@ -1,3 +1,5 @@
+require 'uri'
+
 class Page
   include Comparable
 
@@ -14,27 +16,19 @@ class Page
   end
 
   def relative_url
-    if url.start_with?('/')
-      url
-    else
-      "#{source_directory}/#{url}"
-    end
-  end
-
-  def source_directory
-    File.split(source).first.sub(/^\./, '').sub(/\/$/, '')
+    @relative_url ||= URI.join('http://example.com', source, url).path
   end
 
   def <=>(other)
-    url <=> other.url
+    relative_url <=> other.relative_url
   end
 
   def eql?(other)
-    url.eql?(other.url)
+    relative_url.eql?(other.relative_url)
   end
 
   def hash
-    url.hash
+    relative_url.hash
   end
 
   def success

data/lib/crawl/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Crawl
-  VERSION = "1.1.4"
+  VERSION = "1.1.6"
 end

data/spec/page_spec.rb

@@ -3,9 +3,11 @@ require './lib/crawl/page'
 RSpec.describe Page do
   describe "#relative_url" do
     specify { expect(Page.new(:register, "/", "/").relative_url).to eq "/" }
+    specify { expect(Page.new(:register, "./", "/").relative_url).to eq "/" }
     specify { expect(Page.new(:register, "page.html", "").relative_url).to eq "/page.html" }
     specify { expect(Page.new(:register, "/interview", "/").relative_url).to eq "/interview" }
     specify { expect(Page.new(:register, "overview.html", "/").relative_url).to eq "/overview.html" }
     specify { expect(Page.new(:register, "post-5.html", "/posts/index.html").relative_url).to eq "/posts/post-5.html" }
+    specify { expect(Page.new(:register, "https://staging.alphasights.com/careers/meet-us", "/posts/foo").relative_url).to eq "/careers/meet-us" }
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawl
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.1.6
 platform: ruby
 authors:
 - Tor Erik Linnerud