storexplore 0.0.1

data/lib/storexplore/testing/matchers/mostly_matcher.rb

@@ -0,0 +1,45 @@
+# -*- encoding: utf-8 -*-
+#
+# mostly_matcher.rb
+#
+# Copyright (c) 2010, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+# Matcher to verify that most items match something else
+RSpec::Matchers.define :mostly do |item_matcher|
+
+  match do |actual_items|
+    expected_uniques_count(actual_items) <= actual_matches(actual_items, item_matcher).length
+  end
+  description do
+    "#{item_matcher.description} to be true for at least #{threshold*100}% of the items"
+  end
+
+  #private
+
+  def expected_uniques_count(actual_items)
+    (actual_items.length * threshold).round
+  end
+  def actual_matches(actual_items, item_matcher)
+    actual_items.find_all {|item| item_matcher.matches?(item)}
+  end
+
+  def threshold
+    0.7
+  end
+
+end

data/lib/storexplore/testing.rb

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+#
+# storexplore.rb
+#
+# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require 'storexplore'
+require 'storexplore/testing/api_shared_examples'
+require 'storexplore/testing/configuration'
+require 'storexplore/testing/dummy_data'
+require 'storexplore/testing/dummy_store'
+require 'storexplore/testing/dummy_store_api'
+require 'storexplore/testing/dummy_store_generator'
+require 'storexplore/testing/matchers/have_unique_matcher'
+require 'storexplore/testing/matchers/mostly_matcher'

data/lib/storexplore/uri_utils.rb

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+#
+# uri.rb
+#
+# Copyright (c) 2011, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+
+module Storexplore
+
+  # Extra URI utilities
+  class UriUtils
+
+    # extracts the domain from an uri
+    def self.domain(uri)
+
+      return "localhost" if uri.scheme == "file"
+      return nil if uri.host.nil?
+      return nil if uri.host =~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/
+
+      /([^\.]+\.[^\.]+)$/.match(uri.host)[0]
+    end
+  end
+end

data/lib/storexplore/version.rb

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+#
+# version.rb
+#
+# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+  VERSION = "0.0.1"
+end

data/lib/storexplore/walker.rb

@@ -0,0 +1,84 @@
+# -*- encoding: utf-8 -*-
+#
+# walker.rb
+#
+# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+
+  class Walker
+
+    attr_accessor :categories_digger, :items_digger, :scrap_attributes_block, :father, :index
+
+    def initialize(getter)
+      self.categories_digger = NullDigger.new
+      self.items_digger = NullDigger.new
+      self.scrap_attributes_block = proc do { } end
+      @getter = getter
+    end
+
+    def title
+      @getter.text
+    end
+
+    def uri
+      page.uri
+    end
+
+    def attributes
+      @attributes ||= scrap_attributes
+    end
+
+    def categories
+      categories_digger.sub_walkers(page, self)
+    end
+
+    def items
+      items_digger.sub_walkers(page, self)
+    end
+
+    def to_s
+      "#{self.class} ##{index} @#{uri}"
+    end
+
+    def genealogy
+      genealogy_prefix + to_s
+    end
+
+    private
+    def page
+      @page ||= @getter.get
+    end
+
+    def genealogy_prefix
+      if father.nil?
+        ""
+      else
+        father.genealogy + "\n"
+      end
+    end
+
+    def scrap_attributes
+      begin
+        instance_eval(&@scrap_attributes_block)
+      rescue WalkerPageError => e
+        raise BrowsingError.new("#{e.message}\n#{genealogy}")
+      end
+    end
+  end
+end

data/lib/storexplore/walker_page.rb

@@ -0,0 +1,142 @@
+# -*- encoding: utf-8 -*-
+#
+# walker_page.rb
+#
+# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require 'mechanize'
+
+# monkey patch to avoid a regex uri encoding error when importing
+#      incompatible encoding regexp match (ASCII-8BIT regexp with UTF-8 string) (Encoding::CompatibilityError)
+#      /home/philou/.rbenv/versions/1.9.3-p194/lib/ruby/1.9.1/webrick/httputils.rb:353:in `gsub'
+#      /home/philou/.rbenv/versions/1.9.3-p194/lib/ruby/1.9.1/webrick/httputils.rb:353:in `_escape'
+#      /home/philou/.rbenv/versions/1.9.3-p194/lib/ruby/1.9.1/webrick/httputils.rb:363:in `escape'
+#      from uri method
+require "webrick/httputils"
+module WEBrick::HTTPUtils
+  def self.escape(s)
+    URI.escape(s)
+  end
+end
+
+module Storexplore
+
+  class WalkerPage
+    extend Forwardable
+
+    def self.open(uri)
+      Getter.new(uri)
+    end
+
+    def_delegator :@mechanize_page, :uri
+
+    def search_links(selector)
+      uri2links = {}
+      search_all_links(selector).each do |link|
+        target_uri = link.uri
+        uri2links[target_uri.to_s] = link if same_domain? uri, target_uri
+      end
+      # enforcing deterministicity for testing and debugging
+      uri2links.values.sort_by {|link| link.uri.to_s }
+    end
+
+    def get_one(selector)
+      first_or_throw(@mechanize_page.search(selector), "elements", selector)
+    end
+
+    def get_all(selector, separator)
+      elements = @mechanize_page.search(selector)
+      throw_if_empty(elements, "elements", selector)
+
+      (elements.map &:text).join(separator)
+    end
+
+    def get_image(selector)
+      first_or_throw(@mechanize_page.images_with(search: selector), "images", selector)
+    end
+
+    private
+
+    def initialize(mechanize_page)
+      @mechanize_page = mechanize_page
+    end
+
+    def same_domain?(source_uri, target_uri)
+      target_uri.relative? || (UriUtils.domain(source_uri) == UriUtils.domain(target_uri))
+    end
+
+    def search_all_links(selector)
+      @mechanize_page.links_with(search: selector).map { |link| Link.new(link) }
+    end
+
+    def first_or_throw(elements, name, selector)
+      throw_if_empty(elements, name, selector)
+      elements.first
+    end
+
+    def throw_if_empty(elements, name, selector)
+      if elements.empty?
+        raise WalkerPageError.new("Page \"#{uri}\" does not contain any #{name} like \"#{selector}\"")
+      end
+    end
+
+    class Getter
+      attr_reader :uri
+
+      def initialize(uri)
+        @uri = uri
+      end
+      def get
+        @page ||= get_page
+      end
+
+      def text
+        @uri.to_s
+      end
+
+      private
+
+      def get_page
+        agent = Mechanize.new do |it|
+          # NOTE: by default Mechanize has infinite history, and causes memory leaks
+          it.history.max_size = 0
+        end
+
+        WalkerPage.new(agent.get(@uri))
+      end
+    end
+
+    class Link
+      extend Forwardable
+
+      def initialize(mechanize_link)
+        @mechanize_link = mechanize_link
+      end
+
+      def_delegator :@mechanize_link, :uri
+
+      def get
+        WalkerPage.new(@mechanize_link.click)
+      end
+
+      def text
+        @mechanize_link.text
+      end
+    end
+  end
+end

data/lib/storexplore/walker_page_error.rb

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+#
+# walker_page_error.rb
+#
+# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+  class WalkerPageError < StandardError
+  end
+end

data/lib/storexplore.rb

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+#
+# storexplore.rb
+#
+# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require "storexplore/version"
+require "storexplore/array_utils"
+require "storexplore/api"
+require "storexplore/api_builder"
+require "storexplore/browsing_error"
+require "storexplore/digger"
+require "storexplore/hash_utils"
+require "storexplore/null_digger"
+require "storexplore/uri_utils"
+require "storexplore/walker"
+require "storexplore/walker_page"
+require "storexplore/walker_page_error"
+

data/spec/lib/storexplore/api_builder_spec.rb

@@ -0,0 +1,99 @@
+# -*- encoding: utf-8 -*-
+#
+# api_builder_spec.rb
+#
+# Copyright (c) 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require "spec_helper"
+
+module Storexplore
+
+  describe ApiBuilder do
+
+    before :each do
+      @url = "http://www.mega-store.com"
+      @api = double("Store api").as_null_object
+      @api_class = double("Store api class")
+      @api_class.stub(:new).with(@url).and_return(@api)
+
+      @selector = "a.child"
+      @digger = double("Digger")
+      @digger_class = double("Digger class")
+    end
+
+    context "using define method" do
+      it "creates new store api" do
+        @builder = ApiBuilder.define(@api_class, Digger) { }
+
+        expect(@builder.new(@url)).to eq @api
+      end
+
+      it "initializes nested definition through its block" do
+        ApiBuilder.stub(:new).and_return(builder = double(ApiBuilder))
+
+        expect(builder).to receive(:complex_builder_initialization)
+
+        ApiBuilder.define(@api_class, Digger) do
+          complex_builder_initialization
+        end
+      end
+    end
+
+    context "when nesting definitions" do
+
+      before :each do
+        @builder = ApiBuilder.new(@api_class, @digger_class)
+      end
+
+      after :each do
+        @builder.new(@url)
+      end
+
+      [:categories, :items].each do |sub_definition|
+
+        before :each do
+          ApiBuilder.stub(:new).and_return(@sub_builder = double(ApiBuilder))
+          @digger_class.stub(:new).with(@selector, @sub_builder).and_return(@digger)
+        end
+
+        it "tells the store api how to find sub #{sub_definition}" do
+          expect(@api).to receive("#{sub_definition}_digger=").with(@digger)
+
+          @builder.send(sub_definition, @selector) do end
+        end
+
+        it "initialises the sub #{sub_definition} builder" do
+          expect(@sub_builder).to receive(:sub_builder_initialization)
+
+          @builder.send(sub_definition, @selector) do
+            sub_builder_initialization
+          end
+        end
+      end
+
+      it "tells the store api how to parse attributes" do
+        scrap_attributes_block = Proc.new { |page| @scrap_attributes_block_is_unique = true }
+
+        expect(@api).to receive(:scrap_attributes_block=).with(scrap_attributes_block)
+
+        @builder.attributes(&scrap_attributes_block)
+      end
+
+    end
+  end
+end

data/spec/lib/storexplore/api_spec.rb

@@ -0,0 +1,44 @@
+# -*- encoding: utf-8 -*-
+#
+# api_spec.rb
+#
+# Copyright (C) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require 'spec_helper'
+
+module Storexplore
+
+  describe Api do
+
+    before :each do
+      Api.register_builder(my_store = "www.my-store.com", builder = double(ApiBuilder.class))
+      @url = "http://#{my_store}"
+      WalkerPage.stub(:open).with(@url).and_return(walker = double(WalkerPage))
+      builder.stub(:new).with(walker).and_return(@store_api = double(ApiBuilder))
+    end
+
+    it "select the good store items api builder to browse a store" do
+      expect(Api.browse(@url)).to eq @store_api
+    end
+
+    it "fails when it does not know how to browse a store" do
+      expect(lambda { Api.browse("http://unknown.store.com") }).to raise_error(NotImplementedError)
+    end
+
+  end
+end

data/spec/lib/storexplore/digger_spec.rb

@@ -0,0 +1,53 @@
+# -*- encoding: utf-8 -*-
+#
+# digger_spec.rb
+#
+# Copyright (c) 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require "spec_helper"
+
+module Storexplore
+
+  describe Digger do
+
+    before :each do
+      @digger = Digger.new(@selector = "a.items", @factory = double("Sub walker factory"))
+      @page = double(WalkerPage)
+      @page.stub(:search_links).with(@selector).and_return(@links = [double("Link"),double("Link")])
+    end
+
+    it "creates sub walkers for each link it finds" do
+      @links.each do |link|
+        expect(@factory).to receive(:new).with(link, anything, anything)
+      end
+
+      @digger.sub_walkers(@page, nil).to_a
+    end
+
+    it "for debugging purpose, provides father walker and link index to sub walkers" do
+      father = double("Father walker")
+
+      @links.each_with_index do |link, index|
+        expect(@factory).to receive(:new).with(link, father, index)
+      end
+
+      @digger.sub_walkers(@page, father).to_a
+    end
+
+  end
+end

data/spec/lib/storexplore/store_walker_page_spec_fixture.html

@@ -0,0 +1,21 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <div id="unique"></div>
+    <p class="number">0</p>
+    <p class="number">1</p>
+
+    <a href="b.html" class="letter">b</a>
+    <a href="a.html" class="letter">a</a>
+
+    <a href="http://www.twilight-zone.com" id="outbound">twilight</a>
+
+    <a href="unique.html" class="twin">first twin</a>
+    <a href="unique.html" class="twin">second twin</a>
+
+    <a href="store_walker_page_spec_fixture.html" id="myself">myself</a>
+
+    <img class="image" src="toto.txt"/>
+  </body>
+</html>