storexplore 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e30362dde2dd4fe6383bc76ff48c9de16642f54
-  data.tar.gz: 7d81d28ab8f78dc61c8334239161d437f1ce1f0b
+  metadata.gz: f7da700982a0fbe6f80b7aa37ff7e4f231f43935
+  data.tar.gz: 5fdb4f7684e506964f3fff42bdde441f70e71e92
 SHA512:
-  metadata.gz: 3d3b2741dd23d1142d54c99dbc4069445b46bc50b47dffed6f58a84d8f27fb3d70f3f359d980e1bb74ce433a83c32c80ffbba68bbc46723298df51cf3ad0d385
-  data.tar.gz: 5f9dcd46b4549a41a966cee549b956b3f32defef3ca013aaf996028a1e0d7c9e9e7f1857de603bd28201570f964f0cb0bd86e36db3eae6e058e09c1ab49c70bd
+  metadata.gz: 97e8fe6604856ffe5c4baa1287fe40ce5cfe40c4a950af74496bf848b3e42210973b218a117fc9a011d17814f82ca499e141fe810035e78a1d76a6bc45c58170
+  data.tar.gz: c5645631ef4ec8569277f15c2d7b9354f26a134833ee79e9cefd52ac0b3fea37919fabe8d51eab67be94479dd18c8cf2400aa8ca362d6aca846c6c1e41971c79

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 2.0.0
+# no lazy for enumerators in jruby 2.0
+#  - jruby-20mode
+# no rubynius working at the moment
+#  - rbx-20mode
+script: bundle exec rake

data/README.md

@@ -4,6 +4,9 @@ A declarative scrapping DSL that lets one define directory like apis to an onlin
 
 ## Installation
 
+In order to be able to enumerate all items of a store in constant memory,
+Storexplore requires Matz Ruby 2.0 for its lazy enumerators.
+
 Add this line to your application's Gemfile:
 
     gem 'storexplore'
@@ -18,7 +21,132 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+The library builds hierarchical APIs on online stores. Stores are typicaly
+organized in the following way :
+
+    Store > Categories > ... > Sub Categories > Items
+
+The store is like a root category. Any category, at any depth level can have
+both children categories and items. Items cannot have children of any kind.
+Both categories and items can have attributes.
+
+All searching of children and attributes is done through mechanize/nokogiri
+selectors (css or xpath).
+
+Here is a sample store api declaration :
+
+```ruby
+Storexplore::define_api 'dummy-store.com' do
+
+  categories 'a.category' do
+    attributes do
+      { :name => page.get_one("h1").content }
+    end
+
+    categories 'a.category' do
+      attributes do
+        { :name => page.get_one("h1").content }
+      end
+
+      items 'a.item' do
+        attributes do
+          {
+            :name => page.get_one('h1').content,
+            :brand => page.get_one('#brand').content,
+            :price => page.get_one('#price').content.to_f,
+            :image => page.get_one('#image').content,
+            :remote_id => page.get_one('#remote_id').content
+          }
+        end
+      end
+    end
+  end
+end
+```
+
+This build a hierarchical API on the 'dummy-store.com' online store. This
+registers a new api definition that will be used to browse any store which
+uri contains 'dummy-store.com'.
+
+Now here is how this API can be accessed to pretty print all its content:
+
+```ruby
+Api.browse('http://www.dummy-store.com').categories.each do |category|
+
+  puts "category: #{category.title}"
+  puts "attributes: #{category.attributes}"
+
+  category.categories.each do |sub_category|
+
+    puts "  category: #{sub_category.title}"
+    puts "  attributes: #{sub_category.attributes}"
+
+    sub_category.items.each do |item|
+
+      puts "    item: #{item.title}"
+      puts "    attributes: #{item.attributes}"
+
+    end
+  end
+end
+```
+
+### Testing
+
+Storexplore ships with some dummy store generation utilities. Dummy stores can
+be generated to the file system using the Storexplore::Testing::DummyStore and
+Storexplore::Testing::DummyStoreGenerator classes. This is particularly useful
+while testing.
+
+To use it, add the following, to your spec_helper.rb for example :
+
+```ruby
+require 'storexplore/testing'
+
+Storexplore::Testing.config do |config|
+  config.dummy_store_generation_dir= File.join(Rails.root, '../tmp')
+end
+```
+
+It is then possible to generate a store with the following :
+
+```ruby
+DummyStore.wipe_out_store(store_name)
+@store_generator = DummyStore.open(store_name)
+@store_generator.generate(3).categories.and(3).categories.and(item_count).items
+```
+
+It is also possibe to add elements with explicit values :
+
+```ruby
+@store_generator.
+  category(cat_name = "extra long category name").
+  category(sub_cat_name = "extra long sub category name").
+  item(item_name = "super extra long item name").generate().
+    attributes(price: 12.3)
+```
+
+Storexplore provides an api definition for dummy stores in
+'storexplore/testing/dummy_store_api'. It can be required independently if
+needed.
+
+### RSpec shared examples
+
+Storexplore also ships with an rspec shared examples macro. It can be used for
+any custom store API definition.
+
+```ruby
+require 'storexplore/testing'
+
+describe "MyStoreApi" do
+  include Storexplore::Testing::ApiSpecMacros
+
+  it_should_behave_like_any_store_items_api
+
+  ...
+
+end
+```
 
 ## Contributing
 

data/Rakefile

@@ -1 +1,7 @@
 require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+desc 'Run specs'
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/storexplore.rb

@@ -2,7 +2,7 @@
 #
 # storexplore.rb
 #
-# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (c) 2010, 2011, 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -22,9 +22,9 @@
 require "storexplore/version"
 require "storexplore/array_utils"
 require "storexplore/api"
-require "storexplore/api_builder"
 require "storexplore/browsing_error"
 require "storexplore/digger"
+require "storexplore/dsl"
 require "storexplore/hash_utils"
 require "storexplore/null_digger"
 require "storexplore/uri_utils"

data/lib/storexplore/api.rb

@@ -2,7 +2,7 @@
 #
 # api.rb
 #
-# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (c) 2010, 2011, 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -24,12 +24,18 @@ module Storexplore
   # Objects able to walk a store and discover available items
   class Api
 
+    def self.define(name, &block)
+      builder = Dsl.walker_builder(&block)
+
+      register_builder(name, builder)
+    end
+
     def self.browse(store_url)
-      builder(store_url).new(WalkerPage.open(store_url))
+      builder(store_url).new_walker(WalkerPage.open(store_url))
     end
 
-    def self.register_builder(name, builder)
-      builders[name] = builder
+    def self.undef(name)
+      builders.delete(name)
     end
 
     # Uri of the main page of the store
@@ -46,6 +52,10 @@ module Storexplore
 
     private
 
+    def self.register_builder(name, builder)
+      builders[name] = builder
+    end
+
     def self.builder(store_url)
       builders.each do |name, builder|
         if store_url.include?(name)

data/lib/storexplore/digger.rb

@@ -2,7 +2,7 @@
 #
 # digger.rb
 #
-# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (c) 2010, 2011, 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -21,14 +21,14 @@
 
 module Storexplore
   class Digger
-    def initialize(selector, factory)
+    def initialize(selector, sub_walker_builder)
       @selector = selector
-      @factory = factory
+      @sub_walker_builder = sub_walker_builder
     end
 
     def sub_walkers(page, father)
       page.search_links(@selector).each_with_index.to_a.lazy.map do |link, i|
-        @factory.new(link, father, i)
+        @sub_walker_builder.new_walker(link, father, i)
       end
     end
   end

data/lib/storexplore/dsl.rb

@@ -0,0 +1,61 @@
+# -*- encoding: utf-8 -*-
+#
+# dsl.rb
+#
+# Copyright (c) 2011, 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+
+  class Dsl
+
+    def self.walker_builder(&block)
+      new.tap do |dsl|
+        dsl.instance_eval(&block)
+      end
+    end
+
+    def initialize()
+      @scrap_attributes_block = lambda do |_| {} end
+      @categories_digger = NullDigger.new
+      @items_digger = NullDigger.new
+    end
+
+    def attributes(&block)
+      @scrap_attributes_block = block
+    end
+
+    def categories(selector, &block)
+      @categories_digger = Digger.new(selector, Dsl.walker_builder(&block))
+    end
+
+    def items(selector, &block)
+      @items_digger = Digger.new(selector, Dsl.walker_builder(&block))
+    end
+
+    def new_walker(page_getter, father = nil, index = nil)
+      Walker.new(page_getter).tap do |walker|
+        walker.categories_digger = @categories_digger
+        walker.items_digger = @items_digger
+        walker.scrap_attributes_block = @scrap_attributes_block
+        walker.father = father
+        walker.index = index
+      end
+    end
+  end
+
+end

data/lib/storexplore/testing/dummy_store_api.rb

@@ -2,7 +2,7 @@
 #
 # dummy_store_api.rb
 #
-# Copyright (c) 2012, 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (c) 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -24,7 +24,7 @@ require_relative 'dummy_store_constants'
 module Storexplore
   module Testing
 
-    Storexplore::define_api DummyStoreConstants::NAME do
+    Storexplore::Api.define DummyStoreConstants::NAME do
 
       categories 'a.category' do
         attributes do

data/lib/storexplore/version.rb

@@ -2,7 +2,7 @@
 #
 # version.rb
 #
-# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (c) 2010, 2011, 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,5 +20,5 @@
 # MA 02110-1301  USA
 
 module Storexplore
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/lib/storexplore/walker_page.rb

@@ -2,7 +2,7 @@
 #
 # walker_page.rb
 #
-# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (c) 2010, 2011, 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -21,19 +21,6 @@
 
 require 'mechanize'
 
-# monkey patch to avoid a regex uri encoding error when importing
-#      incompatible encoding regexp match (ASCII-8BIT regexp with UTF-8 string) (Encoding::CompatibilityError)
-#      /home/philou/.rbenv/versions/1.9.3-p194/lib/ruby/1.9.1/webrick/httputils.rb:353:in `gsub'
-#      /home/philou/.rbenv/versions/1.9.3-p194/lib/ruby/1.9.1/webrick/httputils.rb:353:in `_escape'
-#      /home/philou/.rbenv/versions/1.9.3-p194/lib/ruby/1.9.1/webrick/httputils.rb:363:in `escape'
-#      from uri method
-require "webrick/httputils"
-module WEBrick::HTTPUtils
-  def self.escape(s)
-    URI.escape(s)
-  end
-end
-
 module Storexplore
 
   class WalkerPage

data/spec/lib/storexplore/api_spec.rb

@@ -2,7 +2,7 @@
 #
 # api_spec.rb
 #
-# Copyright (C) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (C) 2010, 2011, 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -25,20 +25,27 @@ module Storexplore
 
   describe Api do
 
-    before :each do
-      Api.register_builder(my_store = "www.my-store.com", builder = double(ApiBuilder.class))
-      @url = "http://#{my_store}"
-      WalkerPage.stub(:open).with(@url).and_return(walker = double(WalkerPage))
-      builder.stub(:new).with(walker).and_return(@store_api = double(ApiBuilder))
+    before :all do
+      Storexplore::Api.define 'cats' do
+        attributes do
+          {animal: :cats}
+        end
+      end
     end
 
     it "select the good store items api builder to browse a store" do
-      expect(Api.browse(@url)).to eq @store_api
+      expect(Api.browse("http://www.cats.net").attributes[:animal]).to eq(:cats)
     end
 
     it "fails when it does not know how to browse a store" do
       expect(lambda { Api.browse("http://unknown.store.com") }).to raise_error(NotImplementedError)
     end
 
+    it "allows to unregister an installed api (mostly for testing)" do
+      Api.undef 'cats'
+
+      expect(lambda { Api.browse("http://www.cats.com") }).to raise_error(NotImplementedError)
+    end
+
   end
 end

data/spec/lib/storexplore/dsl_spec.rb

@@ -0,0 +1,215 @@
+# -*- encoding: utf-8 -*-
+#
+# dsl_spec.rb
+#
+# Copyright (C) 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require "spec_helper"
+
+module Storexplore
+
+  describe Dsl do
+
+    def browse
+      @walker = Storexplore::Api.browse("http://www.cats-surplus.com")
+    end
+
+    after :each do
+      Storexplore::Api.undef 'cats'
+    end
+
+    context 'a simple store' do
+      before :each do
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com", content_type: 'text/html', body: <<-eos)
+          <html>
+            <body>
+              <a href="category1.html" class="category">Cats with fur</a>
+              <a href="category2.html" class="category">Naked cats</a>
+              <a href="category3.html" class="category">Cats with feathers</a>
+
+              <a href="item1.html" class="item">The first thing we sell</a>
+              <a href="item2.html" class="item">The second thing we sell</a>
+              <a href="legal.html" class="legal">How we sell it</a>
+            </body>
+          </html>
+        eos
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/category1.html", content_type: 'text/html', body: <<-eos)
+          <html>
+            <body>
+              <a href="category4.html" class="sub-category">Cats with red fur</a>
+              <a href="category5.html" class="sub-category">Cats with green fur</a>
+
+              <a href="item3.html" class="item">The first thing we sell</a>
+              <a href="item4.html" class="item">The second thing we sell</a>
+              <a href="item5.html" class="item">The second thing we sell</a>
+            </body>
+          </html>
+        eos
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/category2.html", content_type: 'text/html', body: "")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/category3.html", content_type: 'text/html', body: "")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/category4.html", content_type: 'text/html', body: "")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/category5.html", content_type: 'text/html', body: "")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/item1.html", content_type: 'text/html', body: "")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/item2.html", content_type: 'text/html', body: "")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/item3.html", content_type: 'text/html', body: "")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/item4.html", content_type: 'text/html', body: "")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/item5.html", content_type: 'text/html', body: "")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/legal.html", content_type: 'text/html', body: "")
+
+        Storexplore::Api.define 'cats' do
+          items 'a.item' do
+            attributes do
+              { page: page,
+                this_is_the: :item}
+            end
+          end
+          categories 'a.category' do
+            categories 'a.sub-category' do
+            end
+            items 'a.item' do
+              attributes do
+                raise WalkerPageError.new("Dummy error message")
+              end
+            end
+            attributes do
+              { page: page,
+                this_is_the: :category}
+            end
+          end
+          attributes do
+            { page: page,
+              this_is_the: :root}
+          end
+        end
+
+        browse
+      end
+
+      it "root walker has the uri of the home page" do
+        expect(@walker.uri).to eq URI("http://www.cats-surplus.com")
+      end
+
+      it "sub walkers have the uri of their page" do
+        expect(@walker.items.first.uri).to eq URI("http://www.cats-surplus.com/item1.html")
+      end
+
+      it "root walker title is the store uri" do
+        expect(@walker.title).to eq "http://www.cats-surplus.com"
+      end
+
+      it "sub walkers title is the text of its origin link" do
+        expect(@walker.items.first.title).to eq "The first thing we sell"
+      end
+
+      it "uses a selector to spot sub items" do
+        expect(@walker.items).to have_exactly(2).items
+      end
+      it "uses a selector to spot sub categories" do
+        expect(@walker.categories).to have_exactly(3).categories
+      end
+      it "root walker can have attributes" do
+        expect(@walker.attributes[:this_is_the]).to eq :root
+        expect(@walker.attributes[:page]).to be_instance_of(WalkerPage)
+      end
+      it "categories can have sub categories" do
+        expect(@walker.categories.first.categories).to have_exactly(2).categories
+      end
+      it "categories can have sub items" do
+        expect(@walker.categories.first.items).to have_exactly(3).items
+      end
+      it "categories can have attributes" do
+        expect(@walker.categories.first.attributes[:this_is_the]).to eq :category
+        expect(@walker.categories.first.attributes[:page]).to be_instance_of(WalkerPage)
+      end
+      it "items can have attributes" do
+        expect(@walker.items.first.attributes[:this_is_the]).to eq :item
+        expect(@walker.items.first.attributes[:page]).to be_instance_of(WalkerPage)
+      end
+
+
+      context "when troubleshooting" do
+
+        before :each do
+          @sub_walker = @walker.categories.first.items.drop(1).first
+        end
+
+        it "walkers have an index" do
+          expect(@sub_walker.index).to eq 1
+        end
+
+        it "has a meaningfull string representation" do
+          expect(@sub_walker.to_s).to include(Walker.to_s)
+          expect(@sub_walker.to_s).to include("##{@sub_walker.index}")
+          expect(@sub_walker.to_s).to include("@#{@sub_walker.uri}")
+        end
+
+        it "has a full genealogy" do
+          genealogy = @sub_walker.genealogy.split("\n")
+
+          expect(genealogy).to eq [@walker.to_s, @walker.categories.first.to_s, @sub_walker.to_s]
+        end
+
+        it "wraps parsing errors with debug errors" do
+          expect(lambda { @sub_walker.attributes }).to raise_error(BrowsingError, "Dummy error message\n#{@sub_walker.genealogy}")
+        end
+      end
+
+    end
+
+    context 'a redirected home page' do
+      before :each do
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com", status: [301, "Moved Permanently"], location: "http://www.cats-surplus.com/index.html")
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com/index.html", content_type: 'text/html', body: "")
+
+        Storexplore::Api.define 'cats' do
+        end
+
+        browse
+      end
+
+      it "root walker has the uri of finaly page" do
+        expect(@walker.uri).to eq URI("http://www.cats-surplus.com/index.html")
+      end
+
+      it "root walker title is the store uri" do
+        expect(@walker.title).to eq "http://www.cats-surplus.com"
+      end
+    end
+
+    context 'an empty store' do
+      before :each do
+        FakeWeb.register_uri(:get, "http://www.cats-surplus.com", content_type: 'text/html', body: "")
+
+        Storexplore::Api.define 'cats' do
+        end
+
+        browse
+      end
+
+      it "has no items" do
+        expect(@walker.items).to be_empty
+      end
+      it "has no sub categories" do
+        expect(@walker.categories).to be_empty
+      end
+      it "has no sub attributes" do
+        expect(@walker.attributes).to be_empty
+      end
+    end
+  end
+end

data/spec/lib/storexplore/testing/dummy_store_api_spec.rb

@@ -2,7 +2,7 @@
 #
 # dummy_store_api_spec.rb
 #
-# Copyright (c) 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (c) 2011, 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -59,19 +59,28 @@ module Storexplore
       end
 
       it "should use constant memory" do
-        warm_up_measure = memory_usage_for_items(1)
+        FEW = 10
+        MANY = 100
+        RUNS = 2
 
-        small_inputs_memory = memory_usage_for_items(1)
-        large_inputs_memory = memory_usage_for_items(200)
+        many_inputs_memory = memory_usage_for_items(MANY, RUNS)
+        few_inputs_memory = memory_usage_for_items(FEW, RUNS)
 
-        expect(large_inputs_memory).to be <= small_inputs_memory * 1.25
+        slope = (many_inputs_memory - few_inputs_memory) / (MANY - FEW)
+
+        zero_inputs_memory = few_inputs_memory - FEW * slope
+
+        expect(slope).to be_within(zero_inputs_memory * 0.05).of(0.0)
       end
 
-      def memory_usage_for_items(item_count)
+      def memory_usage_for_items(item_count, runs)
         generate_store(store_name = "www.spec-perf-store.com", item_count)
-        memory_peak_of do
-          walk_store(store_name)
+        data = runs.times.map do
+          memory_peak_of do
+            walk_store(store_name)
+          end
         end
+        mean(data)
       end
 
       def memory_peak_of
@@ -95,25 +104,38 @@ module Storexplore
       end
 
       def current_living_objects
-        GC.start
         object_counts = ObjectSpace.count_objects
         object_counts[:TOTAL] - object_counts[:FREE]
       end
 
       def walk_store(store_name)
         new_store(store_name).categories.each do |category|
-          @title = category.title
-          @attributes = category.attributes
+          register(category)
+
           category.categories.each do |sub_category|
-            @title = sub_category.title
-            @attributes = sub_category.attributes
-            category.items.each do |item|
-              @title = item.title
-              @attributes = item.attributes
+            register(sub_category)
+
+            sub_category.items.each do |item|
+              register(item)
             end
           end
         end
       end
+
+      def register(store_node)
+        @title = store_node.title
+        @attributes = store_node.attributes
+
+        # No GC is explicitly done, because:
+        #  - large inputs forces it anyway
+        #  - it greatly slows tests
+        #  - GCing should not change the complexity of the system
+        # GC.start
+      end
+
+      def mean(data)
+        data.reduce(:+)/data.size
+      end
     end
 
   end

data/spec/lib/storexplore/walker_page_spec.rb

@@ -2,7 +2,7 @@
 #
 # walker_page_spec.rb
 #
-# Copyright (c) 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (c) 2011, 2012, 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -23,7 +23,6 @@ require 'spec_helper'
 
 module Storexplore
 
-  # @integration
   describe WalkerPage, slow: true do
 
     before :each do

data/spec/spec_helper.rb

@@ -2,7 +2,7 @@
 #
 # spec_helper.rb
 #
-# Copyright (c) 2013 by Philippe Bourgau. All rights reserved.
+# Copyright (c) 2013, 2014 by Philippe Bourgau. All rights reserved.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,10 +19,20 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA 02110-1301  USA
 
+require 'fakeweb'
 require 'spec_combos'
 require 'storexplore'
 require 'storexplore/testing'
 
+# Dummy store generation
 Storexplore::Testing.config do |config|
   config.dummy_store_generation_dir= File.join(File.dirname(__FILE__), '../tmp')
 end
+
+# Clean up fakeweb registry after every test
+FakeWeb.allow_net_connect = false
+RSpec.configure do |config|
+  config.after(:each) do
+    FakeWeb.clean_registry
+  end
+end

data/storexplore.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.1"
   spec.add_development_dependency "guard-rspec", "~> 4.0"
   spec.add_development_dependency "spec_combos", "~> 0.2"
+  spec.add_development_dependency "fakeweb", "~> 1.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: storexplore
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Philou
@@ -31,7 +31,7 @@ cert_chain:
   yLcl1cmm5ALtJ/+Bkkmp0i4amXeTDMvq9r8PBsVsQwxYOYJBP+Umxz3PX6HjFHrQ
   XdkXx3oZ
   -----END CERTIFICATE-----
-date: 2013-12-29 00:00:00.000000000 Z
+date: 2014-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -103,6 +103,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: fakeweb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
 description: A declarative scrapping DSL that lets one define directory like apis
   to an online store
 email:
@@ -113,6 +127,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .rspec
+- .travis.yml
 - Gemfile
 - Guardfile
 - LICENSE
@@ -120,10 +135,10 @@ files:
 - Rakefile
 - lib/storexplore.rb
 - lib/storexplore/api.rb
-- lib/storexplore/api_builder.rb
 - lib/storexplore/array_utils.rb
 - lib/storexplore/browsing_error.rb
 - lib/storexplore/digger.rb
+- lib/storexplore/dsl.rb
 - lib/storexplore/hash_utils.rb
 - lib/storexplore/null_digger.rb
 - lib/storexplore/testing.rb
@@ -141,14 +156,12 @@ files:
 - lib/storexplore/walker.rb
 - lib/storexplore/walker_page.rb
 - lib/storexplore/walker_page_error.rb
-- spec/lib/storexplore/api_builder_spec.rb
 - spec/lib/storexplore/api_spec.rb
-- spec/lib/storexplore/digger_spec.rb
+- spec/lib/storexplore/dsl_spec.rb
 - spec/lib/storexplore/store_walker_page_spec_fixture.html
 - spec/lib/storexplore/testing/dummy_store_api_spec.rb
 - spec/lib/storexplore/uri_utils_spec.rb
 - spec/lib/storexplore/walker_page_spec.rb
-- spec/lib/storexplore/walker_spec.rb
 - spec/spec_helper.rb
 - storexplore.gemspec
 homepage: https://github.com/philou/storexplore
@@ -176,12 +189,10 @@ signing_key:
 specification_version: 4
 summary: Online store scraping library
 test_files:
-- spec/lib/storexplore/api_builder_spec.rb
 - spec/lib/storexplore/api_spec.rb
-- spec/lib/storexplore/digger_spec.rb
+- spec/lib/storexplore/dsl_spec.rb
 - spec/lib/storexplore/store_walker_page_spec_fixture.html
 - spec/lib/storexplore/testing/dummy_store_api_spec.rb
 - spec/lib/storexplore/uri_utils_spec.rb
 - spec/lib/storexplore/walker_page_spec.rb
-- spec/lib/storexplore/walker_spec.rb
 - spec/spec_helper.rb

data/lib/storexplore/api_builder.rb

@@ -1,68 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# api_builder.rb
-#
-# Copyright (c) 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 3.0 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA 02110-1301  USA
-
-module Storexplore
-
-  class ApiBuilder
-
-    def self.define(api_class, digger_class, &block)
-      new(api_class, digger_class).tap do |result|
-        result.instance_eval(&block)
-      end
-    end
-
-    def initialize(api_class, digger_class)
-      @api_class = api_class
-      @digger_class = digger_class
-      @scrap_attributes_block = lambda do {} end
-      @categories_digger = NullDigger.new
-      @items_digger = NullDigger.new
-    end
-
-    def attributes(&block)
-      @scrap_attributes_block = block
-    end
-
-    def categories(selector, &block)
-      @categories_digger = @digger_class.new(selector, ApiBuilder.define(@api_class, @digger_class, &block))
-    end
-
-    def items(selector, &block)
-      @items_digger = @digger_class.new(selector, ApiBuilder.define(@api_class, @digger_class, &block))
-    end
-
-    def new(page_getter, father = nil, index = nil)
-      @api_class.new(page_getter).tap do |result|
-        result.categories_digger = @categories_digger
-        result.items_digger = @items_digger
-        result.scrap_attributes_block = @scrap_attributes_block
-        result.father = father
-        result.index = index
-      end
-    end
-  end
-
-  def self.define_api(name, &block)
-    builder = ApiBuilder.define(Walker, Digger, &block)
-
-    Api.register_builder(name, builder)
-  end
-end

data/spec/lib/storexplore/api_builder_spec.rb

@@ -1,99 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# api_builder_spec.rb
-#
-# Copyright (c) 2012, 2013 by Philippe Bourgau. All rights reserved.
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 3.0 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA 02110-1301  USA
-
-require "spec_helper"
-
-module Storexplore
-
-  describe ApiBuilder do
-
-    before :each do
-      @url = "http://www.mega-store.com"
-      @api = double("Store api").as_null_object
-      @api_class = double("Store api class")
-      @api_class.stub(:new).with(@url).and_return(@api)
-
-      @selector = "a.child"
-      @digger = double("Digger")
-      @digger_class = double("Digger class")
-    end
-
-    context "using define method" do
-      it "creates new store api" do
-        @builder = ApiBuilder.define(@api_class, Digger) { }
-
-        expect(@builder.new(@url)).to eq @api
-      end
-
-      it "initializes nested definition through its block" do
-        ApiBuilder.stub(:new).and_return(builder = double(ApiBuilder))
-
-        expect(builder).to receive(:complex_builder_initialization)
-
-        ApiBuilder.define(@api_class, Digger) do
-          complex_builder_initialization
-        end
-      end
-    end
-
-    context "when nesting definitions" do
-
-      before :each do
-        @builder = ApiBuilder.new(@api_class, @digger_class)
-      end
-
-      after :each do
-        @builder.new(@url)
-      end
-
-      [:categories, :items].each do |sub_definition|
-
-        before :each do
-          ApiBuilder.stub(:new).and_return(@sub_builder = double(ApiBuilder))
-          @digger_class.stub(:new).with(@selector, @sub_builder).and_return(@digger)
-        end
-
-        it "tells the store api how to find sub #{sub_definition}" do
-          expect(@api).to receive("#{sub_definition}_digger=").with(@digger)
-
-          @builder.send(sub_definition, @selector) do end
-        end
-
-        it "initialises the sub #{sub_definition} builder" do
-          expect(@sub_builder).to receive(:sub_builder_initialization)
-
-          @builder.send(sub_definition, @selector) do
-            sub_builder_initialization
-          end
-        end
-      end
-
-      it "tells the store api how to parse attributes" do
-        scrap_attributes_block = Proc.new { |page| @scrap_attributes_block_is_unique = true }
-
-        expect(@api).to receive(:scrap_attributes_block=).with(scrap_attributes_block)
-
-        @builder.attributes(&scrap_attributes_block)
-      end
-
-    end
-  end
-end

data/spec/lib/storexplore/digger_spec.rb

@@ -1,53 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# digger_spec.rb
-#
-# Copyright (c) 2012, 2013 by Philippe Bourgau. All rights reserved.
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 3.0 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA 02110-1301  USA
-
-require "spec_helper"
-
-module Storexplore
-
-  describe Digger do
-
-    before :each do
-      @digger = Digger.new(@selector = "a.items", @factory = double("Sub walker factory"))
-      @page = double(WalkerPage)
-      @page.stub(:search_links).with(@selector).and_return(@links = [double("Link"),double("Link")])
-    end
-
-    it "creates sub walkers for each link it finds" do
-      @links.each do |link|
-        expect(@factory).to receive(:new).with(link, anything, anything)
-      end
-
-      @digger.sub_walkers(@page, nil).to_a
-    end
-
-    it "for debugging purpose, provides father walker and link index to sub walkers" do
-      father = double("Father walker")
-
-      @links.each_with_index do |link, index|
-        expect(@factory).to receive(:new).with(link, father, index)
-      end
-
-      @digger.sub_walkers(@page, father).to_a
-    end
-
-  end
-end

data/spec/lib/storexplore/walker_spec.rb

@@ -1,97 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# walker_spec.rb
-#
-# Copyright (C) 2012, 2013 by Philippe Bourgau. All rights reserved.
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 3.0 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA 02110-1301  USA
-
-require "spec_helper"
-
-module Storexplore
-
-  describe Walker do
-
-    before :each do
-      @page = double("Page", :uri => "http://www.maxi-discount.com")
-      @page_getter = double("Getter", :get => @page, :text => "Conserves")
-      @walker = Walker.new(@page_getter)
-
-      @sub_walkers = [double("Sub walker")]
-      @digger = double(Digger)
-      @digger.stub(:sub_walkers).with(@page, @walker).and_return(@sub_walkers)
-    end
-
-    it "has the uri of its page" do
-      expect(@walker.uri).to eq @page.uri
-    end
-
-    it "it uses the text of its origin (ex: link) as title" do
-      expect(@walker.title).to eq @page_getter.text
-    end
-
-    context "by default" do
-      it "has no items" do
-        expect(@walker.items).to be_empty
-      end
-      it "has no sub categories" do
-        expect(@walker.categories).to be_empty
-      end
-      it "has no attributes" do
-        expect(@walker.attributes).to be_empty
-      end
-    end
-
-    it "uses its items digger to collect its items" do
-      @walker.items_digger = @digger
-
-      expect(@walker.items).to eq @sub_walkers
-    end
-    it "uses its categories digger to collect its sub categories" do
-      @walker.categories_digger = @digger
-
-      expect(@walker.categories).to eq @sub_walkers
-    end
-    it "uses its scrap attributes block to collect its attributes" do
-      attributes = { :name => "Candy" }
-      @walker.scrap_attributes_block = lambda { |page| attributes }
-
-      expect(@walker.attributes).to eq attributes
-    end
-
-    context "when troubleshooting" do
-
-      it "has a meaningfull string representation" do
-        walker = Walker.new(@page_getter)
-        walker.index= 23
-        expect(walker.to_s).to include(Walker.to_s)
-        expect(walker.to_s).to include("##{walker.index}")
-        expect(walker.to_s).to include("@#{walker.uri}")
-      end
-      it "has a full genealogy" do
-        link = double("Link")
-        link.stub_chain(:get, :uri).and_return(@page.uri + "/viandes")
-        child_walker = Walker.new(link)
-        child_walker.index = 12
-        child_walker.father = @walker
-
-        genealogy = child_walker.genealogy.split("\n")
-
-        expect(genealogy).to eq [@walker.to_s, child_walker.to_s]
-      end
-    end
-  end
-end