storexplore 0.0.1

data/spec/lib/storexplore/testing/dummy_store_api_spec.rb

@@ -0,0 +1,120 @@
+# -*- encoding: utf-8 -*-
+#
+# dummy_store_api_spec.rb
+#
+# Copyright (c) 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require 'spec_helper'
+
+module Storexplore
+  module Testing
+
+    describe "DummyStoreApi", slow: true do
+      include ApiSpecMacros
+
+      it_should_behave_like_any_store_items_api
+
+      DEFAULT_STORE_NAME = "www.spec-store.com"
+
+      def generate_store(store_name = DEFAULT_STORE_NAME, item_count = 3)
+        DummyStore.wipe_out_store(store_name)
+        @store_generator = DummyStore.open(store_name)
+        @store_generator.generate(3).categories.and(3).categories.and(item_count).items
+        @store = new_store(store_name)
+      end
+
+      def new_store(store_name = DEFAULT_STORE_NAME)
+        Api.browse(DummyStore.uri(store_name))
+      end
+
+      it "should not truncate long item names" do
+        @store_generator.
+          category(cat_name = "extra long category name").
+          category(sub_cat_name = "extra long sub category name").
+          item(item_name = "super extra long item name").generate().attributes
+
+        category = new_store.categories.find {|cat| cat_name.start_with?(cat.title)}
+        expect(category.attributes[:name]).to eq cat_name
+
+        sub_category = category.categories.find {|sub_cat| sub_cat_name.start_with?(sub_cat.title)}
+        expect(sub_category.attributes[:name]).to eq sub_cat_name
+
+        item = sub_category.items.find {|it| item_name.start_with?(it.title)}
+        expect(item.attributes[:name]).to eq item_name
+      end
+
+      it "should use constant memory" do
+        warm_up_measure = memory_usage_for_items(1)
+
+        small_inputs_memory = memory_usage_for_items(1)
+        large_inputs_memory = memory_usage_for_items(200)
+
+        expect(large_inputs_memory).to be <= small_inputs_memory * 1.25
+      end
+
+      def memory_usage_for_items(item_count)
+        generate_store(store_name = "www.spec-perf-store.com", item_count)
+        memory_peak_of do
+          walk_store(store_name)
+        end
+      end
+
+      def memory_peak_of
+        peak_usage = 0
+        finished = false
+
+        initial_usage = current_living_objects
+        profiler = Thread.new do
+          while not finished
+            peak_usage = [peak_usage, current_living_objects].max
+            sleep(0.01)
+          end
+        end
+
+        yield
+
+        finished = true
+        profiler.join
+
+        peak_usage - initial_usage
+      end
+
+      def current_living_objects
+        GC.start
+        object_counts = ObjectSpace.count_objects
+        object_counts[:TOTAL] - object_counts[:FREE]
+      end
+
+      def walk_store(store_name)
+        new_store(store_name).categories.each do |category|
+          @title = category.title
+          @attributes = category.attributes
+          category.categories.each do |sub_category|
+            @title = sub_category.title
+            @attributes = sub_category.attributes
+            category.items.each do |item|
+              @title = item.title
+              @attributes = item.attributes
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/spec/lib/storexplore/uri_utils_spec.rb

@@ -0,0 +1,51 @@
+# -*- encoding: utf-8 -*-
+#
+# uri_utils.rb
+#
+# Copyright (c) 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require 'spec_helper'
+
+module Storexplore
+
+  describe "UriUtils" do
+
+    it "should extract the domain of a domain only url" do
+      expect(UriUtils.domain(URI.parse("http://mes-courses.fr"))).to eq "mes-courses.fr"
+    end
+    it "should extract the domain of an url with a subdomain" do
+      expect(UriUtils.domain(URI.parse("http://www.yahoo.fr"))).to eq "yahoo.fr"
+    end
+    it "should extract the domain of an url with a directories" do
+      expect(UriUtils.domain(URI.parse("http://www.amazon.com/books/science-fiction"))).to eq "amazon.com"
+    end
+    it "should get localhost domain for a file uri" do
+      expect(UriUtils.domain(URI.parse("file:///home/user/documents/secret.txt"))).to eq "localhost"
+    end
+
+    it "should get a nil domain for a local path" do
+      expect(UriUtils.domain(URI.parse("root/folder/and/file.txt"))).to be_nil
+    end
+    it "should get a nil domain for a mailto uri" do
+      expect(UriUtils.domain(URI.parse("mailto:philou@mailinator.org"))).to be_nil
+    end
+    it "should get a nil domain for an url with an ip address" do
+      expect(UriUtils.domain(URI.parse("http://192.168.0.101/index.html"))).to be_nil
+    end
+  end
+end

data/spec/lib/storexplore/walker_page_spec.rb

@@ -0,0 +1,120 @@
+# -*- encoding: utf-8 -*-
+#
+# walker_page_spec.rb
+#
+# Copyright (c) 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require 'spec_helper'
+
+module Storexplore
+
+  # @integration
+  describe WalkerPage, slow: true do
+
+    before :each do
+      @uri = URI.parse("file://" + File.expand_path(File.join(File.dirname(__FILE__), 'store_walker_page_spec_fixture.html')))
+      @page_getter = WalkerPage.open(@uri)
+    end
+
+    context "before actually getting the page" do
+      it "nothing should throw if the uri is invalid" do
+        expect(lambda { WalkerPage.open("http://impossible.file.name") }).not_to raise_error
+      end
+
+      it "knows the uri of the page" do
+        expect(@page_getter.uri).to eq @uri
+      end
+
+      it "has text of the uri" do
+        expect(@page_getter.text).to eq @uri.to_s
+      end
+    end
+
+    context "after actually getting the page" do
+      before :each do
+        @page = @page_getter.get
+      end
+
+      it "delegates uri to the mechanize page" do
+        expect(@page.uri).to eq @uri
+      end
+
+      it "finds an element by css" do
+        element = @page.get_one("#unique")
+
+        expect(element).not_to be_nil
+        expect(element.attribute("id").value).to eq "unique"
+      end
+
+      it "finds only the first element by css" do
+        expect(@page.get_one(".number").text).to eq "0"
+      end
+
+      it "throws if it cannot find the element by css" do
+        expect(lambda { @page.get_one("#invalid_id") }).to raise_error(WalkerPageError)
+      end
+
+      it "finds the first element from a list of css" do
+        expect(@page.get_one(".absent, #unique")).to be @page.get_one("#unique")
+      end
+
+      it "finds all elements by css" do
+        expect(@page.get_all(".number", ', ')).to eq "0, 1"
+      end
+
+      it "throws if it cannot find at least one element by css" do
+        expect(lambda{ @page.get_all('#invalid_id', ', ') }).to raise_error(WalkerPageError)
+      end
+
+      it "finds relative links sorted by uri" do
+        links = @page.search_links("a.letter")
+
+        uris = links.map { |link| link.uri.to_s }
+        expect(uris).to eq ["a.html", "b.html"]
+      end
+
+      it "ignores links to other domains" do
+        expect(@page.search_links("#outbound")).to be_empty
+      end
+
+      it "ignores duplicate links" do
+        expect(@page.search_links("a.twin")).to have(1).link
+      end
+
+      it "links to other instances of WalkerPage" do
+        expect(@page.search_links("#myself").map { |link| link.get }).to all_ {be_instance_of(WalkerPage)}
+      end
+
+      it "knows the text of the links" do
+        @page.search_links("#myself").each do |link|
+          expect(link.text).to eq "myself"
+        end
+      end
+
+      it "finds an image by selector" do
+        image = @page.get_image(".image")
+        expect(image).to be_instance_of(Mechanize::Page::Image)
+        expect(image.dom_class).to eq "image"
+      end
+
+      it "throws if it cannot find the image by selector" do
+        expect(lambda { @page.get_image("#invalid_id") }).to raise_error(WalkerPageError)
+      end
+    end
+  end
+end

data/spec/lib/storexplore/walker_spec.rb

@@ -0,0 +1,97 @@
+# -*- encoding: utf-8 -*-
+#
+# walker_spec.rb
+#
+# Copyright (C) 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require "spec_helper"
+
+module Storexplore
+
+  describe Walker do
+
+    before :each do
+      @page = double("Page", :uri => "http://www.maxi-discount.com")
+      @page_getter = double("Getter", :get => @page, :text => "Conserves")
+      @walker = Walker.new(@page_getter)
+
+      @sub_walkers = [double("Sub walker")]
+      @digger = double(Digger)
+      @digger.stub(:sub_walkers).with(@page, @walker).and_return(@sub_walkers)
+    end
+
+    it "has the uri of its page" do
+      expect(@walker.uri).to eq @page.uri
+    end
+
+    it "it uses the text of its origin (ex: link) as title" do
+      expect(@walker.title).to eq @page_getter.text
+    end
+
+    context "by default" do
+      it "has no items" do
+        expect(@walker.items).to be_empty
+      end
+      it "has no sub categories" do
+        expect(@walker.categories).to be_empty
+      end
+      it "has no attributes" do
+        expect(@walker.attributes).to be_empty
+      end
+    end
+
+    it "uses its items digger to collect its items" do
+      @walker.items_digger = @digger
+
+      expect(@walker.items).to eq @sub_walkers
+    end
+    it "uses its categories digger to collect its sub categories" do
+      @walker.categories_digger = @digger
+
+      expect(@walker.categories).to eq @sub_walkers
+    end
+    it "uses its scrap attributes block to collect its attributes" do
+      attributes = { :name => "Candy" }
+      @walker.scrap_attributes_block = lambda { |page| attributes }
+
+      expect(@walker.attributes).to eq attributes
+    end
+
+    context "when troubleshooting" do
+
+      it "has a meaningfull string representation" do
+        walker = Walker.new(@page_getter)
+        walker.index= 23
+        expect(walker.to_s).to include(Walker.to_s)
+        expect(walker.to_s).to include("##{walker.index}")
+        expect(walker.to_s).to include("@#{walker.uri}")
+      end
+      it "has a full genealogy" do
+        link = double("Link")
+        link.stub_chain(:get, :uri).and_return(@page.uri + "/viandes")
+        child_walker = Walker.new(link)
+        child_walker.index = 12
+        child_walker.father = @walker
+
+        genealogy = child_walker.genealogy.split("\n")
+
+        expect(genealogy).to eq [@walker.to_s, child_walker.to_s]
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+#
+# spec_helper.rb
+#
+# Copyright (c) 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require 'spec_combos'
+require 'storexplore'
+require 'storexplore/testing'
+
+Storexplore::Testing.config do |config|
+  config.dummy_store_generation_dir= File.join(File.dirname(__FILE__), '../tmp')
+end

data/storexplore.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'storexplore/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "storexplore"
+  spec.version       = Storexplore::VERSION
+  spec.authors       = ["Philou"]
+  spec.email         = ["philippe.bourgau@gmail.com"]
+  spec.description   = %q{A declarative scrapping DSL that lets one define directory like apis to an online store}
+  spec.summary       = %q{Online store scraping library}
+  spec.homepage      = "https://github.com/philou/storexplore"
+  spec.license       = "LGPL v3"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "mechanize", "~> 2.0"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake", "~> 10.1"
+  spec.add_development_dependency "guard-rspec", "~> 4.0"
+  spec.add_development_dependency "spec_combos", "~> 0.2"
+end

metadata

@@ -0,0 +1,187 @@
+--- !ruby/object:Gem::Specification
+name: storexplore
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Philou
+autorequire: 
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIDkjCCAnqgAwIBAgIBATANBgkqhkiG9w0BAQUFADBHMRkwFwYDVQQDDBBwaGls
+  aXBwZS5ib3VyZ2F1MRUwEwYKCZImiZPyLGQBGRYFZ21haWwxEzARBgoJkiaJk/Is
+  ZAEZFgNjb20wHhcNMTMwOTIzMTEwOTA3WhcNMTQwOTIzMTEwOTA3WjBHMRkwFwYD
+  VQQDDBBwaGlsaXBwZS5ib3VyZ2F1MRUwEwYKCZImiZPyLGQBGRYFZ21haWwxEzAR
+  BgoJkiaJk/IsZAEZFgNjb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+  AQC8CpoqZwEbzXr55EUxdSplgn0MYZ9xPGO/XmRa8bD63n+JYWF0AS+mj452ZY18
+  rwM+yKrKhtsA+aJJdlOafgIUnY5SrZOr7v7kgc6T2YNoUj+M00Um2jv+shQbOtV6
+  qGp0Jw1HfPNUMVa+3pXZyAGCecN6rTnsZJIuW6KNaJUq6lEMVXanoTHgAKrH5aHd
+  Y6ofwQL86d6LDkC1S4p86iMUWvF34w8h5ItVo+JKlPRR22rzsK/ZKgNH3lfjbS6i
+  JWqPva70rL2xz5kCVn6DL7XhNZtqnAO4kvCQyQeWezvcoGXEnbHacKky7B+/WKec
+  OIWEwedl6j+X0OD5OYki3QaTAgMBAAGjgYgwgYUwCQYDVR0TBAIwADALBgNVHQ8E
+  BAMCBLAwHQYDVR0OBBYEFJnLz40Onu/dfpLSipU5FgTy6WLmMCUGA1UdEQQeMByB
+  GnBoaWxpcHBlLmJvdXJnYXVAZ21haWwuY29tMCUGA1UdEgQeMByBGnBoaWxpcHBl
+  LmJvdXJnYXVAZ21haWwuY29tMA0GCSqGSIb3DQEBBQUAA4IBAQBGoH72KWYACGZl
+  cMHMJ9d/DRU7rynJ8c4xuuM4c3Ri8bGPqI/a1BAp4QPJApS4+ANXXJ220hslrekP
+  9/ExEKFiqiywh1clih9ttuN4cHqJzCP6QHqJznQrvaYToZLxGARDf3Mwz4mFSh4W
+  snSep53DZ1vrY2Gzmig/4PuBF4q3nhwPgxV6H3SH4/Py7QF5COZPQlCdBwPYczqW
+  XxIbXhRVXcgjBHT0w2HsXkOwmmYvBzbrfqtTx5NswwHcIeQZB/NID7berIf9awx3
+  yLcl1cmm5ALtJ/+Bkkmp0i4amXeTDMvq9r8PBsVsQwxYOYJBP+Umxz3PX6HjFHrQ
+  XdkXx3oZ
+  -----END CERTIFICATE-----
+date: 2013-12-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.1'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: spec_combos
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+description: A declarative scrapping DSL that lets one define directory like apis
+  to an online store
+email:
+- philippe.bourgau@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/storexplore.rb
+- lib/storexplore/api.rb
+- lib/storexplore/api_builder.rb
+- lib/storexplore/array_utils.rb
+- lib/storexplore/browsing_error.rb
+- lib/storexplore/digger.rb
+- lib/storexplore/hash_utils.rb
+- lib/storexplore/null_digger.rb
+- lib/storexplore/testing.rb
+- lib/storexplore/testing/api_shared_examples.rb
+- lib/storexplore/testing/configuration.rb
+- lib/storexplore/testing/dummy_data.rb
+- lib/storexplore/testing/dummy_store.rb
+- lib/storexplore/testing/dummy_store_api.rb
+- lib/storexplore/testing/dummy_store_constants.rb
+- lib/storexplore/testing/dummy_store_generator.rb
+- lib/storexplore/testing/matchers/have_unique_matcher.rb
+- lib/storexplore/testing/matchers/mostly_matcher.rb
+- lib/storexplore/uri_utils.rb
+- lib/storexplore/version.rb
+- lib/storexplore/walker.rb
+- lib/storexplore/walker_page.rb
+- lib/storexplore/walker_page_error.rb
+- spec/lib/storexplore/api_builder_spec.rb
+- spec/lib/storexplore/api_spec.rb
+- spec/lib/storexplore/digger_spec.rb
+- spec/lib/storexplore/store_walker_page_spec_fixture.html
+- spec/lib/storexplore/testing/dummy_store_api_spec.rb
+- spec/lib/storexplore/uri_utils_spec.rb
+- spec/lib/storexplore/walker_page_spec.rb
+- spec/lib/storexplore/walker_spec.rb
+- spec/spec_helper.rb
+- storexplore.gemspec
+homepage: https://github.com/philou/storexplore
+licenses:
+- LGPL v3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Online store scraping library
+test_files:
+- spec/lib/storexplore/api_builder_spec.rb
+- spec/lib/storexplore/api_spec.rb
+- spec/lib/storexplore/digger_spec.rb
+- spec/lib/storexplore/store_walker_page_spec_fixture.html
+- spec/lib/storexplore/testing/dummy_store_api_spec.rb
+- spec/lib/storexplore/uri_utils_spec.rb
+- spec/lib/storexplore/walker_page_spec.rb
+- spec/lib/storexplore/walker_spec.rb
+- spec/spec_helper.rb