storexplore 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4e30362dde2dd4fe6383bc76ff48c9de16642f54
+  data.tar.gz: 7d81d28ab8f78dc61c8334239161d437f1ce1f0b
+SHA512:
+  metadata.gz: 3d3b2741dd23d1142d54c99dbc4069445b46bc50b47dffed6f58a84d8f27fb3d70f3f359d980e1bb74ce433a83c32c80ffbba68bbc46723298df51cf3ad0d385
+  data.tar.gz: 5f9dcd46b4549a41a966cee549b956b3f32defef3ca013aaf996028a1e0d7c9e9e7f1857de603bd28201570f964f0cb0bd86e36db3eae6e058e09c1ab49c70bd

data/.gitignore

@@ -0,0 +1,26 @@
+*.gem
+*.rbc
+.bundle
+.config
+.ruby-version
+Gemfile.lock
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+
+# emacs files
+*~
+*#
+.#*

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in storexplore.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,9 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/LICENSE

@@ -0,0 +1,165 @@
+GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/README.md

@@ -0,0 +1,29 @@
+# Storexplore
+
+A declarative scrapping DSL that lets one define directory like apis to an online store
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'storexplore'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install storexplore
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/storexplore/api.rb

@@ -0,0 +1,63 @@
+# -*- encoding: utf-8 -*-
+#
+# api.rb
+#
+# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+
+  # Objects able to walk a store and discover available items
+  class Api
+
+    def self.browse(store_url)
+      builder(store_url).new(WalkerPage.open(store_url))
+    end
+
+    def self.register_builder(name, builder)
+      builders[name] = builder
+    end
+
+    # Uri of the main page of the store
+    # def uri
+
+    # Attributes of the page
+    # def attributes
+
+    # Walkers of the root categories of the store
+    # def categories
+
+    # Walkers of the root items in the store
+    # def items
+
+    private
+
+    def self.builder(store_url)
+      builders.each do |name, builder|
+        if store_url.include?(name)
+          return builder
+        end
+      end
+      raise NotImplementedError.new("Could not find a store item api for '#{store_url}'")
+    end
+
+    def self.builders
+      @builders ||= {}
+    end
+  end
+
+end

data/lib/storexplore/api_builder.rb

@@ -0,0 +1,68 @@
+# -*- encoding: utf-8 -*-
+#
+# api_builder.rb
+#
+# Copyright (c) 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+
+  class ApiBuilder
+
+    def self.define(api_class, digger_class, &block)
+      new(api_class, digger_class).tap do |result|
+        result.instance_eval(&block)
+      end
+    end
+
+    def initialize(api_class, digger_class)
+      @api_class = api_class
+      @digger_class = digger_class
+      @scrap_attributes_block = lambda do {} end
+      @categories_digger = NullDigger.new
+      @items_digger = NullDigger.new
+    end
+
+    def attributes(&block)
+      @scrap_attributes_block = block
+    end
+
+    def categories(selector, &block)
+      @categories_digger = @digger_class.new(selector, ApiBuilder.define(@api_class, @digger_class, &block))
+    end
+
+    def items(selector, &block)
+      @items_digger = @digger_class.new(selector, ApiBuilder.define(@api_class, @digger_class, &block))
+    end
+
+    def new(page_getter, father = nil, index = nil)
+      @api_class.new(page_getter).tap do |result|
+        result.categories_digger = @categories_digger
+        result.items_digger = @items_digger
+        result.scrap_attributes_block = @scrap_attributes_block
+        result.father = father
+        result.index = index
+      end
+    end
+  end
+
+  def self.define_api(name, &block)
+    builder = ApiBuilder.define(Walker, Digger, &block)
+
+    Api.register_builder(name, builder)
+  end
+end

data/lib/storexplore/array_utils.rb

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# array_utils.rb
+#
+# Copyright (c) 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+
+  class ArrayUtils
+
+    def self.contains?(array, other)
+      other.all? do |x|
+        array.include?(x)
+      end
+    end
+
+    def self.stringify(array)
+      array.map &:to_s
+    end
+  end
+end

data/lib/storexplore/browsing_error.rb

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+#
+# browsing_error.rh
+#
+# Copyright (c) 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+  # Error thrown when store item browsing fails due to unexpected page formatting
+  class BrowsingError < StandardError
+  end
+end

data/lib/storexplore/digger.rb

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+#
+# digger.rb
+#
+# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+  class Digger
+    def initialize(selector, factory)
+      @selector = selector
+      @factory = factory
+    end
+
+    def sub_walkers(page, father)
+      page.search_links(@selector).each_with_index.to_a.lazy.map do |link, i|
+        @factory.new(link, father, i)
+      end
+    end
+  end
+end

data/lib/storexplore/hash_utils.rb

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# hash_utils.rb
+#
+# Copyright (c) 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+require "fileutils"
+
+module Storexplore
+
+  class HashUtils
+
+    def self.contains?(hash,other)
+      other.all? do |key, value|
+        hash.include?(key) && hash[key] == value
+      end
+    end
+
+    def self.without(hash,keys)
+      hash.reject do |key, value|
+        keys.include?(key)
+      end
+    end
+
+    def self.stringify_keys(hash)
+      result = {}
+      hash.each do |key, value|
+        result[key.to_s] = value
+      end
+      result
+    end
+
+    def self.internalize_keys(hash)
+      result = {}
+      hash.each do |key, value|
+        result[key.intern] = value
+      end
+      result
+    end
+  end
+end

data/lib/storexplore/null_digger.rb

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+#
+# null_digger.rb
+#
+# Copyright (c) 2010, 2011, 2012, 2013 by Philippe Bourgau. All rights reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301  USA
+
+module Storexplore
+  class NullDigger
+    def sub_walkers(page, father)
+      []
+    end
+  end
+end
+
+