spidy 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1034253dcc3f68d566c3b67ff9ec5c6aeca4ec1b6a2ed66723bda8041154011
-  data.tar.gz: '08ef4a5426111b1824c5547465d0473507f68d9f0ea499bacddc4411395dd25a'
+  metadata.gz: 5ee6041bae6fd932f5b19a52890766192fd59e7ab2be070390e3a8ac16b511cf
+  data.tar.gz: 49d00e22394f0d83d3b49aa8926710ee23c2bffae1c14ce469f3227b338b9cad
 SHA512:
-  metadata.gz: '01745823727ff14e7b8a4fc97a0487fa32000ae7e09c0241a4bceeab5722df060162b275e99fe77991139788e41f7cb46d1f9d113c5cf93d96efc98855910af3'
-  data.tar.gz: ae0d7b3b6707b939f83e1b8e453c0ad87c60faec363017665ddb38baf77f763ed44994f3223208e92545eaa362b8ae28bcafb5b0c818b050c35aa221d87e00b7
+  metadata.gz: 82dd5bb591c47412c0648afe1a5ac7b971c8f1ccbf01708e3556d6edd2ca2a8d514ee3160cbf5cdc550c77d8e10b470eb6b70ff493f30d0b5458384cb18379a7
+  data.tar.gz: 798f5ef00e24d12d14058bc6ad87a54d3a9d67f9d091ee64f1450ceed1008bee027d4a396d2d8cf449bacb895220384b2ddbf141ccfbe7fb8b9e6a5290183d74

data/.rubocop.yml

@@ -13,6 +13,10 @@ Naming/UncommunicativeMethodParamName:
   AllowedNames:
     - as
 
+Metrics/AbcSize:
+  Max: 21
+  Exclude:
+
 Metrics/LineLength:
   Max: 120
 

data/README.md

@@ -1,8 +1,6 @@
 # Spidy
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/crawler`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+![logo](https://github.com/aileron-inc/spidy/raw/master/spidy.png)
 
 ## Installation
 
@@ -32,7 +30,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/crawler. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/aileron-inc/spidy. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 
@@ -40,4 +38,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Crawler project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/crawler/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Crawler project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/aileron-inc/spidy/blob/master/CODE_OF_CONDUCT.md).

data/exe/spidy

@@ -3,15 +3,21 @@
 
 require 'spidy'
 
-case ARGV[0]&.to_sym
-when :spider then Spidy.open(ARGV[1]).shell.spider(ARGV[2])
-when :scraper then Spidy.open(ARGV[1]).shell.scraper(ARGV[2])
-when :shell then Spidy.open(ARGV[1]).shell.function
-when :new then Spidy.open(ARGV[1]).shell.build
-when :console
+if ARGV[0]&.to_sym == :console
   if ARGV[1].blank?
     Spidy.console
   else
     Spidy.open(ARGV[1]).console
   end
+  return
+end
+
+shell = Spidy.open(ARGV[1]).shell
+
+case ARGV[0]&.to_sym
+when :shell then shell.function
+when :call then shell.call(ARGV[2])
+when :each then shell.each(ARGV[2])
+else
+  fail 'usage: spidy [call shell new console] [file]'
 end

data/lib/spidy.rb

@@ -17,12 +17,7 @@ module Spidy
   autoload :Definition
   autoload :DefinitionFile
   autoload :Binder
-  autoload :Spider
-  autoload :Looper
   autoload :Connector
-  autoload :Result
-
-  const_set(:Crawler, Module.new) unless const_defined?(:Crawler)
 
   def self.console
     require 'pry'
@@ -33,15 +28,7 @@ module Spidy
     ::Spidy::DefinitionFile.open(filepath)
   end
 
-  def self.define(name = nil, domain: nil, &block)
-    crawler_definition = Class.new(::Spidy::Definition, &block)
-    crawler_definition.domain = domain
-
-    if name
-      crawler_class_name = name.to_s.camelize
-      Crawler.class_eval { remove_const(crawler_class_name) } if Crawler.const_defined?(crawler_class_name)
-      Crawler.const_set(crawler_class_name, crawler_definition)
-    end
-    crawler_definition
+  def self.define(&block)
+    Class.new(::Spidy::Definition, &block)
   end
 end

data/lib/spidy/binder.rb

@@ -3,75 +3,14 @@
 #
 # Bind resource received from the connection to the result object
 #
-class Spidy::Binder
-  #
-  # binding multiple
-  #
-  class Multiple
-    def self.bind(connector:, binder:, query:, block:)
-      multiple_binding_class = self
-      connector.field.call(binder, query) do |elements|
-        multiple_binding_class.new(binder.class).instance_exec(elements, &block)
-      end
-    end
+module Spidy::Binder
+  extend ActiveSupport::Autoload
+  autoload :Json
+  autoload :Html
 
-    def initialize(binder)
-      @binder = binder
-    end
+  def self.get(name)
+    return unless name.is_a?(String) || name.is_a?(Symbol)
 
-    def field(name)
-      @binder.field_names << name
-      @binder.field_names.uniq!
-      @binder.result_class.define(name)
-      result = yield
-      @binder.define_method(name) { result }
-    end
-  end
-
-  class_attribute :field_names, default: []
-  attr_reader :resource
-
-  def initialize(resource)
-    @resource = resource
-    self.class.fields_call(self)
-  end
-
-  def result
-    definition = self
-    fetched_at = Time.current
-    result = self.class.result_class.new(fetched_at: fetched_at, fetched_on: fetched_at.beginning_of_day, **attributes)
-    result.define_singleton_method(:resource) { definition.resource }
-    result
-  end
-
-  def attributes_to_array
-    field_names.map { |field_name| send(field_name) }
-  end
-
-  def attributes
-    field_names.map { |field_name| [field_name, send(field_name)] }.to_h
-  end
-
-  def self.query(name, query = nil, &block)
-    define_method(name) do
-      connector.field.call(self, query, &block)
-    end
-  end
-
-  def self.field(name, query = nil, optional: false, &block)
-    field_names << name
-    field_names.uniq!
-    result_class.define(name, presence: !optional)
-    define_method(name) do
-      connector.field.call(self, query, &block)
-    end
-  end
-
-  def self.fields(query, &block)
-    @fields = { query: query, block: block }
-  end
-
-  def self.fields_call(binder)
-    Multiple.bind(connector: connector, binder: binder, query: @fields[:query], block: @fields[:block]) if @fields
+    const_get(name.to_s.classify)
   end
 end

data/lib/spidy/binder/html.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+#
+# Bind html and convert to object
+#
+module Spidy::Binder::Html
+  #
+  # Describe the definition to get the necessary elements from the resource object
+  #
+  class Resource
+    class_attribute :names, default: []
+    attr_reader :html
+
+    def initialize(html)
+      @html = html
+    end
+
+    def to_s
+      to_h.to_json
+    end
+
+    def to_h
+      names.map { |name| [name, send(name)] }.to_h
+    end
+
+    def self.let(name, query = nil, &block)
+      names << name
+      define_method(name) do
+        return html.at(query)&.text if block.nil?
+        return instance_exec(&block) if query.blank?
+
+        instance_exec(html.search(query), &block)
+      rescue NoMethodError => e
+        raise "#{html.uri} ##{name} => #{e.message}"
+      end
+    end
+  end
+
+  def self.call(html, define_block)
+    binder = Class.new(Resource) { instance_exec(&define_block) }
+    yield binder.new(html)
+  end
+end

data/lib/spidy/binder/json.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+#
+# Bind json and convert to object
+#
+module Spidy::Binder::Json
+  #
+  # Describe the definition to get the necessary elements from the resource object
+  #
+  class Resource
+    class_attribute :names, default: []
+    attr_reader :resource
+
+    def initialize(resource)
+      @resource = resource
+    end
+
+    def to_s
+      to_h.to_json
+    end
+
+    def to_h
+      names.map { |name| [name, send(name)] }.to_h
+    end
+
+    def self.let(name, *query, &block)
+      names << name
+      define_method(name) do
+        result = resource.dig(*query) if query.present?
+        return result if block.nil?
+
+        instance_exec(result, &block)
+      end
+    end
+  end
+
+  def self.call(resource, define_block)
+    binder = Class.new(Resource) { instance_exec(&define_block) }
+    yield binder.new(resource)
+  end
+end

data/lib/spidy/connector.rb

@@ -6,5 +6,11 @@
 module Spidy::Connector
   extend ActiveSupport::Autoload
   autoload :Html
-  autoload :Xml
+  autoload :Json
+
+  def self.get(name)
+    return unless name.is_a?(String) || name.is_a?(Symbol)
+
+    const_get(name.to_s.classify)
+  end
 end

data/lib/spidy/connector/html.rb

@@ -3,15 +3,7 @@
 #
 # Mechanize wrapper
 #
-class Spidy::Connector::Html
-  class_attribute :field, default: (lambda { |object, query, &block|
-    node = object.resource.search(query)
-    fail "Could not be located #{query}" if node.nil?
-    return node.first.text if block.nil?
-
-    object.instance_exec(node, &block)
-  })
-
+module Spidy::Connector::Html
   USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
@@ -21,22 +13,13 @@ class Spidy::Connector::Html
     'Safari/537.36'
   ].join(' ')
 
-  attr_reader :start_url
-  attr_reader :agent
-
-  def initialize(start_url: nil, encoding: nil)
-    @start_url = start_url
-    @agent = Mechanize.new
+  def self.call(url, encoding: nil, &yielder)
+    agent = Mechanize.new
     if encoding
-      @agent.default_encoding = encoding
-      @agent.force_default_encoding = true
+      agent.default_encoding = encoding
+      agent.force_default_encoding = true
     end
-    @agent.user_agent = USER_AGENT
-  end
-
-  def call(url = @start_url, &block)
-    fail 'URL is undefined' if url.blank?
-
-    agent.get(url, &block)
+    agent.user_agent = USER_AGENT
+    agent.get(url, &yielder)
   end
 end

data/lib/spidy/connector/json.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+#
+# OpenURI to JSON.parse
+#
+module Spidy::Connector::Json
+  def self.call(url, yielder)
+    OpenURI.open_uri(url) { |body| yielder.call(JSON.parse(body)) }
+  end
+end

data/lib/spidy/console.rb

@@ -5,7 +5,7 @@
 #
 class Spidy::Console
   attr_reader :definition_file
-  delegate :spiders, :scrapers, to: :definition_file
+  delegate :namespace, :spiders, to: :definition_file
 
   def initialize(definition_file = nil)
     @definition_file = definition_file
@@ -18,4 +18,12 @@ class Spidy::Console
   def reload!
     @definition_file&.eval_definition
   end
+
+  def call(name, url = nil, &block)
+    namespace[name].call(url, &block)
+  end
+
+  def each(name, url = nil, &block)
+    spiders[name].call(url, &block)
+  end
 end

data/lib/spidy/definition.rb

@@ -4,100 +4,28 @@
 # Class representing a website defined by DSL
 #
 class Spidy::Definition
-  class_attribute :domain
+  class_attribute :namespace, default: {}
   class_attribute :spiders, default: {}
-  class_attribute :scrapers, default: {}
-  class_attribute :output, default: ->(result) { STDOUT.puts(result.attributes.to_json) }
 
-  def output(&block)
-    self.output = block
-  end
-
-  # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
   class << self
-    def spider(name, start_url = nil, encoding: nil, as: :html, &block)
-      connector_class = Spidy::Connector.const_get(as.to_s.classify)
-      connector = connector_class.new(start_url: start_url, encoding: encoding)
-      spider = Spidy::Spider.new(&block)
-      spider_class = Class.new do
-        define_singleton_method(:connector) { connector }
-        define_singleton_method(:call) do |url = start_url, &spider_block|
-          connector.call(url) do |resource|
-            spider.call(resource, &spider_block)
+    def define(name, connector: nil, binder: nil, as: nil, &define_block)
+      connector = Spidy::Connector.get(as || connector) || connector
+      binder = Spidy::Binder.get(as || binder) || binder
+      namespace[name] = proc do |url, &yielder|
+        connection_yielder = lambda do |resource|
+          binder.call(resource, define_block) do |object|
+            yielder.call(object)
           end
         end
-      end
-      const_set("#{name}_spider".classify, spider_class)
-      spiders[name] = spider_class
-    end
-
-    def scraper(name, options, &block)
-      if options[:loop]
-        loop_scraper(name, options, &block)
-      else
-        normal_scraper(name, **options, &block)
+        connector.call(url, &connection_yielder)
       end
     end
 
-    private
-
-    def loop_scraper(name, options, &block)
-      options = { as: :html, start_url: nil, encoding: nil, loop: nil }.merge(options)
-      result_class = Class.new(Spidy::Result)
-
-      # connector
-      connector_class = Spidy::Connector.const_get(options[:as].to_s.classify)
-      connector = connector_class.new(encoding: options[:encoding])
-
-      namespace = Class.new do
-        binder = Class.new(Spidy::Binder) do
-          define_singleton_method(:connector) { connector }
-          define_singleton_method(:result_class) { result_class }
-          define_method(:connector) { connector }
-          instance_exec(&block)
-        end
-        define_singleton_method(:call) do |url = options[:start_url], &yielder|
-          connector.call(url) do |resource|
-            looper = Spidy::Looper.new(resource, binder, options[:loop])
-            looper.call(&yielder)
-          end
-        end
-      end
-      const_set("#{name}_scraper".classify, namespace)
-      scrapers[name] = namespace
-    end
-
-    def normal_scraper(name, encoding: nil, as: :html, &block)
-      # result
-      result_class = Class.new(Spidy::Result)
-
-      # connector
-      connector_class = Spidy::Connector.const_get(as.to_s.classify)
-      connector = connector_class.new(encoding: encoding)
-
-      # namespace
-      namespace = Class.new do
-        binder = Class.new(Spidy::Binder) do
-          define_singleton_method(:connector) { connector }
-          define_singleton_method(:result_class) { result_class }
-          define_method(:connector) { connector }
-          instance_exec(&block)
-        end
-        define_singleton_method(:bind) do |url|
-          connector.call(url) do |resource|
-            binder.new(resource)
-          end
-        end
-        define_singleton_method(:call) do |url, &output|
-          result = bind(url).result
-          fail "#{url}\n#{result.errors.full_messages}" if result.invalid?
-
-          output.call(result)
-        end
+    def spider(name, connector: nil, as: nil)
+      connector = Spidy::Connector.get(as || connector) || connector
+      spiders[name] = proc do |url, &yielder|
+        yield(yielder, connector, url)
       end
-      const_set("#{name}_scraper".classify, namespace)
-      scrapers[name] = namespace
     end
   end
-  # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 end

data/lib/spidy/definition_file.rb

@@ -6,7 +6,7 @@
 class Spidy::DefinitionFile
   attr_reader :path
   attr_reader :definition
-  delegate :spiders, :scrapers, :output, to: :definition
+  delegate :namespace, :spiders, to: :definition
 
   CSV = lambda do |result|
     ::CSV.generate do |csv|

data/lib/spidy/shell.rb

@@ -1,51 +1,25 @@
 # frozen_string_literal: true
 
+require 'pry'
+
 #
 # spidy shell interface
 #
 class Spidy::Shell
   attr_reader :definition_file
-  delegate :spiders, :scrapers, to: :definition_file
+  delegate :namespace, :spiders, to: :definition_file
 
   def initialize(definition_file)
     @definition_file = definition_file
   end
 
-  # rubocop:disable Lint/AssignmentInCondition, Style/RescueStandardError
-  def scraper(name)
-    command = scrapers[name.to_sym]
-    fail "undefined commmand[#{name}]" if command.nil?
-
-    while line = STDIN.gets
-      url = line.strip
-      begin
-        command.call(url, &definition_file.output)
-      rescue => e
-        STDERR.puts "ERROR #{url}: #{e}\n#{e.backtrace}"
-      end
-    end
-  end
-  # rubocop:enable Lint/AssignmentInCondition, Style/RescueStandardError
-
-  def spider(name)
-    command = spiders[name.to_sym]
-    if File.pipe?(STDIN)
-      STDIN.each_line do |line|
-        start_url = line.strip
-        command.call(start_url) { |url| puts url }
-      end
-    else
-      command.call { |url| puts url }
-    end
-  end
-
   def function
     print <<~SHELL
       function spider() {
         spidy spider #{definition_file.path} $1
       }
       function scraper() {
-        spidy scraper #{definition_file.path} $1
+        spidy call #{definition_file.path} $1
       }
     SHELL
   end
@@ -56,12 +30,14 @@ class Spidy::Shell
       f.write <<~RUBY
         # frozen_string_literal: true
 
-        Spidy.define(:#{name}) do
-          spider(:example, 'http://example.com') do |html, yielder|
-            #  yielder.call(url or resource)
+        Spidy.define do
+          spider(:example) do |yielder, connector|
+            # connector.call(url) do |resource|
+            #   yielder.call(url or resource)
+            # end
           end
 
-          scraper(:example) do
+          define(:example) do
           end
         end
       RUBY
@@ -76,4 +52,25 @@ class Spidy::Shell
     end
   end
   # rubocop:enable Metrics/MethodLength
+
+  def call(name)
+    exec(namespace[name&.to_sym] || namespace.values.first)
+  end
+
+  def each(name)
+    exec(spiders[name&.to_sym] || spiders.values.first)
+  end
+
+  private
+
+  def exec(command)
+    fail "undefined commmand[#{name}]" if command.nil?
+
+    yielder = proc { |result| STDOUT.puts(result.to_s) }
+    if FileTest.pipe?(STDIN)
+      STDIN.each { |line| command.call(line.strip, &yielder) }
+    else
+      command.call(&yielder)
+    end
+  end
 end

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-21 00:00:00.000000000 Z
+date: 2019-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -148,18 +148,20 @@ files:
 - exe/spidy
 - lib/spidy.rb
 - lib/spidy/binder.rb
+- lib/spidy/binder/html.rb
+- lib/spidy/binder/json.rb
 - lib/spidy/connector.rb
 - lib/spidy/connector/html.rb
+- lib/spidy/connector/json.rb
 - lib/spidy/connector/xml.rb
 - lib/spidy/console.rb
 - lib/spidy/definition.rb
 - lib/spidy/definition_file.rb
-- lib/spidy/looper.rb
-- lib/spidy/result.rb
 - lib/spidy/shell.rb
 - lib/spidy/spider.rb
 - lib/spidy/version.rb
 - spidy.gemspec
+- spidy.png
 homepage: https://github.com/aileron-inc/spidy
 licenses:
 - MIT

data/lib/spidy/looper.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-#
-# looper
-#
-class Spidy::Looper
-  def initialize(resource, binder, loop_block)
-    @resource = resource
-    @binder = binder
-    @loop_block = loop_block
-  end
-
-  def call
-    yielder = lambda do |element|
-      result = @binder.new(element).result
-      fail "#{element}\n\n#{result.errors.full_messages}" if result.invalid?
-
-      yield result
-    end
-    @loop_block.call(@resource, yielder)
-  end
-end

data/lib/spidy/result.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-#
-# Scrape results
-#
-class Spidy::Result
-  include ActiveModel::Model
-  include ActiveModel::Attributes
-
-  def self.define(name, presence: true)
-    case name
-    when /.*\?/
-      attribute name, :boolean
-      validates name, inclusion: { in: [true, false] } if presence
-    else
-      attribute name
-      validates name, presence: true, allow_blank: true if presence
-    end
-  end
-
-  attribute :fetched_at
-  attribute :fetched_on
-end