botz 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe53c15b160a37521cde364f0a8dbcaf0daba0a7ef472dc8bbb560fc4666bef1
-  data.tar.gz: 51ca8fd5e7272be7bc89b86d510f7b3486c86c3dd7fd42d3c04287d960702010
+  metadata.gz: 8dc85ac7df0f64087255c2aa722ba3e6151baca615ce00d9ce3c7cebd022f231
+  data.tar.gz: 7f77369c8e744b050b2b08273b35f1e95e299d8f8adad3e7760e41f920a10af9
 SHA512:
-  metadata.gz: 10166b292f8c67d624a9c65c39471e02155235977c095b3423bbb7e4a83e45bfd345c8fad2161313678c7208b07c1e99d5ae76444178c4f55189aa5696c6d752
-  data.tar.gz: 23bd88361bf928ac2c52fec73094e07d4f3e535f195fd8acc29d94bd8c49f3e428c6a52f47e61d6f558d48ebc9be941755311a47c4e03c9686e1497f0a78a095
+  metadata.gz: 4d7f5be597513d01464b23d6053d037b014008735745cb267d9299581efd8f11b1c5109929490b80cfbfee850c88202a3f0d298341a192cfe1f96212c72d8eee
+  data.tar.gz: 2f885e634ed4fe24ebd642bd65cf7c7c059eb1ec31d7a4e842cec0663527ba1da8366f719ded80a9e43346de8aa98e6f23564cde7f8e01265684bd44fbf5d5b5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    botz (0.4.0)
+    botz (0.5.0)
       activemodel (~> 5.2)
       activesupport (~> 5.2)
       mechanize

data/lib/botz/binder.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+#
+# Bind resource received from the connection to the result object
+#
+class Botz::Binder
+  class_attribute :field_names, default: []
+  attr_reader :resource
+
+  #
+  # binding multiple
+  #
+  class Multiple
+    def self.bind(connector:, binder:, query:, block:)
+      multiple_binding_class = self
+      connector.field(binder, query) do |elements|
+        multiple_binding_class.new(binder.class).instance_exec(elements, &block)
+      end
+    end
+
+    def initialize(binder)
+      @binder = binder
+    end
+
+    def field(name)
+      @binder.field_names << name
+      @binder.field_names.uniq!
+      @binder.result_class.define(name)
+      result = yield
+      @binder.define_method(name) { result }
+    end
+  end
+
+  def initialize(resource)
+    @resource = resource
+    self.class.fields_call(self)
+  end
+
+  def result
+    new_result(field_names.map { |field_name| [field_name, send(field_name)] }.to_h)
+  end
+
+  def new_result(values)
+    fetched_at = Time.current
+    self.class.result_class.new(fetched_at: fetched_at, fetched_on: fetched_at.beginning_of_day, **values)
+  end
+
+  def self.query(name, query = nil, &block)
+    define_method(name) do
+      connector.field(self, query, &block)
+    end
+  end
+
+  def self.field(name, query = nil, &block)
+    field_names << name
+    field_names.uniq!
+    result_class.define(name)
+    define_method(name) do
+      connector.field(self, query, &block)
+    end
+  end
+
+  def self.fields(query, &block)
+    @fields = { query: query, block: block }
+  end
+
+  def self.fields_call(binder)
+    Multiple.bind(connector: connector, binder: binder, query: @fields[:query], block: @fields[:block]) if @fields
+  end
+end

data/lib/botz/{resource_accessor → connector}/direct_html.rb

@@ -3,7 +3,9 @@
 #
 # Nokogiri wrapper
 #
-class Botz::ResourceAccessor::DirectHtml
+class Botz::Connector::DirectHtml
+  include ::Botz::Connector::Html::Field
+
   def initialize(encoding: nil)
     @encoding = encoding
   end

data/lib/botz/{resource_accessor → connector}/html.rb

@@ -3,7 +3,21 @@
 #
 # Mechanize wrapper
 #
-class Botz::ResourceAccessor::Html
+class Botz::Connector::Html
+  #
+  # field macro
+  #
+  module Field
+    def field(object, query, &block)
+      node = object.resource.search(query)
+      fail "Could not be located #{query}" if node.nil?
+      return node.first.text if block.nil?
+
+      object.instance_exec(node, &block)
+    end
+  end
+  include Field
+
   USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',

data/lib/botz/{resource_accessor.rb → connector.rb}

@@ -3,7 +3,7 @@
 #
 # This class is responsible for actually making a network connection and downloading hypertext
 #
-module Botz::ResourceAccessor
+module Botz::Connector
   extend ActiveSupport::Autoload
   autoload :Html
   autoload :DirectHtml

data/lib/botz/definition.rb

@@ -7,7 +7,7 @@ class Botz::Definition
   class_attribute :domain
   class_attribute :spiders, default: {}
   class_attribute :scrapers, default: {}
-  class_attribute :output, default: ->(result) { STDOUT.puts(result.to_json) }
+  class_attribute :output, default: ->(result) { STDOUT.puts(result.attributes.to_json) }
 
   def output(&block)
     self.output = block
@@ -15,35 +15,14 @@ class Botz::Definition
 
   # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
   class << self
-    def scraper(name, encoding: nil, as: :html, &block)
-      class_name = "#{name}_scraper".classify
-      accessor_class = Botz::ResourceAccessor.const_get(as.to_s.classify)
-      accessor = accessor_class.new(encoding: encoding)
-      binder_base = Botz::Scraper.const_get(as.to_s.classify)
-      binder = Class.new(binder_base, &block)
-      binder.define_singleton_method(:name) { class_name }
-      crawler_class = self
-      scraper_class = Class.new do
-        define_singleton_method(:crawler_class) { crawler_class }
-        define_singleton_method(:bind) do |url|
-          accessor.call(url) do |resource|
-            binder.new(scraper_class, resource)
-          end
-        end
-        define_singleton_method(:call) { |url, &output| bind(url).call(&output) }
-      end
-      const_set(class_name, scraper_class)
-      scrapers[name] = scraper_class
-    end
-
     def spider(name, start_url = nil, encoding: nil, as: :html, &block)
-      accessor_class = Botz::ResourceAccessor.const_get(as.to_s.classify)
-      accessor = accessor_class.new(start_url: start_url, encoding: encoding)
+      connector_class = Botz::Connector.const_get(as.to_s.classify)
+      connector = connector_class.new(start_url: start_url, encoding: encoding)
       spider = Botz::Spider.new(&block)
       spider_class = Class.new do
-        define_singleton_method(:accessor) { accessor }
+        define_singleton_method(:connector) { connector }
         define_singleton_method(:call) do |url = start_url, &spider_block|
-          accessor.call(url) do |resource|
+          connector.call(url) do |resource|
             spider.call(resource, &spider_block)
           end
         end
@@ -51,6 +30,74 @@ class Botz::Definition
       const_set("#{name}_spider".classify, spider_class)
       spiders[name] = spider_class
     end
+
+    def scraper(name, options, &block)
+      if options[:loop]
+        loop_scraper(name, options, &block)
+      else
+        normal_scraper(name, **options, &block)
+      end
+    end
+
+    private
+
+    def loop_scraper(name, options, &block)
+      options = { as: :html, start_url: nil, encoding: nil, loop: nil }.merge(options)
+      result_class = Class.new(Botz::Result)
+
+      # connector
+      connector_class = Botz::Connector.const_get(options[:as].to_s.classify)
+      connector = connector_class.new(encoding: options[:encoding])
+
+      namespace = Class.new do
+        binder = Class.new(Botz::Binder) do
+          define_singleton_method(:connector) { connector }
+          define_singleton_method(:result_class) { result_class }
+          define_method(:connector) { connector }
+          instance_exec(&block)
+        end
+        define_singleton_method(:call) do |url = options[:start_url], &yielder|
+          connector.call(url) do |resource|
+            looper = Botz::Looper.new(resource, binder, options[:loop])
+            looper.call(&yielder)
+          end
+        end
+      end
+      const_set("#{name}_scraper".classify, namespace)
+      scrapers[name] = namespace
+    end
+
+    def normal_scraper(name, encoding: nil, as: :html, &block)
+      # result
+      result_class = Class.new(Botz::Result)
+
+      # connector
+      connector_class = Botz::Connector.const_get(as.to_s.classify)
+      connector = connector_class.new(encoding: encoding)
+
+      # namespace
+      namespace = Class.new do
+        binder = Class.new(Botz::Binder) do
+          define_singleton_method(:connector) { connector }
+          define_singleton_method(:result_class) { result_class }
+          define_method(:connector) { connector }
+          instance_exec(&block)
+        end
+        define_singleton_method(:bind) do |url|
+          connector.call(url) do |resource|
+            binder.new(resource)
+          end
+        end
+        define_singleton_method(:call) do |url, &output|
+          result = bind(url).result
+          fail "#{url}\n#{result.errors.full_messages}" if result.invalid?
+
+          output.call(result)
+        end
+      end
+      const_set("#{name}_scraper".classify, namespace)
+      scrapers[name] = namespace
+    end
   end
   # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 end

data/lib/botz/looper.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+#
+# looper
+#
+class Botz::Looper
+  def initialize(resource, binder, loop_block)
+    @resource = resource
+    @binder = binder
+    @loop_block = loop_block
+  end
+
+  def call
+    yielder = lambda do |element|
+      result = @binder.new(element).result
+      fail "#{element}\n\n#{result.errors.full_messages}" if result.invalid?
+
+      yield result
+    end
+    @loop_block.call(@resource, yielder)
+  end
+end

data/lib/botz/result.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+#
+# Scrape results
+#
+class Botz::Result
+  include ActiveModel::Model
+  include ActiveModel::Attributes
+
+  def self.define(name)
+    case name
+    when /.*\?/
+      attribute name, :boolean
+      validates name, inclusion: { in: [true, false] }
+    else
+      attribute name
+      validates name, presence: true, allow_blank: true
+    end
+  end
+
+  attribute :fetched_at
+  attribute :fetched_on
+end

data/lib/botz/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Botz
-  VERSION = '0.4.0'
+  VERSION = '0.6.0'
 end

data/lib/botz.rb

@@ -14,9 +14,11 @@ module Botz
   autoload :Console
   autoload :Definition
   autoload :DefinitionFile
+  autoload :Binder
   autoload :Spider
-  autoload :Scraper
-  autoload :ResourceAccessor
+  autoload :Looper
+  autoload :Connector
+  autoload :Result
 
   const_set(:Crawler, Module.new) unless const_defined?(:Crawler)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: botz
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-07 00:00:00.000000000 Z
+date: 2019-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -148,15 +148,15 @@ files:
 - botz.gemspec
 - exe/botz
 - lib/botz.rb
+- lib/botz/binder.rb
+- lib/botz/connector.rb
+- lib/botz/connector/direct_html.rb
+- lib/botz/connector/html.rb
 - lib/botz/console.rb
 - lib/botz/definition.rb
 - lib/botz/definition_file.rb
-- lib/botz/resource_accessor.rb
-- lib/botz/resource_accessor/direct_html.rb
-- lib/botz/resource_accessor/html.rb
-- lib/botz/scraper.rb
-- lib/botz/scraper/direct_html.rb
-- lib/botz/scraper/html.rb
+- lib/botz/looper.rb
+- lib/botz/result.rb
 - lib/botz/shell.rb
 - lib/botz/spider.rb
 - lib/botz/version.rb

data/lib/botz/scraper/direct_html.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-#
-# direct resource to html scraping
-#
-class Botz::Scraper::DirectHtml
-  include ActiveModel::Model
-  include ActiveModel::Attributes
-
-  #
-  # Scraper error class
-  #
-  class Error < StandardError
-    def initialize(scraper_class, errors)
-      super("#{scraper_class} # #{errors.full_messages}")
-    end
-  end
-
-  attr_reader :scraper_class
-  attr_reader :html
-
-  def initialize(scraper_class, resource)
-    @scraper_class = scraper_class
-    @html = resource
-  end
-
-  class << self
-    def field_names
-      @field_names ||= []
-    end
-  end
-
-  def to_h
-    fetched_at = Time.current
-    fetched_on = fetched_at.beginning_of_day
-    timestamps = { fetched_on: fetched_on, fetched_at: fetched_at }
-    self.class.field_names.map { |field| [field, send(field)] }.to_h.merge(timestamps)
-  end
-
-  def call
-    fail Error.new(scraper_class, errors) if invalid?
-
-    yield(to_h)
-  end
-
-  # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-  def self.field(name, path = nil, persist: true, &block)
-    if persist
-      field_names << name
-      case name
-      when /.*\?/
-        validates name, inclusion: { in: [true, false] }
-      else
-        validates name, presence: true, allow_blank: true
-      end
-    end
-
-    return define_method(name) { instance_exec(html, &block) } if path.nil?
-    return define_method(name) { html.search(path).text.strip } if block.nil?
-
-    define_method(name) { html.search(path).first.try { |e| instance_exec(e, &block) } }
-  end
-  # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
-end

data/lib/botz/scraper/html.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-#
-# DSL for parsing html into objects
-#
-class Botz::Scraper::Html
-  include ActiveModel::Model
-  include ActiveModel::Attributes
-
-  #
-  # Scraper error class
-  #
-  class Error < StandardError
-    def initialize(scraper_class, url, errors)
-      super("#{scraper_class}(#{url}) # #{errors.full_messages}")
-    end
-  end
-
-  attr_reader :scraper_class
-  attr_reader :url
-  attr_reader :html
-
-  def initialize(scraper_class, resource)
-    @scraper_class = scraper_class
-    @url = resource.uri
-    @html = resource
-  end
-
-  class << self
-    def field_names
-      @field_names ||= []
-    end
-  end
-
-  def primary_key
-    url.to_s
-  end
-
-  def to_h
-    fetched_at = Time.current
-    fetched_on = fetched_at.beginning_of_day
-    timestamps = { fetched_on: fetched_on, fetched_at: fetched_at }
-    self.class.field_names.map { |field| [field, send(field)] }.to_h.merge(timestamps)
-  end
-
-  def call
-    fail Error.new(scraper_class, url, errors) if invalid?
-
-    yield(to_h)
-  end
-
-  # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-  def self.field(name, path = nil, persist: true, &block)
-    if persist
-      field_names << name
-      case name
-      when /.*\?/
-        validates name, inclusion: { in: [true, false] }
-      else
-        validates name, presence: true, allow_blank: true
-      end
-    end
-
-    return define_method(name) { instance_exec(html, &block) } if path.nil?
-    return define_method(name) { html.search(path).text.strip } if block.nil?
-
-    define_method(name) { html.search(path).first.try { |e| instance_exec(e, &block) } }
-  end
-  # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
-end

data/lib/botz/scraper.rb

@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-#
-# scraper namespace
-#
-module Botz::Scraper
-  extend ActiveSupport::Autoload
-  autoload :Html
-  autoload :DirectHtml
-end