botz 0.6.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8dc85ac7df0f64087255c2aa722ba3e6151baca615ce00d9ce3c7cebd022f231
-  data.tar.gz: 7f77369c8e744b050b2b08273b35f1e95e299d8f8adad3e7760e41f920a10af9
+  metadata.gz: c5d5f01d8ad6efa00d67e23c3f7252151685ce114f8519b8b93fdbe1ce05f42e
+  data.tar.gz: c11d70d862a38a7526e630d19e00f84564c894ca7c57ddb761712bdffe68afe6
 SHA512:
-  metadata.gz: 4d7f5be597513d01464b23d6053d037b014008735745cb267d9299581efd8f11b1c5109929490b80cfbfee850c88202a3f0d298341a192cfe1f96212c72d8eee
-  data.tar.gz: 2f885e634ed4fe24ebd642bd65cf7c7c059eb1ec31d7a4e842cec0663527ba1da8366f719ded80a9e43346de8aa98e6f23564cde7f8e01265684bd44fbf5d5b5
+  metadata.gz: 256330ef0de6b6106dacd0a9d87bc28ed0bf051a1e5f09be5e3d054709744541f60c5699fbbc8abd311942a2136a209044ca9e1a00937e1dcba3b7f5db117d0f
+  data.tar.gz: 33a770696b537a3ffc3a61486d4cf758f80829a31b3b791ee588ce9cd0d59aca1008de7722897cae3a5eafa9e2e9e9a12763dc9f46cfc6f0ac98127409924adf

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    botz (0.5.0)
+    botz (0.7.0)
       activemodel (~> 5.2)
       activesupport (~> 5.2)
       mechanize

data/lib/botz.rb

@@ -4,6 +4,8 @@ require 'botz/version'
 require 'active_support/all'
 require 'active_model'
 require 'mechanize'
+require 'csv'
+require 'open-uri'
 
 #
 # web bot dsl engine

data/lib/botz/binder.rb

@@ -4,16 +4,13 @@
 # Bind resource received from the connection to the result object
 #
 class Botz::Binder
-  class_attribute :field_names, default: []
-  attr_reader :resource
-
   #
   # binding multiple
   #
   class Multiple
     def self.bind(connector:, binder:, query:, block:)
       multiple_binding_class = self
-      connector.field(binder, query) do |elements|
+      connector.field.call(binder, query) do |elements|
         multiple_binding_class.new(binder.class).instance_exec(elements, &block)
       end
     end
@@ -31,32 +28,42 @@ class Botz::Binder
     end
   end
 
+  class_attribute :field_names, default: []
+  attr_reader :resource
+
   def initialize(resource)
     @resource = resource
     self.class.fields_call(self)
   end
 
   def result
-    new_result(field_names.map { |field_name| [field_name, send(field_name)] }.to_h)
+    definition = self
+    fetched_at = Time.current
+    result = self.class.result_class.new(fetched_at: fetched_at, fetched_on: fetched_at.beginning_of_day, **attributes)
+    result.define_singleton_method(:resource) { definition.resource }
+    result
   end
 
-  def new_result(values)
-    fetched_at = Time.current
-    self.class.result_class.new(fetched_at: fetched_at, fetched_on: fetched_at.beginning_of_day, **values)
+  def attributes_to_array
+    field_names.map { |field_name| send(field_name) }
+  end
+
+  def attributes
+    field_names.map { |field_name| [field_name, send(field_name)] }.to_h
   end
 
   def self.query(name, query = nil, &block)
     define_method(name) do
-      connector.field(self, query, &block)
+      connector.field.call(self, query, &block)
     end
   end
 
-  def self.field(name, query = nil, &block)
+  def self.field(name, query = nil, optional: false, &block)
     field_names << name
     field_names.uniq!
-    result_class.define(name)
+    result_class.define(name, presence: !optional)
     define_method(name) do
-      connector.field(self, query, &block)
+      connector.field.call(self, query, &block)
     end
   end
 

data/lib/botz/connector.rb

@@ -6,5 +6,5 @@
 module Botz::Connector
   extend ActiveSupport::Autoload
   autoload :Html
-  autoload :DirectHtml
+  autoload :Xml
 end

data/lib/botz/connector/html.rb

@@ -4,19 +4,13 @@
 # Mechanize wrapper
 #
 class Botz::Connector::Html
-  #
-  # field macro
-  #
-  module Field
-    def field(object, query, &block)
-      node = object.resource.search(query)
-      fail "Could not be located #{query}" if node.nil?
-      return node.first.text if block.nil?
+  class_attribute :field, default: (lambda { |object, query, &block|
+    node = object.resource.search(query)
+    fail "Could not be located #{query}" if node.nil?
+    return node.first.text if block.nil?
 
-      object.instance_exec(node, &block)
-    end
-  end
-  include Field
+    object.instance_exec(node, &block)
+  })
 
   USER_AGENT = [
     'Mozilla/5.0',

data/lib/botz/connector/xml.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+#
+# xml
+#
+class Botz::Connector::Xml
+  class_attribute :field, default: (lambda { |object, query, optional: false, &block|
+    return object.instance_exec(object.resource, &block) if query.nil?
+
+    node = object.resource.search(query)
+    return if optional && node.empty?
+
+    fail "Could not be located #{query}" if node.empty?
+    return node.first.text if block.nil?
+
+    object.instance_exec(node, &block)
+  })
+
+  def initialize(start_url: nil, encoding: nil)
+    @start_url = start_url
+    @encoding = encoding
+  end
+
+  def call(url = @start_url)
+    fail 'URL is undefined' if url.blank?
+
+    xml =
+      Nokogiri::XML(OpenURI.open_uri(url).read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''))
+    yield xml
+  end
+end

data/lib/botz/definition_file.rb

@@ -8,6 +8,12 @@ class Botz::DefinitionFile
   attr_reader :definition
   delegate :spiders, :scrapers, :output, to: :definition
 
+  CSV = lambda do |result|
+    ::CSV.generate do |csv|
+      csv << result.definition.attributes_to_array
+    end
+  end
+
   def self.open(filepath)
     object = new(filepath)
     object.eval_definition

data/lib/botz/result.rb

@@ -7,14 +7,14 @@ class Botz::Result
   include ActiveModel::Model
   include ActiveModel::Attributes
 
-  def self.define(name)
+  def self.define(name, presence: true)
     case name
     when /.*\?/
       attribute name, :boolean
-      validates name, inclusion: { in: [true, false] }
+      validates name, inclusion: { in: [true, false] } if presence
     else
       attribute name
-      validates name, presence: true, allow_blank: true
+      validates name, presence: true, allow_blank: true if presence
     end
   end
 

data/lib/botz/shell.rb

@@ -14,12 +14,14 @@ class Botz::Shell
   # rubocop:disable Lint/AssignmentInCondition, Style/RescueStandardError
   def scraper(name)
     command = scrapers[name.to_sym]
+    fail "undefined commmand[#{name}]" if command.nil?
+
     while line = STDIN.gets
       url = line.strip
       begin
         command.call(url, &definition_file.output)
       rescue => e
-        STDERR.puts "ERROR #{e}"
+        STDERR.puts "ERROR #{url}: #{e}\n#{e.backtrace}"
       end
     end
   end

data/lib/botz/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Botz
-  VERSION = '0.6.0'
+  VERSION = '0.7.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: botz
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.1
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-16 00:00:00.000000000 Z
+date: 2019-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -150,8 +150,8 @@ files:
 - lib/botz.rb
 - lib/botz/binder.rb
 - lib/botz/connector.rb
-- lib/botz/connector/direct_html.rb
 - lib/botz/connector/html.rb
+- lib/botz/connector/xml.rb
 - lib/botz/console.rb
 - lib/botz/definition.rb
 - lib/botz/definition_file.rb

data/lib/botz/connector/direct_html.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-#
-# Nokogiri wrapper
-#
-class Botz::Connector::DirectHtml
-  include ::Botz::Connector::Html::Field
-
-  def initialize(encoding: nil)
-    @encoding = encoding
-  end
-
-  def call(html)
-    yield Nokogiri::HTML.parse(html)
-  end
-end