spidy 0.0.24 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11b932ab32a92fe7cca2a4c7fb91204098204d2b60d3226137fc41c783f50997
-  data.tar.gz: 34fcddd2519415792d0222bf0c8176dd06a734cc48db3025ccbadd6d10a80582
+  metadata.gz: 2418b53cec63dbd0cebe2dd7b0d10e9be1ea1177cd4be3584d3f04e2adbf6c10
+  data.tar.gz: bc8b78146e0f1f08b9b6fb052093ba92ca03aecf546a8ca9b079eab5991b8acf
 SHA512:
-  metadata.gz: 64fe7aa6a460a4c13d90749827f85904c1cf8ca14391cc2e12f22e2ec4b5b8a3fd7776ad282e178cadddaa6db197a83f49b5870b27aaa8356b79d20565992383
-  data.tar.gz: a8405babf239188f933a13b5213bbc67efe5610a7e7f68bfa49c646a755ecebba049cdab4dbe6b52b085c71f6a0ffd82609ea5aadf9ef355bd8bf6200bdfe5f0
+  metadata.gz: fb04e182f956b07c7bfc2290dabacdca588a940ba7aacfea1b417be94e7332bb5d9793e698150455b7fd484c2e745015c3cd4efe66f697f92710275d322376f1
+  data.tar.gz: 5cef25c2ca15ace3137cedb5181da48a95c67390304a4ab28c92f57de05801c4492296578e275cbcea51a597fa501a0ef661c20f4c0e67075dc492be57e558fa

data/.ruby-version

@@ -1 +1 @@
-2.6.2
+2.7.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.0.21)
+    spidy (0.0.24)
       activesupport
       mechanize
       pry
@@ -21,6 +21,7 @@ GEM
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
+    ffaker (2.10.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     i18n (1.7.0)
@@ -77,6 +78,7 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 2.0)
+  ffaker
   pry
   rake (~> 10.0)
   rspec (~> 3.0)

data/lib/spidy/binder/html.rb

@@ -5,11 +5,11 @@
 #
 class Spidy::Binder::Html
   class << self
-    attr_reader :names
+    attr_reader :attribute_names
 
-    @names = []
     def let(name, query = nil, &block)
-      @names << name
+      @attribute_names ||= []
+      @attribute_names << name
       define_method(name) do
         return html.at(query)&.text if block.nil?
         return instance_exec(&block) if query.blank?
@@ -21,12 +21,11 @@ class Spidy::Binder::Html
     end
   end
 
-  attr_reader :html, :source, :url
+  attr_reader :html, :url
 
   def initialize(html, url: nil)
-    @html = html
     @url = url
-    @source = html.body
+    @html = html
   end
 
   def to_s
@@ -34,6 +33,6 @@ class Spidy::Binder::Html
   end
 
   def to_h
-    self.class.names.map { |name| [name, send(name)] }.to_h
+    self.class.attribute_names.map { |name| [name, send(name)] }.to_h
   end
 end

data/lib/spidy/binder/json.rb

@@ -20,12 +20,11 @@ class Spidy::Binder::Json
     end
   end
 
-  attr_reader :json, :source, :url
+  attr_reader :json, :url
 
   def initialize(json, url: nil)
     @json = json
     @url = url
-    @source = json.to_json
   end
 
   def to_s

data/lib/spidy/binder/xml.rb

@@ -21,12 +21,11 @@ class Spidy::Binder::Xml
     end
   end
 
-  attr_reader :xml, :source, :url
+  attr_reader :xml, :url
 
   def initialize(xml, url: nil)
     @xml = xml
     @url = url
-    @source = xml.to_s
   end
 
   def to_s

data/lib/spidy/connector.rb

@@ -5,10 +5,20 @@
 #
 module Spidy::Connector
   extend ActiveSupport::Autoload
+  autoload :Direct
   autoload :Html
   autoload :Json
   autoload :Xml
 
+  USER_AGENT = [
+    'Mozilla/5.0',
+    '(Macintosh; Intel Mac OS X 10_12_6)',
+    'AppleWebKit/537.36',
+    '(KHTML, like Gecko)',
+    'Chrome/64.0.3282.186',
+    'Safari/537.36'
+  ].join(' ')
+
   def self.get(value)
     return const_get(value.to_s.classify) if value.is_a?(String) || value.is_a?(Symbol)
 

data/lib/spidy/connector/direct.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+#
+# Direct resource ( not network resource  )
+#
+module Spidy::Connector::Direct
+  def self.call(resource, &yielder)
+    yielder.call(resource)
+  end
+end

data/lib/spidy/connector/html.rb

@@ -18,18 +18,10 @@ module Spidy::Connector::Html
       @response_code = error.try(:response_code) || page.try(:response_code)
     end
   end
-  USER_AGENT = [
-    'Mozilla/5.0',
-    '(Macintosh; Intel Mac OS X 10_12_6)',
-    'AppleWebKit/537.36',
-    '(KHTML, like Gecko)',
-    'Chrome/64.0.3282.186',
-    'Safari/537.36'
-  ].join(' ')
 
-  @logger = proc { |values| STDERR.puts(values.map { |k, v| "#{k}:#{v}" }.join("\t")) }
+  @logger = proc { |values| STDERR.puts(values.to_json) }
   @agent = Mechanize.new
-  @agent.user_agent = USER_AGENT
+  @agent.user_agent = Spidy::Connector::USER_AGENT
 
   class << self
     attr_reader :agent
@@ -56,7 +48,7 @@ module Spidy::Connector::Html
                   'retry.rest_count': retry_count)
 
       @agent = Mechanize.new
-      @agent.user_agent = USER_AGENT
+      @agent.user_agent = Spidy::Connector::USER_AGENT
 
       retry_count -= 1
       if retry_count.positive?

data/lib/spidy/connector/json.rb

@@ -4,8 +4,8 @@
 # OpenURI to JSON.parse
 #
 module Spidy::Connector::Json
-  def self.call(url, &yielder)
+  def self.call(url)
     fail 'url is not specified' if url.blank?
-    OpenURI.open_uri(url) { |body| yielder.call(JSON.parse(body.read, symbolize_names: true)) }
+    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) { |body| yield JSON.parse(body.read, symbolize_names: true) }
   end
 end

data/lib/spidy/connector/xml.rb

@@ -7,7 +7,8 @@ module Spidy::Connector::Xml
   def self.call(url)
     fail 'URL is undefined' if url.blank?
 
-    xml = OpenURI.open_uri(url).read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, '')
-    yield Nokogiri::XML(xml, url)
+    OpenURI.open_uri(url, "User-Agent" => Spidy::Connector::USER_AGENT) do |body|
+      yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
+    end
   end
 end

data/lib/spidy/console.rb

@@ -19,11 +19,11 @@ class Spidy::Console
     @definition_file&.eval_definition
   end
 
-  def call(name = nil, url: nil, &block)
-    spidy.call(name, url: url, &block)
+  def call(name = :default, url: nil, &block)
+    spidy.call(name: name, url: url, &block)
   end
 
-  def each(name = nil, url: nil, &block)
-    spidy.each(name, url: url, &block)
+  def each(name = :default, url: nil, &block)
+    spidy.each(name: name, url: url, &block)
   end
 end

data/lib/spidy/definition.rb

@@ -8,27 +8,27 @@ module Spidy::Definition
     @namespace
   end
 
-  def call(name = :default, url: nil, stream: nil, err: nil, &output)
+  def call(source = nil, name: :default, &yielder)
     name = name.presence || :default
     spidy = @namespace[:"#{name}_scraper"]
     fail "undefined spidy [#{name}]" if spidy.nil?
 
-    exec(spidy, url: url, stream: stream, err: err, &output)
+    spidy.call(source, &yielder)
   end
 
-  def each(name = :default, url: nil, stream: nil, err: nil, &output)
+  def each(source = nil, name: :default, &yielder)
     name = name.presence || :default
     spidy = @namespace[:"#{name}_spider"]
     fail "undefined spidy [#{name}]" if spidy.nil?
 
-    exec(spidy, url: url, stream: stream, err: err, &output)
+    spidy.call(source, &yielder)
   end
 
-  def spider(name = :default, connector: nil, as: nil)
+  def spider(name = :default, connector: nil, as: nil, &define_block)
     @namespace ||= {}
-    connector = Spidy::Connector.get(as || connector) || connector
-    @namespace[:"#{name}_spider"] = proc do |url, &yielder|
-      yield(yielder, connector, url)
+    connector = Spidy::Connector.get(connector || as)
+    @namespace[:"#{name}_spider"] = proc do |source, &yielder|
+      define_block.call(yielder, connector, source)
     end
   end
 
@@ -41,26 +41,14 @@ module Spidy::Definition
 
   private
 
-  def exec(spidy, url: nil, stream: nil, err: nil, &output)
-    return spidy.call(url, &output) if stream.nil?
-
-    stream.each do |value|
-      spidy.call(value.strip, &output)
-    rescue StandardError => e
-      raise e if err.nil?
-
-      err.call(e, value.strip)
-    end
-  end
-
   def define_proc(connector, binder, define_block)
-    proc do |url, &yielder|
+    proc do |source, &yielder|
       fail 'block is not specified' if yielder.nil?
 
-      connection_yielder = lambda do |resource|
-        binder.call(resource, url: url, define: define_block) { |object| yielder.call(object) }
+      connection_yielder = lambda do |page|
+        binder.call(page, url: source, define: define_block) { |object| yielder.call(object) }
       end
-      connector.call(url, &connection_yielder)
+      connector.call(source, &connection_yielder)
     end
   end
 end

data/lib/spidy/shell.rb

@@ -8,23 +8,45 @@ require 'pry'
 class Spidy::Shell
   attr_reader :definition_file, :spidy
   class_attribute :output, default: (proc { |result| STDOUT.puts(result.to_s) })
-  class_attribute :error_handler, default: (proc { |e, url| STDERR.puts("#{url}\n #{e.message}") })
+  class_attribute :error_handler, default: (proc { |e, url| STDERR.puts({ url: url, message: e.message, backtrace: e.backtrace }.to_json) })
   delegate :spidy, to: :definition_file
 
   def initialize(definition_file)
     @definition_file = definition_file
   end
 
-  def call(name)
-    return spidy.call(name, stream: STDIN, err: error_handler, &output) if FileTest.pipe?(STDIN)
+  def each_stdin_lines(name)
+    STDIN.each_line do |url|
+      begin
+        spidy.each(url.strip, name: name, &output)
+      rescue => e
+        error_handler.call(e, url)
+      end
+    end
+  end
+
+  def call_stdin_lines(name)
+    STDIN.each_line do |url|
+      begin
+        spidy.call(url.strip, name: name, &output)
+      rescue => e
+        error_handler.call(e, url)
+      end
+    end
+  end
 
-    spidy.call(name, err: error_handler, &output)
+  def call(name)
+    return call_stdin_lines(name) if FileTest.pipe?(STDIN)
+    spidy.call(name: name, &output) unless FileTest.pipe?(STDIN)
+  rescue => e
+    error_handler.call(e, nil)
   end
 
   def each(name)
-    return spidy.each(name, stream: STDIN, err: error_handler, &output) if FileTest.pipe?(STDIN)
-
-    spidy.each(name, err: error_handler, &output)
+    return each_stdin_lines(name) if FileTest.pipe?(STDIN)
+    spidy.each(name: name, &output)
+  rescue => e
+    error_handler.call(e, nil)
   end
 
   def function

data/lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.0.24'
+  VERSION = '0.1.0'
 end

data/spidy.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'ffaker'
 
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'mechanize'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.0.24
+  version: 0.1.0
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-26 00:00:00.000000000 Z
+date: 2020-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: ffaker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -138,6 +152,7 @@ files:
 - lib/spidy/binder/json.rb
 - lib/spidy/binder/xml.rb
 - lib/spidy/connector.rb
+- lib/spidy/connector/direct.rb
 - lib/spidy/connector/html.rb
 - lib/spidy/connector/json.rb
 - lib/spidy/connector/xml.rb
@@ -168,7 +183,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: web spider dsl