botz 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8f0f5bd0c1efc41fefee5be1ad905144c6c2af0e20c7163ec1ddac75b59b03c
-  data.tar.gz: d1fd0bf8a10ae82a5ca51a81993143964ff20911573785a8cb5d1f3e8facbed5
+  metadata.gz: 9fc66e77b306d2024ecaf98a7de9cb68a4489a42d0ccd7435245b4d6a385161b
+  data.tar.gz: f42f5b708de3b5e52ea29947a5d08cc31bcb0dadd89dcaa69b71de9b5ad001a9
 SHA512:
-  metadata.gz: 582acc3d2f94b021943d517ab62c77d42f759d8c197c4728e2a49ce4c4127f105ce0cbcb3a822e5fb7aefe8d90e8d6234d682e855cff5209d1c12d8f78f31d8d
-  data.tar.gz: 43efe75756b292eb80b8acce890fd37f15845cc9868cca6c46946ab0008d947afe9abbbef23bc71256fb9c4c87ff79a809d81f8f6fcd178ee0add6117fa9290f
+  metadata.gz: 4e8c1e31edde612bcf993fd470fc4249bfd236ffd2e3640cca22f1098d22a5c7b0bfa9ade04f0263c94df5faa9901b4743dcc5cc7f343512c9b6e079711a78f4
+  data.tar.gz: 52e4dbb1fe4ac0f9b07654d012a51e3a17b86a8279fe99eb34bb445fbc9845589b8adce472a92f636609c938ef486a9b86f81f31a5831f7315afdabafca5a492

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    botz (0.1.0)
+    botz (0.2.0)
       activemodel (~> 5.2)
       activesupport (~> 5.2)
       mechanize
+      pry
 
 GEM
   remote: https://rubygems.org/

data/bin/console

@@ -10,9 +10,9 @@ require 'botz'
 # (If you use this, don't forget to add pry to your Gemfile!)
 require 'pry'
 def reload!
-  files = $LOADED_FEATURES.select { |feat| feat =~ %r{/gem_name/} }
-  files.each { |file| load file }
-  load 'lib/botz.rb'
+  ActiveSupport::Dependencies.clear
+  ActiveSupport::DescendantsTracker.clear
+  ActiveSupport::Reloader.reload!
 end
 Pry.start
 

data/botz.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
 
+  spec.add_runtime_dependency 'pry'
   spec.add_runtime_dependency 'activemodel', '~> 5.2'
   spec.add_runtime_dependency 'activesupport', '~> 5.2'
   spec.add_runtime_dependency 'mechanize'

data/exe/botz

@@ -2,9 +2,15 @@
 # frozen_string_literal: true
 
 require 'botz'
-shell = Botz.open(ARGV[0]).shell
-case ARGV[1]&.to_sym
-when :spider then shell.spider(ARGV[2])
-when :scraper then shell.scraper(ARGV[2])
-when :shell then shell.function
+
+case ARGV[0]&.to_sym
+when :spider then Botz.open(ARGV[1]).shell.spider(ARGV[2])
+when :scraper then Botz.open(ARGV[1]).shell.scraper(ARGV[2])
+when :shell then Botz.open(ARGV[1]).shell.function
+when :console
+  if ARGV[1].blank?
+    Botz.console
+  else
+    Botz.open(ARGV[1]).console
+  end
 end

data/lib/botz/console.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+#
+# botz console
+#
+class Botz::Console
+  attr_reader :definition_file
+  delegate :spiders, :scrapers, to: :definition_file
+
+  def initialize(definition_file = nil)
+    @definition_file = definition_file
+  end
+
+  def open(filepath)
+    @definition_file = Botz::DefinitionFile.open(filepath)
+  end
+
+  def reload!
+    @definition_file&.eval_definition
+  end
+
+  def scraper(name, url, &block)
+    scrapers[name.to_sym].call(url, &block)
+  end
+
+  def spider(name, url = nil, &block)
+    if url
+      spiders[name.to_sym].call(url, &block)
+    else
+      spiders[name.to_sym].call(&block)
+    end
+  end
+end

data/lib/botz/{macro.rb → definition.rb}

@@ -4,21 +4,20 @@
 # Class representing a website defined by DSL
 #
 class Botz::Definition
-  Output = ->(result) { STDOUT.puts(result.to_json) }
-
-  class_attribute :name
   class_attribute :domain
   class_attribute :spiders, default: {}
   class_attribute :scrapers, default: {}
 
+  Output = ->(result) { STDOUT.puts(result.to_json) }
+
+  def output(&block)
+    remove_const(:Output)
+    const_set(:Output, block)
+  end
+
   # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
   class << self
-    def output(&block)
-      remove_const(:Output)
-      const_set(:Output, block)
-    end
-
-    def scraper(name, as: :html, output: Output, &block)
+    def scraper(name, as: :html, &block)
       class_name = "#{name}_scraper".classify
       downloader = Botz::Downloader.new(as)
       binder_base = Botz.const_get "#{as}_scraper_macro".classify
@@ -29,10 +28,10 @@ class Botz::Definition
         define_singleton_method(:crawler_class) { crawler_class }
         define_singleton_method(:bind) do |url|
           downloader.call(url) do |resource|
-            binder.new(scraper_class, resource, output)
+            binder.new(scraper_class, resource)
           end
         end
-        define_singleton_method(:call) { |url| bind(url).save }
+        define_singleton_method(:call) { |url, &output| bind(url).call(&output) }
       end
       const_set(class_name, scraper_class)
       scrapers[name] = scraper_class

data/lib/botz/definition_file.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+#
+# botz interface binding
+#
+class Botz::DefinitionFile
+  attr_reader :path
+  attr_reader :definition
+  delegate :spiders, :scrapers, :Output, to: :definition
+
+  def self.open(filepath)
+    object = new(filepath)
+    object.eval_definition
+    object
+  end
+
+  # rubocop:disable Security/Eval
+  def eval_definition
+    @definition = eval(File.open(path).read)
+  end
+  # rubocop:enable Security/Eval
+
+  def shell
+    @shell ||= Botz::Shell.new(self)
+  end
+
+  def console
+    require 'pry'
+    Pry.start(Botz::Console.new(self))
+  end
+
+  private
+
+  def initialize(path)
+    @path = path
+  end
+end

data/lib/botz/html_scraper_macro.rb

@@ -19,9 +19,8 @@ class Botz::HtmlScraperMacro
   attr_reader :scraper_class
   attr_reader :url
   attr_reader :html
-  attr_reader :writer
 
-  def initialize(scraper_class, resource, writer)
+  def initialize(scraper_class, resource)
     @scraper_class = scraper_class
     @url = resource.uri
     @html = resource
@@ -45,10 +44,10 @@ class Botz::HtmlScraperMacro
     self.class.field_names.map { |field| [field, send(field)] }.to_h.merge(timestamps)
   end
 
-  def save
+  def call
     fail Error.new(scraper_class, url, errors) if invalid?
 
-    writer.call(to_h)
+    yield(to_h)
   end
 
   # rubocop:disable Metrics/AbcSize, Metrics/MethodLength

data/lib/botz/shell.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
 #
-# botz shell access interface
+# botz shell interface
 #
 class Botz::Shell
-  include ActiveModel::Model
-  include ActiveModel::Attributes
+  attr_reader :definition_file
+  delegate :spiders, :scrapers, to: :definition_file
 
-  attribute :filepath
-  attribute :definition
-  delegate :domain, :spiders, :scrapers, to: :definition
+  def initialize(definition_file)
+    @definition_file = definition_file
+  end
 
   # rubocop:disable Lint/AssignmentInCondition, Style/RescueStandardError
   def scraper(name)
@@ -17,7 +17,7 @@ class Botz::Shell
     while line = STDIN.gets
       url = line.strip
       begin
-        command.call(url)
+        command.call(url, &definition_file.Output)
       rescue
         STDERR.puts "ERROR #{command} #{url}"
       end
@@ -40,10 +40,10 @@ class Botz::Shell
   def function
     print <<~SHELL
       function spider() {
-        botz #{filepath} spider $1
+        botz spider #{definition_file.path} $1
       }
       function scraper() {
-        botz #{filepath} scraper $1
+        botz scraper #{definition_file.path} $1
       }
     SHELL
   end

data/lib/botz/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Botz
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/botz.rb

@@ -11,27 +11,31 @@ require 'mechanize'
 module Botz
   extend ActiveSupport::Autoload
   autoload :Shell
-  autoload :Command
+  autoload :Console
   autoload :Definition
+  autoload :DefinitionFile
   autoload :Spider
   autoload :Downloader
   autoload :HtmlScraperMacro
 
-  def self.open(filepath)
-    remove_const(:Crawler) if const_defined?(:Crawler)
-    const_set(:Crawler, Module.new)
+  const_set(:Crawler, Module.new) unless const_defined?(:Crawler)
 
-    # rubocop:disable Security/Eval
-    definition = eval(File.open(filepath).read)
-    # rubocop:enable Security/Eval
+  def self.console
+    require 'pry'
+    Pry.start(Botz::Console.new)
+  end
 
-    Botz::Command.new(filepath: filepath, definition: definition)
+  def self.open(filepath)
+    ::Botz::DefinitionFile.open(filepath)
   end
 
   def self.define(name, domain:, &block)
-    crawler_definition = Class.new(Botz::Definition, &block)
+    crawler_definition = Class.new(::Botz::Definition, &block)
     crawler_definition.domain = domain
-    Crawler.const_set(name.to_s.camelize, crawler_definition)
+    crawler_class_name = name.to_s.camelize
+
+    Crawler.class_eval { remove_const(crawler_class_name) } if Crawler.const_defined?(crawler_class_name)
+    Crawler.const_set(crawler_class_name, crawler_definition)
     crawler_definition
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: botz
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - aileron
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-28 00:00:00.000000000 Z
+date: 2019-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
@@ -134,10 +148,11 @@ files:
 - botz.gemspec
 - exe/botz
 - lib/botz.rb
-- lib/botz/command.rb
+- lib/botz/console.rb
+- lib/botz/definition.rb
+- lib/botz/definition_file.rb
 - lib/botz/downloader.rb
 - lib/botz/html_scraper_macro.rb
-- lib/botz/macro.rb
 - lib/botz/shell.rb
 - lib/botz/spider.rb
 - lib/botz/version.rb

data/lib/botz/command.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-class Botz::Command
-  attr_reader :definition
-  attr_reader :shell
-
-  def initialize(definition)
-    @definition = definition
-    @shell = Botz::Shell.new(definition: definition)
-  end
-end