maxwell 0.4.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 099c077db029111f4ab30b731e75395c110b9a69
-  data.tar.gz: 8ad88ceda71a2735edb3125737c404af5e34c973
+  metadata.gz: 344d408e0e787f44fb1a559ea51d5a7810f67101
+  data.tar.gz: 6c9f7f9167bc6cb3f981be5ceda9335370172d8b
 SHA512:
-  metadata.gz: 69e93a319d297f2df1d14671737c129e9eb22aa901a4073b2f3130381ca672b92f49ddb65a213d8fef5d318be2159ee99dd2d3f4c7175ff04305015518bd713b
-  data.tar.gz: 701875498b09eb445e0c5898492cff0a22000525ee0c62211a7535c78cc3e7179efcdada2982b2ee5b4e55e6189b46a9bc7cc6aa2a9323f0968b53e5364358f6
+  metadata.gz: 12e76e792218e7116a1e3d55056831ec369919d6eaf4fb25b6ba262eb8f8ea329d2f870e755921b40887267d4576c43a4060f0dbce1b4ff75e0c2f7104e44ba7
+  data.tar.gz: 3e2ba946c4e3625c3a2149a778331e05bcc4f1a7f92708d3b46ae04cad34a0a4be7b05acde2274e62daca861dd83a047828e62aba57c0bd1620def294c668400

data/Gemfile

@@ -1,5 +1,3 @@
 source 'https://rubygems.org'
-
 gem "pry"
-
 gemspec

data/README.md

@@ -15,38 +15,61 @@ Or install it yourself as:
 ## Usage
 
 ```ruby
-class YahooScraper < Maxwell::Base
-  attr_accessor :title, :url, :address
+class WikipediaScraper < Maxwell::Base
+  attr_accessor :title, :image_urls # attributes which you want to get
 
-  javascript true
+  # You need to define 2 methods
+  # parser ... define how to parse attributes from html.
+  # handler ... define what to do with result which is come from parser.
 
-  concurrency 4
-
-  def parser html
-    @title   = html.title
-    @url     = html.css("td.sdhk jdj").text
-    @address = html.css("table tr.ddad").text
+  def parser(html) # html is Nokogiri::HTML::Document object
+    @title      = html.css('title').text # Ruby - Wikipedia
+    @image_urls = html.css('img').map { |img| img[:src] } # ["//upload.wikimedia.org/wikipedia/commons/thumb/8/80/Ruby_-_Winza%2C_Tanzania.jpg/240px-Ruby_-_Winza%2C_Tanzania.jpg", ...]
   end
 
-  def handler result
+  def handler(result) # result is Hash which contain parsed attributes
     p result
-    #=> { title: "...", url: "...", address: "..." }
   end
 end
 
-YahooScraper.execute ["https://www.yahoo.com/"]
+WikipediaScraper.execute urls: %w[https://en.wikipedia.org/wiki/Ruby]
+
+# output is
+# {
+#   :url => "https://en.wikipedia.org/wiki/Ruby",
+#   :title => "Ruby - Wikipedia",
+#   :image_urls => [
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/8/80/Ruby_-_Winza%2C_Tanzania.jpg/240px-Ruby_-_Winza%2C_Tanzania.jpg",
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/f/f0/Corundum.png/220px-Corundum.png",
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/b/b1/Ruby_transmittance.svg/220px-Ruby_transmittance.svg.png",
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/0/0d/Ruby_cristal.jpg/100px-Ruby_cristal.jpg",
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/f/f0/Ruby_gem.JPG/160px-Ruby_gem.JPG",
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/a/a3/Ruby_and_diamond_bracelet.jpg/160px-Ruby_and_diamond_bracelet.jpg",
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/c/c4/Cut_Ruby.jpg/158px-Cut_Ruby.jpg",
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/4/46/Artificial_ruby_hemisphere_under_a_normal_light.jpg/200px-Artificial_ruby_hemisphere_under_a_normal_light.jpg",
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/3/38/Artificial_ruby_hemisphere_under_a_monochromatic_light.jpg/200px-Artificial_ruby_hemisphere_under_a_monochromatic_light.jpg",
+#     "//upload.wikimedia.org/wikipedia/commons/thumb/1/12/NMNH-Rubies-CroppedRotated.png/220px-NMNH-Rubies-CroppedRotated.png",
+#     "//upload.wikimedia.org/wikipedia/en/thumb/4/4a/Commons-logo.svg/30px-Commons-logo.svg.png",
+#     "//en.wikipedia.org/wiki/Special:CentralAutoLogin/start?type=1x1",
+#     "/static/images/wikimedia-button.png",
+#     "/static/images/poweredby_mediawiki_88x31.png"
+#   ]
+# }
 ```
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. Run `bundle exec maxwell` to use the gem in this directory, ignoring other installed copies of this gem.
+`bin/setup` ... install dependencies.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+`rake test` ... run the tests.
 
-## Contributing
+`bin/console` ... interactive prompt that will allow you to experiment.
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/maxwell. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+`bundle exec rake install` ... install this gem onto your local machine.
+
+## Contributing
 
+Bug reports and pull requests are welcome on GitHub at https://github.com/gogotanaka/maxwell. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
 
 ## License
 

data/exe/maxwell

@@ -1,3 +1,2 @@
 #!/usr/bin/env ruby
-
 require "maxwell"

data/lib/maxwell.rb

@@ -10,18 +10,24 @@ module Maxwell
 
   class Base
     class << self
-      def execute(urls)
-        Parallel.
-          map_with_index(urls, in_threads: @concurrency || 1) do |url, id|
-            p "scraping: #{ id + 1 }"
+      def execute(args)
+        if !args[:urls].nil?
+          urls = args[:urls]
+          Parallel.
+            map_with_index(urls, in_threads: @concurrency || 1) do |url, id|
+              puts "\e[34m[#{id + 1}] scraping: #{ url }\e[0m"
 
-            scraper = self.new
-            html = Maxwell::Converter.call(url, @use_poltergeist)
+              scraper = self.new
+              html = Maxwell::Converter.call(url, @use_poltergeist)
 
-            scraper.parser html
-
-            scraper.handler ({ url: url }).merge(scraper.result)
-          end
+              scraper.parser html
+              ({ url: url }).merge(scraper.result).tap do |result_hash|
+                scraper.handler result_hash
+              end
+            end
+        else
+          raise 'You need pass an argument urls: or raw_htmls:'
+        end
       end
 
       def attr_accessor(*attrs)

data/lib/maxwell/version.rb

@@ -1,3 +1,3 @@
 module Maxwell
-  VERSION = "0.4.3"
+  VERSION = "1.0.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maxwell
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 1.0.0
 platform: ruby
 authors:
 - gogotanaka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-03-17 00:00:00.000000000 Z
+date: 2017-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri