onebox 1.8.39 → 1.8.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6e4e0c8a8a62c9b418ddbf15ec75621fc48243a18cb105e50f0dd6d9847f1e7
-  data.tar.gz: 2e5b72ec7e9c108727a2dbeedca54a54e7b4a455ff768ea84d45b8452b4f867b
+  metadata.gz: 90d13b7abf1d64e58716b0468c0241ba8dc7e17dca617a4bc42f2de9fb4418d9
+  data.tar.gz: 8ffa07d4b9c4808a5b1d319de5e8167164966544a98eab71e329323fc0cdfd73
 SHA512:
-  metadata.gz: 152595e5f532e5ddf676d7ab67e6521de6dce24c5baf78be1f20e6322cbf364b73bcb8cac34c7fc191548f6fdbd24e2e94faeb38612bf4826ef01d4655b47c86
-  data.tar.gz: 1df2097ce7f36b7d5a59b1337a638e1e2d5fe4c6671fe86c51130fa6756bd9e00bdff0435d8ff273af57bd206c42ed31961518e6c7d43443496b9d78dfcd0ac4
+  metadata.gz: ff64a9341af4b04893fbcc87fae1b429e9202cc873cf4eb4d90d484c188bc0eb57bff809c633bfd57a4ba8304cfeae788f1d803a0114a9c79440372bd78e631a
+  data.tar.gz: 39e2c32467f56af30aa31819d208791c2353df5e9d63d3d6212f032d5333b485692a44f4cc70b6daae7a6779bc8faf1a2cf535b5ee9ac7cf87b31a13075c87f7

data/.rubocop.yml

@@ -1,5 +1,5 @@
 AllCops:
-  TargetRubyVersion: 2.2
+  TargetRubyVersion: 2.3
   DisabledByDefault: true
   Include:
     - '**/*.gemspec'

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 sudo: false
 rvm:
-  - 2.2
   - 2.3
   - 2.4
 notifications:

data/README.md

@@ -76,7 +76,7 @@ Ruby Support
 
 The onebox library is supported on all "officially" supported versions of Ruby.
 
-This means you must be on Ruby 2.2 or above for it to work.
+This means you must be on Ruby 2.3 or above for it to work.
 
 Development Preview Interface
 -----------------------------

data/lib/onebox/engine/amazon_onebox.rb

@@ -49,33 +49,114 @@ module Onebox
         if (landing_image = raw.css("#landingImage")) && landing_image.any?
           landing_image.first["src"].to_s
         end
+
+        if (ebook_image = raw.css("#ebooksImgBlkFront")) && ebook_image.any?
+          ::JSON.parse(ebook_image.first.attributes["data-a-dynamic-image"].value).keys.first
+        end
+      end
+
+      def price
+        # get item price (Amazon markup is inconsistent, deal with it)
+        if raw.css("#priceblock_ourprice .restOfPrice")[0] && raw.css("#priceblock_ourprice .restOfPrice")[0].inner_text
+          "#{raw.css("#priceblock_ourprice .restOfPrice")[0].inner_text}#{raw.css("#priceblock_ourprice .buyingPrice")[0].inner_text}.#{raw.css("#priceblock_ourprice .restOfPrice")[1].inner_text}"
+        elsif raw.css("#priceblock_dealprice") && (dealprice = raw.css("#priceblock_dealprice span")[0])
+          dealprice.inner_text
+        elsif !raw.css("#priceblock_ourprice").inner_text.empty?
+          raw.css("#priceblock_ourprice").inner_text
+        else
+          raw.css(".mediaMatrixListItem.a-active .a-color-price").inner_text
+        end
+      end
+
+      def multiple_authors(authors_xpath)
+        author_list = raw.xpath(authors_xpath)
+        authors = []
+        author_list.each { |a| authors << a.inner_text.strip }
+        authors.join(", ")
       end
 
       def data
         og = ::Onebox::Helpers.extract_opengraph(raw)
-        title = og[:title] || CGI.unescapeHTML(raw.css("title").inner_text)
 
-        result = {
-          link: link,
-          title: title,
-          image: og[:image] || image
-        }
+        if raw.at_css('#dp.book_mobile') #printed books
+          title = raw.at("h1#title")&.inner_text
+          authors = raw.at_css('#byline_secondary_view_div') ? multiple_authors("//div[@id='byline_secondary_view_div']//span[@class='a-text-bold']") : raw.at("#byline")&.inner_text
+          rating = raw.at("#averageCustomerReviews_feature_div .a-icon")&.inner_text || raw.at("#cmrsArcLink .a-icon")&.inner_text
 
-        result[:by_info] = raw.at("#by-line")
-        result[:by_info] = Onebox::Helpers.clean(result[:by_info].inner_html) if result[:by_info]
+          table_xpath = "//div[@id='productDetails_secondary_view_div']//table[@id='productDetails_techSpec_section_1']"
+          isbn = raw.xpath("#{table_xpath}//tr[8]//td").inner_text.strip
 
-        # get item price (Amazon markup is inconsistent, deal with it)
-        result[:price] =
-          if raw.css("#priceblock_ourprice .restOfPrice")[0] && raw.css("#priceblock_ourprice .restOfPrice")[0].inner_text
-            "#{raw.css("#priceblock_ourprice .restOfPrice")[0].inner_text}#{raw.css("#priceblock_ourprice .buyingPrice")[0].inner_text}.#{raw.css("#priceblock_ourprice .restOfPrice")[1].inner_text}"
-          elsif raw.css("#priceblock_dealprice") && (dealprice = raw.css("#priceblock_dealprice span")[0])
-            dealprice.inner_text
+          # if ISBN is misplaced or absent it's hard to find out which data is
+          # available and where to find it so just set it all to nil
+          if /^\d(\-?\d){12}$/.match(isbn)
+            publisher = raw.xpath("#{table_xpath}//tr[1]//td").inner_text.strip
+            published = raw.xpath("#{table_xpath}//tr[2]//td").inner_text.strip
+            book_length = raw.xpath("#{table_xpath}//tr[6]//td").inner_text.strip
           else
-            raw.css("#priceblock_ourprice").inner_text
+            isbn = publisher = published = book_length = nil
           end
 
-        summary = raw.at("#productDescription")
-        result[:description] = og[:description] || (summary && summary.inner_text)
+          result = {
+            link: link,
+            title: title,
+            by_info: authors,
+            image: og[:image] || image,
+            description: raw.at("#productDescription")&.inner_text,
+            rating: "#{rating}#{', ' if rating && (!isbn&.empty? || !price&.empty?)}",
+            price: price,
+            isbn_asin_text: "ISBN",
+            isbn_asin: isbn,
+            publisher: publisher,
+            published: "#{published}#{', ' if published && !price&.empty?}"
+          }
+
+        elsif raw.at_css('#dp.ebooks_mobile') # ebooks
+          title = raw.at("#ebooksTitle")&.inner_text
+          authors = raw.at_css('#a-popover-mobile-udp-contributor-popover-id') ? multiple_authors("//div[@id='a-popover-mobile-udp-contributor-popover-id']//span[contains(@class,'a-text-bold')]") : (raw.at("#byline")&.inner_text&.strip || raw.at("#bylineInfo")&.inner_text&.strip)
+          rating = raw.at("#averageCustomerReviews_feature_div .a-icon")&.inner_text || raw.at("#cmrsArcLink .a-icon")&.inner_text || raw.at("#acrCustomerReviewLink .a-icon")&.inner_text
+
+          table_xpath = "//div[@id='detailBullets_secondary_view_div']//ul"
+          asin = raw.xpath("#{table_xpath}//li[4]/span/span[2]").inner_text
+
+          # if ASIN is misplaced or absent it's hard to find out which data is
+          # available and where to find it so just set it all to nil
+          if /^[0-9A-Z]{10}$/.match(asin)
+            publisher = raw.xpath("#{table_xpath}//li[2]/span/span[2]").inner_text
+            published = raw.xpath("#{table_xpath}//li[1]/span/span[2]").inner_text
+          else
+            asin = publisher = published = nil
+          end
+
+          result = {
+            link: link,
+            title: title,
+            by_info: authors,
+            image: og[:image] || image,
+            description: raw.at("#productDescription")&.inner_text,
+            rating: "#{rating}#{', ' if rating && (!asin&.empty? || !price&.empty?)}",
+            price: price,
+            isbn_asin_text: "ASIN",
+            isbn_asin: asin,
+            publisher: publisher,
+            published: "#{published}#{', ' if published && !price&.empty?}"
+          }
+
+        else
+          title = og[:title] || CGI.unescapeHTML(raw.css("title").inner_text)
+          result = {
+            link: link,
+            title: title,
+            image: og[:image] || image,
+            price: price
+          }
+
+          result[:by_info] = raw.at("#by-line")
+          result[:by_info] = Onebox::Helpers.clean(result[:by_info].inner_html) if result[:by_info]
+
+          summary = raw.at("#productDescription")
+          result[:description] = og[:description] || (summary && summary.inner_text)
+        end
+
         result
       end
     end

data/lib/onebox/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Onebox
-  VERSION = "1.8.39"
+  VERSION = "1.8.40"
 end

data/onebox.gemspec

@@ -10,14 +10,14 @@ Gem::Specification.new do |spec|
   spec.version       = Onebox::VERSION
   spec.authors       = ['Joanna Zeta', 'Vyki Englert', 'Robin Ward']
   spec.email         = ['holla@jzeta.com', 'vyki.englert@gmail.com', 'robin.ward@gmail.com']
-  spec.description   = %q{A gem for turning URLs into previews.}
+  spec.description   = %q{A gem for generating embeddable HTML previews from URLs.}
   spec.summary       = spec.description
   spec.homepage      = 'https://github.com/discourse/onebox'
   spec.license       = 'MIT'
 
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  # specs are too heavy to include they have tons of big fixtures
+  # Git repo link exists so you can download from there
+  spec.files         = `git ls-files`.split($/).reject { |s| s =~ /^(spec|web)/ }
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'multi_json', '~> 1.11'
@@ -42,5 +42,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'haml', '~> 4.0'
   spec.add_development_dependency 'listen', '~> 2.10.0'
 
-  spec.required_ruby_version = '>=2.2.0'
+  spec.required_ruby_version = '>=2.3.0'
 end

data/templates/amazon.mustache

@@ -3,4 +3,10 @@
 <h3><a href='{{link}}' target='_blank'>{{title}}</a></h3>
 {{#by_info}}<b>{{by_info}}</b>{{/by_info}}
 <p>{{description}}</p>
-<p><strong>{{price}}</strong></p>
+<p>
+  {{#rating}}{{rating}}{{/rating}}
+  {{#isbn_asin}}{{isbn_asin_text}}: {{isbn_asin}}, {{/isbn_asin}}
+  {{#publisher}}{{publisher}}, {{/publisher}}
+  {{#published}}{{published}}{{/published}}
+  <strong>{{price}}</strong>
+</p>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onebox
 version: !ruby/object:Gem::Version
-  version: 1.8.39
+  version: 1.8.40
 platform: ruby
 authors:
 - Joanna Zeta
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-16 00:00:00.000000000 Z
+date: 2018-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -292,7 +292,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.10.0
-description: A gem for turning URLs into previews.
+description: A gem for generating embeddable HTML previews from URLs.
 email:
 - holla@jzeta.com
 - vyki.englert@gmail.com
@@ -383,88 +383,6 @@ files:
 - lib/onebox/web.rb
 - lib/onebox/web_helpers.rb
 - onebox.gemspec
-- spec/fixtures/amazon-og.response
-- spec/fixtures/amazon.response
-- spec/fixtures/cloudapp-gif.response
-- spec/fixtures/cloudapp-jpg.response
-- spec/fixtures/cloudapp-mp4.response
-- spec/fixtures/cloudapp-others.response
-- spec/fixtures/dailymail.response
-- spec/fixtures/discourse_topic.response
-- spec/fixtures/discourse_topic_reply.response
-- spec/fixtures/douban.response
-- spec/fixtures/etsy.response
-- spec/fixtures/etsy_mobile.response
-- spec/fixtures/gfycat.response
-- spec/fixtures/githubblob.response
-- spec/fixtures/githubcommit.response
-- spec/fixtures/githubgist.response
-- spec/fixtures/githubpullrequest.response
-- spec/fixtures/googleplayapp.response
-- spec/fixtures/image.response
-- spec/fixtures/meetup.response.html
-- spec/fixtures/meetup_oembed.response
-- spec/fixtures/pubmed-electronic-xml.response
-- spec/fixtures/pubmed-electronic.response
-- spec/fixtures/pubmed-xml.response
-- spec/fixtures/pubmed.response
-- spec/fixtures/slides.response
-- spec/fixtures/stackexchange-answer.response
-- spec/fixtures/stackexchange-question.response
-- spec/fixtures/twitterstatus.response
-- spec/fixtures/video.response
-- spec/fixtures/wechat-mp.response
-- spec/fixtures/wikimedia.response
-- spec/fixtures/wikipedia.response
-- spec/fixtures/wikipediaredirected.response
-- spec/fixtures/xkcd.response
-- spec/fixtures/youku-meta.response
-- spec/fixtures/youku.response
-- spec/fixtures/youtube-channel.response
-- spec/fixtures/youtube-json.response
-- spec/fixtures/youtube-list-json.response
-- spec/fixtures/youtube.response
-- spec/lib/onebox/engine/amazon_onebox_spec.rb
-- spec/lib/onebox/engine/audio_onebox_spec.rb
-- spec/lib/onebox/engine/cloudapp_onebox_spec.rb
-- spec/lib/onebox/engine/douban_onebox_spec.rb
-- spec/lib/onebox/engine/gfycat_onebox_spec.rb
-- spec/lib/onebox/engine/github_blob_onebox_spec.rb
-- spec/lib/onebox/engine/github_commit_onebox_spec.rb
-- spec/lib/onebox/engine/github_gist_onebox_spec.rb
-- spec/lib/onebox/engine/github_pullrequest_onebox_spec.rb
-- spec/lib/onebox/engine/google_docs_onebox_spec.rb
-- spec/lib/onebox/engine/google_maps_onebox_spec.rb
-- spec/lib/onebox/engine/google_play_app_onebox_spec.rb
-- spec/lib/onebox/engine/html_spec.rb
-- spec/lib/onebox/engine/image_onebox_spec.rb
-- spec/lib/onebox/engine/json_spec.rb
-- spec/lib/onebox/engine/pdf_onebox_spec.rb
-- spec/lib/onebox/engine/pubmed_onebox_spec.rb
-- spec/lib/onebox/engine/slides_onebox_spec.rb
-- spec/lib/onebox/engine/stack_exchange_onebox_spec.rb
-- spec/lib/onebox/engine/trello_onebox_spec.rb
-- spec/lib/onebox/engine/twitch_clips_onebox_spec.rb
-- spec/lib/onebox/engine/twitch_stream_onebox_spec.rb
-- spec/lib/onebox/engine/twitch_video_onebox_spec.rb
-- spec/lib/onebox/engine/twitter_status_onebox_spec.rb
-- spec/lib/onebox/engine/video_onebox_spec.rb
-- spec/lib/onebox/engine/wechat_mp_onebox_spec.rb
-- spec/lib/onebox/engine/whitelisted_generic_onebox_spec.rb
-- spec/lib/onebox/engine/wikimedia_onebox_spec.rb
-- spec/lib/onebox/engine/wikipedia_onebox_spec.rb
-- spec/lib/onebox/engine/xkcd_spec.rb
-- spec/lib/onebox/engine/youku_onebox_spec.rb
-- spec/lib/onebox/engine/youtube_onebox_spec.rb
-- spec/lib/onebox/engine_spec.rb
-- spec/lib/onebox/helpers_spec.rb
-- spec/lib/onebox/layout_spec.rb
-- spec/lib/onebox/matcher_spec.rb
-- spec/lib/onebox/preview_spec.rb
-- spec/lib/onebox/status_check_spec.rb
-- spec/lib/onebox_spec.rb
-- spec/spec_helper.rb
-- spec/support/html_spec_helper.rb
 - templates/_layout.mustache
 - templates/amazon.mustache
 - templates/douban.mustache
@@ -485,12 +403,6 @@ files:
 - templates/wikimedia.mustache
 - templates/wikipedia.mustache
 - templates/xkcd.mustache
-- web/assets/javascripts/jquery.js
-- web/assets/javascripts/main.js
-- web/assets/stylesheets/normalize.css
-- web/assets/stylesheets/style.css
-- web/views/index.haml
-- web/views/layout.haml
 homepage: https://github.com/discourse/onebox
 licenses:
 - MIT
@@ -503,7 +415,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -511,90 +423,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.4
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
-summary: A gem for turning URLs into previews.
-test_files:
-- spec/fixtures/amazon-og.response
-- spec/fixtures/amazon.response
-- spec/fixtures/cloudapp-gif.response
-- spec/fixtures/cloudapp-jpg.response
-- spec/fixtures/cloudapp-mp4.response
-- spec/fixtures/cloudapp-others.response
-- spec/fixtures/dailymail.response
-- spec/fixtures/discourse_topic.response
-- spec/fixtures/discourse_topic_reply.response
-- spec/fixtures/douban.response
-- spec/fixtures/etsy.response
-- spec/fixtures/etsy_mobile.response
-- spec/fixtures/gfycat.response
-- spec/fixtures/githubblob.response
-- spec/fixtures/githubcommit.response
-- spec/fixtures/githubgist.response
-- spec/fixtures/githubpullrequest.response
-- spec/fixtures/googleplayapp.response
-- spec/fixtures/image.response
-- spec/fixtures/meetup.response.html
-- spec/fixtures/meetup_oembed.response
-- spec/fixtures/pubmed-electronic-xml.response
-- spec/fixtures/pubmed-electronic.response
-- spec/fixtures/pubmed-xml.response
-- spec/fixtures/pubmed.response
-- spec/fixtures/slides.response
-- spec/fixtures/stackexchange-answer.response
-- spec/fixtures/stackexchange-question.response
-- spec/fixtures/twitterstatus.response
-- spec/fixtures/video.response
-- spec/fixtures/wechat-mp.response
-- spec/fixtures/wikimedia.response
-- spec/fixtures/wikipedia.response
-- spec/fixtures/wikipediaredirected.response
-- spec/fixtures/xkcd.response
-- spec/fixtures/youku-meta.response
-- spec/fixtures/youku.response
-- spec/fixtures/youtube-channel.response
-- spec/fixtures/youtube-json.response
-- spec/fixtures/youtube-list-json.response
-- spec/fixtures/youtube.response
-- spec/lib/onebox/engine/amazon_onebox_spec.rb
-- spec/lib/onebox/engine/audio_onebox_spec.rb
-- spec/lib/onebox/engine/cloudapp_onebox_spec.rb
-- spec/lib/onebox/engine/douban_onebox_spec.rb
-- spec/lib/onebox/engine/gfycat_onebox_spec.rb
-- spec/lib/onebox/engine/github_blob_onebox_spec.rb
-- spec/lib/onebox/engine/github_commit_onebox_spec.rb
-- spec/lib/onebox/engine/github_gist_onebox_spec.rb
-- spec/lib/onebox/engine/github_pullrequest_onebox_spec.rb
-- spec/lib/onebox/engine/google_docs_onebox_spec.rb
-- spec/lib/onebox/engine/google_maps_onebox_spec.rb
-- spec/lib/onebox/engine/google_play_app_onebox_spec.rb
-- spec/lib/onebox/engine/html_spec.rb
-- spec/lib/onebox/engine/image_onebox_spec.rb
-- spec/lib/onebox/engine/json_spec.rb
-- spec/lib/onebox/engine/pdf_onebox_spec.rb
-- spec/lib/onebox/engine/pubmed_onebox_spec.rb
-- spec/lib/onebox/engine/slides_onebox_spec.rb
-- spec/lib/onebox/engine/stack_exchange_onebox_spec.rb
-- spec/lib/onebox/engine/trello_onebox_spec.rb
-- spec/lib/onebox/engine/twitch_clips_onebox_spec.rb
-- spec/lib/onebox/engine/twitch_stream_onebox_spec.rb
-- spec/lib/onebox/engine/twitch_video_onebox_spec.rb
-- spec/lib/onebox/engine/twitter_status_onebox_spec.rb
-- spec/lib/onebox/engine/video_onebox_spec.rb
-- spec/lib/onebox/engine/wechat_mp_onebox_spec.rb
-- spec/lib/onebox/engine/whitelisted_generic_onebox_spec.rb
-- spec/lib/onebox/engine/wikimedia_onebox_spec.rb
-- spec/lib/onebox/engine/wikipedia_onebox_spec.rb
-- spec/lib/onebox/engine/xkcd_spec.rb
-- spec/lib/onebox/engine/youku_onebox_spec.rb
-- spec/lib/onebox/engine/youtube_onebox_spec.rb
-- spec/lib/onebox/engine_spec.rb
-- spec/lib/onebox/helpers_spec.rb
-- spec/lib/onebox/layout_spec.rb
-- spec/lib/onebox/matcher_spec.rb
-- spec/lib/onebox/preview_spec.rb
-- spec/lib/onebox/status_check_spec.rb
-- spec/lib/onebox_spec.rb
-- spec/spec_helper.rb
-- spec/support/html_spec_helper.rb
+summary: A gem for generating embeddable HTML previews from URLs.
+test_files: []