pageflow-chart 2.1.0 → 2.4.0

data/app/jobs/pageflow/chart/scrape_site_job.rb

@@ -8,7 +8,8 @@ module Pageflow
       def perform_with_result(scraped_site,
                               _options = {},
                               downloader: downloader_for(scraped_site))
-        downloader.load_following_refresh_tags(scraped_site.url) do |file|
+        downloader.load_following_refresh_tags(scraped_site.url,
+                                               raise_on_http_error: true) do |file|
           scraper = Scraper.new(file.read, Chart.config.scraper_options)
           scraped_site.html_file = StringIOWithContentType.new(
             scraper.html,
@@ -16,11 +17,18 @@ module Pageflow
             content_type: 'text/html'
           )
 
-          downloader.load_all(scraper.javascript_urls,
+          downloader.load_all(scraper.javascript_urls_in_head,
                               extension: '.js',
                               before_each: begin_try_catch,
-                              after_each: end_try_catch) do |javascript_file|
-            scraped_site.javascript_file = javascript_file
+                              after_each: end_try_catch) do |javascript_head_file|
+            scraped_site.javascript_file = javascript_head_file
+          end
+
+          downloader.load_all(scraper.javascript_urls_in_body,
+                              extension: '.js',
+                              before_each: begin_try_catch,
+                              after_each: end_try_catch) do |javascript_body_file|
+            scraped_site.javascript_body_file = javascript_body_file
           end
 
           downloader.load_all(scraper.stylesheet_urls,
@@ -35,6 +43,8 @@ module Pageflow
         end
 
         :ok
+      rescue Downloader::HTTPError
+        :error
       end
 
       private

data/app/models/pageflow/chart/scraped_site.rb

@@ -1,12 +1,16 @@
 module Pageflow
   module Chart
     class ScrapedSite < ActiveRecord::Base
+      include Pageflow::ReusableFile
+
       has_attached_file :javascript_file, Chart.config.paperclip_options(extension: 'js')
+      has_attached_file :javascript_body_file, Chart.config.paperclip_options(basename: 'all_body', extension: 'js')
       has_attached_file :stylesheet_file, Chart.config.paperclip_options(extension: 'css')
       has_attached_file :html_file, Chart.config.paperclip_options(extension: 'html')
       has_attached_file :csv_file, Chart.config.paperclip_options(basename: 'data', extension: 'csv')
 
       do_not_validate_attachment_file_type(:javascript_file)
+      do_not_validate_attachment_file_type(:javascript_body_file)
       do_not_validate_attachment_file_type(:stylesheet_file)
       do_not_validate_attachment_file_type(:html_file)
       do_not_validate_attachment_file_type(:csv_file)
@@ -23,6 +27,10 @@ module Pageflow
           transition 'unprocessed' => 'processing'
         end
 
+        event :skip_reprocessing_imported_site do
+          transition 'unprocessed' => 'processed'
+        end
+
         event :reprocess do
           transition 'processed' => 'processing'
           transition 'processing_failed' => 'processing'
@@ -44,10 +52,6 @@ module Pageflow
         URI.join(url, 'data.csv').to_s
       end
 
-      def as_json(*)
-        super.merge(html_file_url: html_file_url)
-      end
-
       def html_file_url
         return unless html_file.try(:path)
         if Chart.config.scraped_sites_root_url.present?
@@ -56,6 +60,35 @@ module Pageflow
           html_file.url
         end
       end
+
+      # ReusableFile-overrides:
+      def url
+        read_attribute(:url)
+      end
+
+      def retryable?
+        processing_failed?
+      end
+
+      def ready?
+        processed?
+      end
+
+      def publish!
+        if html_file.present?
+          skip_reprocessing_imported_site!
+        else
+          process!
+        end
+      end
+
+      def retry!
+        reprocess!
+      end
+
+      def attachments_for_export
+        [javascript_file, javascript_body_file, stylesheet_file, html_file, csv_file]
+      end
     end
   end
 end

data/app/views/pageflow/chart/editor/scraped_sites/_scraped_site.json.jbuilder

@@ -0,0 +1 @@
+json.call(scraped_site, :url, :html_file_url, :use_custom_theme)

data/app/views/pageflow/chart/page.html.erb

@@ -7,9 +7,16 @@
 
   <div class="content">
     <div class="iframeWrapper">
-      <%= scraped_site_iframe(configuration['scraped_site_id']) %>
-      <div class="iframe_overlay"></div>
+      <%= scraped_site_iframe(configuration) %>
+      <%= third_party_embed_opt_in(
+            entry: entry,
+            vendor_name: 'datawrapper',
+            message: t('pageflow.public.chart.opt_in_prompt')
+          ) %>
       <div class="bigscreen_toggler" tabindex="4" title="<%= t('pageflow.public.chart.toggle') %>"><%= t('pageflow.public.chart.toggle') %></div>
+      <div class="opt_out_wrapper">
+        <%= third_party_embed_opt_out_info(entry) %>
+      </div>
     </div>
     <div class="scroller">
       <div>

data/chart.gemspec

@@ -18,11 +18,11 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '~> 2.1'
 
-  spec.add_runtime_dependency 'pageflow', '~> 14.x'
+  spec.add_runtime_dependency 'pageflow', '~> 15.7.x'
   spec.add_runtime_dependency 'nokogiri', '~> 1.0'
   spec.add_runtime_dependency 'pageflow-public-i18n', '~> 1.0'
 
-  spec.add_development_dependency 'pageflow-support', '~> 14.x'
+  spec.add_development_dependency 'pageflow-support', '~> 15.x'
   spec.add_development_dependency 'bundler', ['>= 1.0', '< 3']
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'rspec-rails', '~> 3.0'

data/config/locales/de.yml

@@ -31,6 +31,8 @@ de:
 
             Typische Anwendungsbeispiele: Statistiken, Diagramme, Zahlen&Fakten
       page_attributes:
+        chart_url:
+          label: Diagramm URL
         full_width:
           inline_help: Verbreitert das Diagramm, so dass es die maximal verfügbare Breite nutzt.
           label: Gesamte Breite nutzen
@@ -39,3 +41,5 @@ de:
       page_type_category_name: Daten und Diagramme
       page_type_description: Einbindung von Diagrammen, die mit Datawrapper erstellt wurden
       page_type_name: Diagramm
+    chart_embed_opt_in:
+      feature_name: Opt-In für Datawrapper Embeds

data/config/locales/en.yml

@@ -17,6 +17,8 @@ en:
           menu_item: Chart
           text: "# Chart\n\nIntegration of a Datawrapper-Diagram\n\nHere you can add animated infographics to your Pageflow. The diagram is embedded into a background-picture/video and text. To enlarge the graphic you simply have to click on it. \n\nBut first of all you have to create your graphic externally and generate a link. You can find examples and requirements for this under www.datawrapper.de.\n\nNote that Datawrapper offers a special layout option for charts that shall be embedded into a Pageflow: In the \"Visualize\" step of the chart creation wizard, click the \"Design\" tab and select the \"Pageflow\" layout. That way a color scheme is used which complements Pageflow's look.\n\nExamples of application: statistics, diagrams, numbers & facts"
       page_attributes:
+        chart_url:
+          label: Chart URL
         full_width:
           inline_help: Extends the chart to use the maximum width available.
           label: Use full width
@@ -25,3 +27,5 @@ en:
       page_type_category_name: Data and Charts
       page_type_description: Embedded Datawrapper chart
       page_type_name: Chart
+    chart_embed_opt_in:
+      feature_name: Opt-in for Datawrapper embeds

data/db/migrate/20190531141820_add_file_attributes_to_scraped_sites.rb

@@ -0,0 +1,8 @@
+class AddFileAttributesToScrapedSites < ActiveRecord::Migration[5.2]
+  def change
+    add_reference :pageflow_chart_scraped_sites, :entry, index: true
+    add_column :pageflow_chart_scraped_sites, :rights, :string
+    add_column :pageflow_chart_scraped_sites, :parent_file_id, :integer
+    add_column :pageflow_chart_scraped_sites, :parent_file_model_type, :string
+  end
+end

data/db/migrate/20190531145431_insert_file_usages_for_scraped_sites.rb

@@ -0,0 +1,59 @@
+class InsertFileUsagesForScrapedSites < ActiveRecord::Migration[5.2]
+  # Pageflow models might have gotten out of sync with schema at this
+  # point. Use local models instead.
+  class MigratedPage < ActiveRecord::Base
+    self.table_name = 'pageflow_pages'
+
+    belongs_to :chapter, class_name: 'MigratedChapter'
+
+    serialize :configuration, JSON
+
+    def configuration
+      super || {}
+    end
+  end
+
+  class MigratedChapter < ActiveRecord::Base
+    self.table_name = 'pageflow_chapters'
+    belongs_to :storyline, class_name: 'MigratedStoryline'
+  end
+
+  class MigratedStoryline < ActiveRecord::Base
+    self.table_name = 'pageflow_storylines'
+  end
+
+  class MigratedFileUsage < ActiveRecord::Base
+    self.table_name = 'pageflow_file_usages'
+  end
+
+  def up
+    MigratedPage.where(template: 'chart').find_each do |page|
+      scraped_site_id = page.configuration['scraped_site_id']
+      next unless scraped_site_id
+
+      scraped_site = Pageflow::Chart::ScrapedSite.find_by_id(scraped_site_id)
+
+      unless scraped_site
+        puts "Scraped site #{scraped_site_id} not found"
+        next
+      end
+
+      revision_id = page&.chapter&.storyline&.revision_id
+
+      unless revision_id
+        puts "No revision_id for page #{page.id}"
+        next
+      end
+
+      MigratedFileUsage.create(file_id: scraped_site.id,
+                               file_type: 'Pageflow::Chart::ScrapedSite',
+                               revision_id: revision_id)
+    end
+  end
+
+  def down
+    MigratedFileUsage
+      .where(file_type: 'Pageflow::Chart::ScrapedSite')
+      .delete_all
+  end
+end

data/db/migrate/20200507141608_add_javascript_body_attachment_to_scraped_site.rb

@@ -0,0 +1,5 @@
+class AddJavascriptBodyAttachmentToScrapedSite < ActiveRecord::Migration[5.2]
+  def change
+    add_attachment :pageflow_chart_scraped_sites, :javascript_body_file
+  end
+end

data/lib/pageflow/chart/configuration.rb

@@ -22,6 +22,7 @@ module Pageflow
       #
       # @param [Hash] opts
       # @option opts [Array<Regexp>] :head_script_blacklist Script tags in page head are ignored if they match any of this list of regexes.
+      # @option opts [Array<Regexp>] :body_script_blacklist Script tags in page body are ignored if they match any of this list of regexes.
       # @option opts [Array<Regexp>] :inline_script_blacklist Inline script tags are ignored if they match any of this list of regexes.
       # @option opts [Array<String>] :selector_blacklist HTML-elements matched by selectors in this list will not be scraped.
       # @return [Hash]
@@ -55,6 +56,7 @@ module Pageflow
       def initialize
         @scraper_options = {
           head_script_blacklist: [/piwik/],
+          body_script_blacklist: [/piwik/],
           inline_script_blacklist: [/piwik/],
           selector_blacklist: ['body .noscript']
         }
@@ -62,9 +64,10 @@ module Pageflow
         @paperclip_base_path = ':pageflow_s3_root'
         @scraped_sites_root_url = nil
         @supported_hosts = [
-          'http://cf.datawrapper.de',
-          'http://datawrapper.dwcdn.de',
-          'http://datawrapper.dwcdn.net'
+          'cf.datawrapper.de',
+          'charts.datawrapper.de',
+          'datawrapper.dwcdn.de',
+          'datawrapper.dwcdn.net'
         ]
         @use_custom_theme = false
         @datawrapper_themes_with_transparent_background_support = ['pageflow']

data/lib/pageflow/chart/downloader.rb

@@ -6,15 +6,18 @@ module Pageflow
     class Downloader
       attr_reader :options
 
+      class HTTPError < StandardError; end
+
       def initialize(options = {})
         @options = options
       end
 
-      def load(url)
+      def load(url, raise_on_http_error: false)
         file = open(make_absolute(url))
         yield(file)
       rescue OpenURI::HTTPError => exception
         Rails.logger.error "Exception loading url #{url}: #{exception.message}"
+        raise(HTTPError) if raise_on_http_error
       ensure
         file.close if file
       end

data/lib/pageflow/chart/page_type.rb

@@ -10,6 +10,23 @@ module Pageflow
       def json_seed_template
         'pageflow/chart/page_type.json.jbuilder'
       end
+
+      def file_types
+        [Chart.scraped_site_file_type]
+      end
+    end
+
+    def self.scraped_site_file_type
+      FileType.new(model: ScrapedSite,
+                   editor_partial: 'pageflow/chart/editor/scraped_sites/scraped_site',
+                   custom_attributes: {
+                     url: {
+                       permitted_create_param: true
+                     },
+                     use_custom_theme: {
+                       permitted_create_param: false
+                     }
+                   })
     end
   end
 end

data/lib/pageflow/chart/plugin.rb

@@ -0,0 +1,10 @@
+module Pageflow
+  module Chart
+    class Plugin < Pageflow::Plugin
+      def configure(config)
+        config.page_types.register(Chart.page_type)
+        config.features.register('chart_embed_opt_in')
+      end
+    end
+  end
+end

data/lib/pageflow/chart/refresh_tag_following_downloader.rb

@@ -9,15 +9,15 @@ module Pageflow
       class TooManyRedirects < StandardError; end
       class NoUrlInRefreshMetaTag < StandardError; end
 
-      def load_following_refresh_tags(url, redirect_count = 0, &block)
-        load(url) do |file|
+      def load_following_refresh_tags(url, options = {}, redirect_count = 0, &block)
+        load(url, options) do |file|
           if (redirect_url = find_refresh_meta_tag_url(file.read))
             if redirect_count >= MAX_REDIRECT_COUNT
               raise TooManyRedirects, 'Too many redirects via refresh meta tags.'
             end
 
             redirect_url = ensure_absolute(redirect_url, url)
-            return load_following_refresh_tags(redirect_url, redirect_count + 1, &block)
+            return load_following_refresh_tags(redirect_url, options, redirect_count + 1, &block)
           end
 
           file.rewind

data/lib/pageflow/chart/scraper.rb

@@ -3,7 +3,11 @@ require 'nokogiri'
 module Pageflow
   module Chart
     class Scraper
-      attr_reader :document, :options, :javascript_urls, :stylesheet_urls
+      attr_reader :document,
+                  :options,
+                  :javascript_urls_in_head,
+                  :javascript_urls_in_body,
+                  :stylesheet_urls
 
       def initialize(html, options = {})
         @document = Nokogiri::HTML(html)
@@ -23,14 +27,21 @@ module Pageflow
       private
 
       def parse
-        parse_javascript_urls
+        parse_javascript_urls(:head)
+        parse_javascript_urls(:body)
         parse_stylesheet_urls
       end
 
-      def parse_javascript_urls
-        @javascript_urls = filtered_script_tags_in_head.map do |tag|
+      def parse_javascript_urls(container)
+        script_tags = filtered_script_tags_in(container).map do |tag|
           tag[:src]
         end
+
+        if container.eql?(:head)
+          @javascript_urls_in_head = script_tags
+        else
+          @javascript_urls_in_body = script_tags
+        end
       end
 
       def parse_stylesheet_urls
@@ -42,7 +53,8 @@ module Pageflow
       def rewrite
         filter_inline_scripts
         filter_by_selectors
-        combine_script_tags_in_head
+        combine_script_tags_in(:head)
+        combine_script_tags_in(:body)
         combine_css_link_tags
       end
 
@@ -66,12 +78,12 @@ module Pageflow
         end
       end
 
-      def combine_script_tags_in_head
-        script_tags_to_remove = script_src_tags_in_head
+      def combine_script_tags_in(container)
+        script_tags_to_remove = script_src_tags_in(container)
         return if script_tags_to_remove.empty?
 
         all_script_src_tag = Nokogiri::XML::Node.new('script', document)
-        all_script_src_tag[:src] = 'all.js'
+        all_script_src_tag[:src] = container.eql?(:head) ? 'all.js' : 'all_body.js'
         all_script_src_tag[:type] = 'text/javascript'
 
         script_tags_to_remove
@@ -91,16 +103,16 @@ module Pageflow
         document.at_css('head') << all_css_link_tag
       end
 
-      def filtered_script_tags_in_head
-        script_src_tags_in_head.reject do |tag|
-          options.fetch(:head_script_blacklist, []).any? do |regexp|
+      def filtered_script_tags_in(container)
+        script_src_tags_in(container).reject do |tag|
+          options.fetch("#{container}_script_blacklist".to_sym, []).any? do |regexp|
             tag[:src] =~ regexp
           end
         end
       end
 
-      def script_src_tags_in_head
-        document.css('head script[src]')
+      def script_src_tags_in(container)
+        document.css("#{container} script[src]")
       end
 
       def css_link_tags

data/lib/pageflow/chart/version.rb

@@ -1,5 +1,5 @@
 module Pageflow
   module Chart
-    VERSION = '2.1.0'.freeze
+    VERSION = '2.4.0'.freeze
   end
 end

data/lib/pageflow/chart.rb

@@ -10,6 +10,10 @@ module Pageflow
       block.call(config)
     end
 
+    def self.plugin
+      Chart::Plugin.new
+    end
+
     def self.page_type
       Chart::PageType.new
     end

data/spec/factories/scraped_sites.rb

@@ -1,5 +1,19 @@
-FactoryBot.define do
-  factory :scraped_site, class: 'Pageflow::Chart::ScrapedSite' do
-    url 'MyString'
+module Pageflow
+  module Chart
+    FactoryBot.define do
+      factory :scraped_site, class: 'Pageflow::Chart::ScrapedSite' do
+        url { 'MyString' }
+
+        trait :processed do
+          state { 'processed' }
+
+          javascript_file { File.open(Engine.root.join('spec', 'fixtures', 'all.js')) }
+          javascript_body_file { File.open(Engine.root.join('spec', 'fixtures', 'all_body.js')) }
+          stylesheet_file { File.open(Engine.root.join('spec', 'fixtures', 'all.css')) }
+          html_file { File.open(Engine.root.join('spec', 'fixtures', 'index.html')) }
+          csv_file { File.open(Engine.root.join('spec', 'fixtures', 'data.csv')) }
+        end
+      end
+    end
   end
 end

data/spec/fixtures/all.css

@@ -0,0 +1,3 @@
+.some {
+  color: #000
+}

data/spec/fixtures/all.js

@@ -0,0 +1 @@
+var chart = {};

data/spec/fixtures/all_body.js

@@ -0,0 +1 @@
+var chart_body = {};

data/spec/fixtures/data.csv

@@ -0,0 +1 @@
+some,values

data/spec/fixtures/index.html

@@ -0,0 +1,7 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Title</title>
+  </head>
+  <body></body>
+</html>

data/spec/helpers/pageflow/chart/scraped_sites_helper_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+require 'pageflow/used_file_test_helper'
+
+module Pageflow
+  module Chart
+    describe ScrapedSitesHelper do
+      include UsedFileTestHelper
+
+      before { Pageflow::Chart.config.use_custom_theme = false }
+
+      it 'renders iframe with data-src attribute for scraped site' do
+        scraped_site = create_used_file(:scraped_site, :processed)
+
+        html = scraped_site_iframe('scraped_site_id' => scraped_site.perma_id)
+
+        iframe = Capybara.string(html).find('iframe')
+        expect(iframe['data-src']).to match(%r{original/index\.html})
+      end
+
+      it 'renders no data-custom-theme attribute by default' do
+        scraped_site = create_used_file(:scraped_site, :processed)
+
+        html = scraped_site_iframe('scraped_site_id' => scraped_site.perma_id)
+
+        iframe = Capybara.string(html).find('iframe')
+        expect(iframe['data-use-custom-theme']).to be_blank
+        expect(iframe['data-customize-layout']).to eq('true')
+      end
+
+      it 'renders data-custom-theme if site has custom theme' do
+        Pageflow::Chart.config.use_custom_theme = true
+        scraped_site = create_used_file(:scraped_site, :processed)
+
+        html = scraped_site_iframe('scraped_site_id' => scraped_site.perma_id)
+
+        iframe = Capybara.string(html).find('iframe')
+        expect(iframe['data-use-custom-theme']).to eq('true')
+        expect(iframe['data-customize-layout']).to eq('true')
+      end
+
+      it 'renders iframe with data-src attribute for chart_url' do
+        html = scraped_site_iframe('chart_url' => 'https://example.com/chart')
+
+        iframe = Capybara.string(html).find('iframe')
+        expect(iframe['data-src']).to eq('//example.com/chart')
+      end
+
+      it 'renders no data-custom-theme attribute for chart_url' do
+        html = scraped_site_iframe('chart_url' => 'https://example.com/chart')
+
+        iframe = Capybara.string(html).find('iframe')
+
+        expect(iframe['data-customize-layout']).to be_blank
+        expect(iframe['data-use-custom-theme']).to be_blank
+      end
+    end
+  end
+end

data/spec/integration/file_type_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+require 'pageflow/lint'
+
+module Pageflow
+  module Chart
+    Pageflow::Lint.file_type(:scraped_site,
+                             create_file_type: -> { Chart.scraped_site_file_type },
+                             create_file: -> { create(:scraped_site, :processed) })
+  end
+end

data/spec/jobs/pageflow/chart/scrape_site_job_spec.rb

@@ -12,10 +12,23 @@ module Pageflow
 
           allow(Scraper).to receive(:new).and_return(scraper)
 
-          expect(downloader).to receive(:load_following_refresh_tags).with('http://example.com')
+          expect(downloader).to receive(:load_following_refresh_tags)
+            .with('http://example.com',
+                  raise_on_http_error: true)
 
           job.perform_with_result(scraped_site, {}, downloader: downloader)
         end
+
+        it 'returns :error on HTTP error' do
+          job = ScrapeSiteJob.new
+          scraped_site = create(:scraped_site, url: 'http://example.com/a')
+
+          stub_request(:get, 'http://example.com/a').to_return(status: 404, body: '')
+
+          result = job.perform_with_result(scraped_site)
+
+          expect(result).to eq(:error)
+        end
       end
     end
   end