pageflow-chart 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69e58dc4cf527ebcc87f4fb761209cbc4c943f851dbbf317c34b1dc386b98b30
-  data.tar.gz: d7e671cc45b645abe390dc01ba7db1a59b117dffc9ff873284d5864a9ec2cad7
+  metadata.gz: c3834ff2c73f8f33f70e43de722ed33e5a763ebe078ac4345074b8c7e49ebf3d
+  data.tar.gz: ab502c12ff710d698df3418fd572f3153d6981b0526cd9a635adc2ab1f8cbe64
 SHA512:
-  metadata.gz: fe79c1727ae70e0184a93c568fc042edcbaddf86ab5d51fda3040dec8a037a57493233271cae64208735c9b3bbc9ece1d72dc5fe07a9e0feddf1d63dd421347d
-  data.tar.gz: 29f7ecd19a0dbd9ed7c0f21db3be6bd1cb7a47d71282ea74e6b7077ce7fe28b16fc1453af7a6af3bb7dc78c53aa77c2cfc8be1c2b024b96eac9f94fc79403e68
+  metadata.gz: e326231888b882249d84824451b0449d598aca5b9c78f6842b20111ffe4c9c0b7079f3966cd0a654c4552cf59eb1c57442556c0cfe5370b194e46f1be1b12525
+  data.tar.gz: e14b7bacc6d778ca93764d48fc2f8fd5e484569cab89edf395848e478699fe0ab91ea6e0944315550aceae6e226428aa9bab678b0e9a9bb8887dfa5687b7c93d

data/.travis.yml

@@ -8,6 +8,7 @@ cache: bundler
 
 services:
   - redis-server
+  - mysql
 
 script:
   - bin/rspec

data/CHANGELOG.md

@@ -1,16 +1,16 @@
 # CHANGELOG
 
-### Version 2.1.0
+### Version 2.2.0
 
-2019-04-08
+2019-11-04
 
-[Compare changes](https://github.com/codevise/pageflow-chart/compare/2-0-stable...v2.1.0)
+[Compare changes](https://github.com/codevise/pageflow-chart/compare/2-1-stable...v2.2.0)
 
-- Adapt page dom for pageflow 14
-  ([#51](https://github.com/codevise/pageflow-chart/pull/51))
-- Setup page type lint specs
-  ([#50](https://github.com/codevise/pageflow-chart/pull/50))
+#### Manual Update Step
+
+- Turn scraped site into file type. Install migrations.
+  ([#55](https://github.com/codevise/pageflow-chart/pull/55))
 
 See
-[2-0-stable branch](https://github.com/codevise/pageflow-chart/blob/2-0-stable/CHANGELOG.md)
+[2-1-stable branch](https://github.com/codevise/pageflow-chart/blob/2-1-stable/CHANGELOG.md)
 for previous changes.

data/app/assets/javascripts/pageflow/chart/editor.js

@@ -1,9 +1,10 @@
 //= require_self
+
 //= require_tree ./editor/models
-//= require_tree ./editor/collections
 //= require_tree ./editor/views/inputs
 //= require_tree ./editor/templates
 //= require_tree ./editor/views
-//= require_tree ./editor/initializers
 
-pageflow.chart = pageflow.chart || {};
+//= require ./editor/config
+
+pageflow.chart = pageflow.chart || {};

data/app/assets/javascripts/pageflow/chart/editor/config.js

@@ -0,0 +1,7 @@
+pageflow.editor.fileTypes.register('pageflow_chart_scraped_sites', {
+  model: pageflow.chart.ScrapedSite,
+
+  matchUpload: function(upload) {
+    return false;
+  }
+});

data/app/assets/javascripts/pageflow/chart/editor/models/scraped_site.js

@@ -1,55 +1,16 @@
-pageflow.chart.ScrapedSite = Backbone.Model.extend({
-  modelName: 'scraped_site',
-  paramRoot: 'scraped_site',
-
-  initialize: function() {
-    this.listenTo(this, 'sync', function() {
-      if (this.isProcessing() && !this.pollingInterval) {
-        this.pollUntilScraped();
-      }
-    });
-  },
-
-  urlRoot: function() {
-    return '/chart/scraped_sites';
-  },
-
-  isProcessed: function() {
-    return this.get('state') === 'processed';
-  },
-
-  isProcessing: function() {
-    return this.get('state') === 'processing';
-  },
-
-  isFailed: function() {
-    return this.get('state') === 'processing_failed';
-  },
-
-  pollUntilScraped: function() {
-    var model = this;
-
-    if (model.isProcessed()) {
-      return;
+pageflow.chart.ScrapedSite = pageflow.ReusableFile.extend({
+  stages: [
+    {
+      name: 'processing',
+      activeStates: ['processing'],
+      finishedStates: ['processed'],
+      failedStates: ['processing_failed']
     }
+  ],
 
-    model.pollingInterval = setInterval(poll, 1000);
+  readyState: 'processed',
 
-    function stopPolling() {
-      if (model.pollingInterval) {
-        clearInterval(model.pollingInterval);
-        model.pollingInterval = null;
-      }
-    }
-
-    function poll() {
-      model.fetch({
-        success: function() {
-          if (!model.isProcessing()) {
-            stopPolling();
-          }
-        }
-      });
-    }
+  toJSON: function() {
+    return _.pick(this.attributes, 'url');
   }
 });

data/app/assets/javascripts/pageflow/chart/editor/views/configuration_editor.js

@@ -12,6 +12,9 @@ pageflow.ConfigurationEditorView.register('chart', {
         displayPropertyName: 'display_scraped_site_url',
         required: true
       });
+      this.input('scraped_site_id', pageflow.FileProcessingStateDisplayView, {
+        collection: 'pageflow_chart_scraped_sites'
+      });
       this.view(pageflow.chart.DatawrapperAdView);
       this.input('full_width', pageflow.CheckBoxInputView);
       this.group('background');
@@ -25,4 +28,4 @@ pageflow.ConfigurationEditorView.register('chart', {
       this.group('options');
     });
   }
-});
+});

data/app/assets/javascripts/pageflow/chart/editor/views/embedded/iframe_embedded_view.js

@@ -15,29 +15,23 @@ pageflow.chart.IframeEmbeddedView = Backbone.Marionette.View.extend({
   },
 
   updateScrapedSite: function() {
-    var view = this;
-
     if (this.scrapedSite) {
       this.stopListening(this.scrapedSite);
     }
 
-    var scrapedSiteId = this.model.get(this.options.propertyName);
-
-    if (scrapedSiteId) {
-      this.scrapedSite = pageflow.chart.scrapedSites.getOrFetch(scrapedSiteId, {
-        success: function(scrapedSite) {
-          view.updateAttributes(scrapedSite);
-        }
-      });
+    this.scrapedSite = this.model.getReference(this.options.propertyName,
+                                               'pageflow_chart_scraped_sites');
+    this.updateAttributes();
 
+    if (this.scrapedSite) {
       this.listenTo(this.scrapedSite, 'change', this.updateAttributes);
     }
   },
 
-  updateAttributes: function(scrapedSite) {
-    scrapedSite = scrapedSite || this.scrapedSite;
+  updateAttributes: function() {
+    var scrapedSite = this.scrapedSite;
 
-    if (scrapedSite && scrapedSite.isProcessed()) {
+    if (scrapedSite && scrapedSite.isReady()) {
       this.$el.attr('src', scrapedSite.get('html_file_url'));
 
       if (scrapedSite.get('use_custom_theme')) {
@@ -51,4 +45,4 @@ pageflow.chart.IframeEmbeddedView = Backbone.Marionette.View.extend({
       this.$el.attr('src', '');
     }
   }
-});
+});

data/app/assets/javascripts/pageflow/chart/editor/views/inputs/scraped_url_input_view.js

@@ -1,49 +1,28 @@
 pageflow.chart.ScrapedUrlInputView = pageflow.UrlInputView.extend({
-  template: 'pageflow/chart/editor/templates/url_input',
+  className: 'chart_scraped_site_url_input_view',
 
-  regions: {
-    statusContainer: '.status_container'
-  },
+  save: function() {
+    var url = this.ui.input.val();
 
-  onLoad: function() {
-    this.listenTo(this.model, 'change:' + this.options.propertyName, function() {
-      this.updateScrapingStatus();
-    });
+    if (url) {
+      var scrapedSite = pageflow
+        .entry
+        .getFileCollection('pageflow_chart_scraped_sites')
+        .findOrCreateBy({url: url});
 
-    this.updateScrapingStatus();
-  },
+      if (scrapedSite.isRetryable()) {
+        scrapedSite.retry();
+      }
 
-  transformPropertyValue: function(url) {
-    return $.Deferred(function(deferred) {
-      pageflow.chart.scrapedSites.create(
-        {
-          url: url
-        },
-        {
-          success: function(scrapedSite) {
-            deferred.resolve(scrapedSite.id);
-          }
-        }
+      this.model.setReference(
+        this.options.propertyName,
+        scrapedSite
       );
-    }).promise();
-  },
-
-  updateScrapingStatus: function() {
-    var scrapedSite = this.getScrapedSite();
-
-    if (scrapedSite) {
-      this.statusContainer.show(new pageflow.chart.ScrapedSiteStatusView({
-        model: scrapedSite
-      }));
     }
     else {
-      this.statusContainer.close();
-    }
-  },
-
-  getScrapedSite: function() {
-    if (this.model.has(this.options.propertyName)) {
-      return pageflow.chart.scrapedSites.getOrFetch(this.model.get(this.options.propertyName));
+      this.model.unsetReference(
+        this.options.propertyName
+      );
     }
   }
-});
+});

data/app/assets/stylesheets/pageflow/chart.scss

@@ -4,7 +4,6 @@
   .iframeWrapper {
     margin-bottom: 40px;
     pointer-events: all;
-    height: 400px;
     position: relative;
 
     @media (min-width: 1500px) {

data/app/assets/stylesheets/pageflow/chart/editor.scss

@@ -1,19 +1,5 @@
-.scraped_site_status {
-  .state {
-    display: none;
-  }
-
-  &.failed {
-    .failed.state {
-      display: block;
-    }
-  }
-
-  &.processing {
-    .processing.state {
-      display: block;
-    }
-  }
+.chart_scraped_site_url_input_view {
+  margin-bottom: 6px;
 }
 
 .datawrapper_ad {
@@ -23,6 +9,5 @@
     @include button('simple', #6AACF7);
     padding: 3px 10px;
     width: auto;
-    float: right;
   }
 }

data/app/assets/stylesheets/pageflow/chart/themes/default.scss

@@ -1,6 +1,7 @@
 @include pageflow-page-type-pictograms("chart");
 
 $chart-background: $basic-background-color;
+$chart-iframe-base-height: 400px !default;
 
 .chart_page {
   .iframeWrapper {
@@ -8,5 +9,7 @@ $chart-background: $basic-background-color;
       opacity: 0.95;
       background-color: $chart-background;
     }
+
+    height: $chart-iframe-base-height;
   }
 }

data/app/helpers/pageflow/chart/scraped_sites_helper.rb

@@ -12,7 +12,7 @@ module Pageflow
       }
 
       def scraped_site_iframe(scraped_site_id)
-        scraped_site = ScrapedSite.find_by_id(scraped_site_id)
+        scraped_site = find_file_in_entry(ScrapedSite, scraped_site_id)
         data_attributes = {}
 
         if scraped_site

data/app/jobs/pageflow/chart/scrape_site_job.rb

@@ -8,7 +8,8 @@ module Pageflow
       def perform_with_result(scraped_site,
                               _options = {},
                               downloader: downloader_for(scraped_site))
-        downloader.load_following_refresh_tags(scraped_site.url) do |file|
+        downloader.load_following_refresh_tags(scraped_site.url,
+                                               raise_on_http_error: true) do |file|
           scraper = Scraper.new(file.read, Chart.config.scraper_options)
           scraped_site.html_file = StringIOWithContentType.new(
             scraper.html,
@@ -35,6 +36,8 @@ module Pageflow
         end
 
         :ok
+      rescue Downloader::HTTPError
+        :error
       end
 
       private

data/app/models/pageflow/chart/scraped_site.rb

@@ -1,6 +1,8 @@
 module Pageflow
   module Chart
     class ScrapedSite < ActiveRecord::Base
+      include Pageflow::ReusableFile
+
       has_attached_file :javascript_file, Chart.config.paperclip_options(extension: 'js')
       has_attached_file :stylesheet_file, Chart.config.paperclip_options(extension: 'css')
       has_attached_file :html_file, Chart.config.paperclip_options(extension: 'html')
@@ -23,6 +25,10 @@ module Pageflow
           transition 'unprocessed' => 'processing'
         end
 
+        event :skip_reprocessing_imported_site do
+          transition 'unprocessed' => 'processed'
+        end
+
         event :reprocess do
           transition 'processed' => 'processing'
           transition 'processing_failed' => 'processing'
@@ -44,10 +50,6 @@ module Pageflow
         URI.join(url, 'data.csv').to_s
       end
 
-      def as_json(*)
-        super.merge(html_file_url: html_file_url)
-      end
-
       def html_file_url
         return unless html_file.try(:path)
         if Chart.config.scraped_sites_root_url.present?
@@ -56,6 +58,35 @@ module Pageflow
           html_file.url
         end
       end
+
+      # ReusableFile-overrides:
+      def url
+        read_attribute(:url)
+      end
+
+      def retryable?
+        processing_failed?
+      end
+
+      def ready?
+        processed?
+      end
+
+      def publish!
+        if html_file.present?
+          skip_reprocessing_imported_site!
+        else
+          process!
+        end
+      end
+
+      def retry!
+        reprocess!
+      end
+
+      def attachments_for_export
+        [javascript_file, stylesheet_file, html_file, csv_file]
+      end
     end
   end
 end

data/app/views/pageflow/chart/editor/scraped_sites/_scraped_site.json.jbuilder

@@ -0,0 +1 @@
+json.call(scraped_site, :url, :html_file_url, :use_custom_theme)

data/chart.gemspec

@@ -18,11 +18,11 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '~> 2.1'
 
-  spec.add_runtime_dependency 'pageflow', '~> 14.x'
+  spec.add_runtime_dependency 'pageflow', '~> 15.x'
   spec.add_runtime_dependency 'nokogiri', '~> 1.0'
   spec.add_runtime_dependency 'pageflow-public-i18n', '~> 1.0'
 
-  spec.add_development_dependency 'pageflow-support', '~> 14.x'
+  spec.add_development_dependency 'pageflow-support', '~> 15.x'
   spec.add_development_dependency 'bundler', ['>= 1.0', '< 3']
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'rspec-rails', '~> 3.0'

data/db/migrate/20190531141820_add_file_attributes_to_scraped_sites.rb

@@ -0,0 +1,8 @@
+class AddFileAttributesToScrapedSites < ActiveRecord::Migration[5.2]
+  def change
+    add_reference :pageflow_chart_scraped_sites, :entry, index: true
+    add_column :pageflow_chart_scraped_sites, :rights, :string
+    add_column :pageflow_chart_scraped_sites, :parent_file_id, :integer
+    add_column :pageflow_chart_scraped_sites, :parent_file_model_type, :string
+  end
+end

data/db/migrate/20190531145431_insert_file_usages_for_scraped_sites.rb

@@ -0,0 +1,59 @@
+class InsertFileUsagesForScrapedSites < ActiveRecord::Migration[5.2]
+  # Pageflow models might have gotten out of sync with schema at this
+  # point. Use local models instead.
+  class MigratedPage < ActiveRecord::Base
+    self.table_name = 'pageflow_pages'
+
+    belongs_to :chapter, class_name: 'MigratedChapter'
+
+    serialize :configuration, JSON
+
+    def configuration
+      super || {}
+    end
+  end
+
+  class MigratedChapter < ActiveRecord::Base
+    self.table_name = 'pageflow_chapters'
+    belongs_to :storyline, class_name: 'MigratedStoryline'
+  end
+
+  class MigratedStoryline < ActiveRecord::Base
+    self.table_name = 'pageflow_storylines'
+  end
+
+  class MigratedFileUsage < ActiveRecord::Base
+    self.table_name = 'pageflow_file_usages'
+  end
+
+  def up
+    MigratedPage.where(template: 'chart').find_each do |page|
+      scraped_site_id = page.configuration['scraped_site_id']
+      next unless scraped_site_id
+
+      scraped_site = Pageflow::Chart::ScrapedSite.find_by_id(scraped_site_id)
+
+      unless scraped_site
+        puts "Scraped site #{scraped_site_id} not found"
+        next
+      end
+
+      revision_id = page&.chapter&.storyline&.revision_id
+
+      unless revision_id
+        puts "No revision_id for page #{page.id}"
+        next
+      end
+
+      MigratedFileUsage.create(file_id: scraped_site.id,
+                               file_type: 'Pageflow::Chart::ScrapedSite',
+                               revision_id: revision_id)
+    end
+  end
+
+  def down
+    MigratedFileUsage
+      .where(file_type: 'Pageflow::Chart::ScrapedSite')
+      .delete_all
+  end
+end

data/lib/pageflow/chart/downloader.rb

@@ -6,15 +6,18 @@ module Pageflow
     class Downloader
       attr_reader :options
 
+      class HTTPError < StandardError; end
+
       def initialize(options = {})
         @options = options
       end
 
-      def load(url)
+      def load(url, raise_on_http_error: false)
         file = open(make_absolute(url))
         yield(file)
       rescue OpenURI::HTTPError => exception
         Rails.logger.error "Exception loading url #{url}: #{exception.message}"
+        raise(HTTPError) if raise_on_http_error
       ensure
         file.close if file
       end

data/lib/pageflow/chart/page_type.rb

@@ -10,6 +10,23 @@ module Pageflow
       def json_seed_template
         'pageflow/chart/page_type.json.jbuilder'
       end
+
+      def file_types
+        [Chart.scraped_site_file_type]
+      end
+    end
+
+    def self.scraped_site_file_type
+      FileType.new(model: ScrapedSite,
+                   editor_partial: 'pageflow/chart/editor/scraped_sites/scraped_site',
+                   custom_attributes: {
+                     url: {
+                       permitted_create_param: true
+                     },
+                     use_custom_theme: {
+                       permitted_create_param: false
+                     }
+                   })
     end
   end
 end

data/lib/pageflow/chart/refresh_tag_following_downloader.rb

@@ -9,15 +9,15 @@ module Pageflow
       class TooManyRedirects < StandardError; end
       class NoUrlInRefreshMetaTag < StandardError; end
 
-      def load_following_refresh_tags(url, redirect_count = 0, &block)
-        load(url) do |file|
+      def load_following_refresh_tags(url, options = {}, redirect_count = 0, &block)
+        load(url, options) do |file|
           if (redirect_url = find_refresh_meta_tag_url(file.read))
             if redirect_count >= MAX_REDIRECT_COUNT
               raise TooManyRedirects, 'Too many redirects via refresh meta tags.'
             end
 
             redirect_url = ensure_absolute(redirect_url, url)
-            return load_following_refresh_tags(redirect_url, redirect_count + 1, &block)
+            return load_following_refresh_tags(redirect_url, options, redirect_count + 1, &block)
           end
 
           file.rewind

data/lib/pageflow/chart/version.rb

@@ -1,5 +1,5 @@
 module Pageflow
   module Chart
-    VERSION = '2.1.0'.freeze
+    VERSION = '2.2.0'.freeze
   end
 end

data/spec/factories/scraped_sites.rb

@@ -1,5 +1,18 @@
-FactoryBot.define do
-  factory :scraped_site, class: 'Pageflow::Chart::ScrapedSite' do
-    url 'MyString'
+module Pageflow
+  module Chart
+    FactoryBot.define do
+      factory :scraped_site, class: 'Pageflow::Chart::ScrapedSite' do
+        url { 'MyString' }
+
+        trait :processed do
+          state { 'processed' }
+
+          javascript_file { File.open(Engine.root.join('spec', 'fixtures', 'all.js')) }
+          stylesheet_file { File.open(Engine.root.join('spec', 'fixtures', 'all.css')) }
+          html_file { File.open(Engine.root.join('spec', 'fixtures', 'index.html')) }
+          csv_file { File.open(Engine.root.join('spec', 'fixtures', 'data.csv')) }
+        end
+      end
+    end
   end
 end

data/spec/fixtures/all.css

@@ -0,0 +1,3 @@
+.some {
+  color: #000
+}

data/spec/fixtures/all.js

@@ -0,0 +1 @@
+var chart = {};

data/spec/fixtures/data.csv

@@ -0,0 +1 @@
+some,values

data/spec/fixtures/index.html

@@ -0,0 +1,7 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Title</title>
+  </head>
+  <body></body>
+</html>

data/spec/integration/file_type_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+require 'pageflow/lint'
+
+module Pageflow
+  module Chart
+    Pageflow::Lint.file_type(:scraped_site,
+                             create_file_type: -> { Chart.scraped_site_file_type },
+                             create_file: -> { create(:scraped_site, :processed) })
+  end
+end

data/spec/jobs/pageflow/chart/scrape_site_job_spec.rb

@@ -12,10 +12,23 @@ module Pageflow
 
           allow(Scraper).to receive(:new).and_return(scraper)
 
-          expect(downloader).to receive(:load_following_refresh_tags).with('http://example.com')
+          expect(downloader).to receive(:load_following_refresh_tags)
+            .with('http://example.com',
+                  raise_on_http_error: true)
 
           job.perform_with_result(scraped_site, {}, downloader: downloader)
         end
+
+        it 'returns :error on HTTP error' do
+          job = ScrapeSiteJob.new
+          scraped_site = create(:scraped_site, url: 'http://example.com/a')
+
+          stub_request(:get, 'http://example.com/a').to_return(status: 404, body: '')
+
+          result = job.perform_with_result(scraped_site)
+
+          expect(result).to eq(:error)
+        end
       end
     end
   end

data/spec/models/pageflow/chart/scraped_site_spec.rb

@@ -26,5 +26,59 @@ module Pageflow::Chart
       expect(scraped_site_with_custom_theme.use_custom_theme).to eq(true)
       expect(scraped_site_without_custom_theme.use_custom_theme).to eq(false)
     end
+
+    it 'exposes all attachments for export' do
+      scraped_site = ScrapedSite.new(url: 'http://example.com/foo/index.html')
+
+      expect(scraped_site.attachments_for_export.map(&:name))
+        .to eq(%i[javascript_file stylesheet_file html_file csv_file])
+    end
+
+    describe '#publish!' do
+      it 'transitions state to processing for new site' do
+        scraped_site = ScrapedSite.new(url: 'http://example.com/foo/index.html')
+
+        scraped_site.publish!
+
+        expect(scraped_site.state).to eq('processing')
+      end
+
+      it 'transitions state to processed if html file is already set ' \
+         '(e.g. for sites that have been created via entry import)' do
+        scraped_site = ScrapedSite.new(url: 'http://example.com/foo/index.html',
+                                       html_file_file_name: 'index.html')
+
+        scraped_site.publish!
+
+        expect(scraped_site.state).to eq('processed')
+      end
+    end
+
+    describe '#retryable?' do
+      it 'is true if processing_failed' do
+        scraped_site = ScrapedSite.new(url: 'http://example.com/foo/index.html',
+                                       state: 'processing_failed')
+
+        expect(scraped_site).to be_retryable
+      end
+
+      it 'is false if processed' do
+        scraped_site = ScrapedSite.new(url: 'http://example.com/foo/index.html',
+                                       state: 'processed')
+
+        expect(scraped_site).not_to be_retryable
+      end
+    end
+
+    describe '#retry!' do
+      it 'transitions state to processing if processing_failed' do
+        scraped_site = ScrapedSite.new(url: 'http://example.com/foo/index.html',
+                                       state: 'processing_failed')
+
+        scraped_site.retry!
+
+        expect(scraped_site.state).to eq('processing')
+      end
+    end
   end
 end

data/spec/pageflow/chart/downloader_spec.rb

@@ -17,17 +17,27 @@ module Pageflow
           expect(result).to eq("aaa")
         end
 
-        it 'ignores HTTP response 404' do
+        it 'ignores HTTP response 404 by default' do
           downloader = Downloader.new
           result = ''
 
-          stub_request(:get, "http://example.com/a").to_return(status: 404, body: 'aaa')
+          stub_request(:get, 'http://example.com/a').to_return(status: 404, body: 'aaa')
 
           downloader.load('http://example.com/a') do |io|
             result = io.read
           end
 
-          expect(result).to eq("")
+          expect(result).to eq('')
+        end
+
+        it 'supports raising error on HTTP response 404 ' do
+          downloader = Downloader.new
+
+          stub_request(:get, 'http://example.com/a').to_return(status: 404, body: 'aaa')
+
+          expect {
+            downloader.load('http://example.com/a', raise_on_http_error: true)
+          }.to raise_error(Downloader::HTTPError)
         end
 
         it 'derives protocol from base_url' do

data/spec/pageflow/chart/refresh_tag_following_downloader_spec.rb

@@ -17,7 +17,7 @@ module Pageflow
           result = ''
 
           allow(downloader).to receive(:load)
-            .with(original_url)
+            .with(original_url, {})
             .and_yield(StringIO.new(chart_html))
 
           refresh_tag_following_downloader.load_following_refresh_tags(original_url) do |file|
@@ -27,6 +27,19 @@ module Pageflow
           expect(result).to eq(chart_html)
         end
 
+        it 'passes raise_on_http_error to downloader' do
+          downloader = double(Downloader).as_null_object
+          refresh_tag_following_downloader = RefreshTagFollowingDownloader.new(downloader)
+
+          original_url = 'http://datawrapper.dwcdn.net/HPKfl/2/'
+
+          expect(downloader).to receive(:load)
+            .with(original_url, raise_on_http_error: true)
+
+          refresh_tag_following_downloader.load_following_refresh_tags(original_url,
+                                                                       raise_on_http_error: true)
+        end
+
         it 'looks for refresh meta tags and loads their url instead' do
           downloader = double(Downloader)
           refresh_tag_following_downloader = RefreshTagFollowingDownloader.new(downloader)
@@ -44,11 +57,11 @@ module Pageflow
           result = ''
 
           allow(downloader).to receive(:load)
-            .with(original_url)
+            .with(original_url, {})
             .and_yield(StringIO.new(redirect_html))
 
           allow(downloader).to receive(:load)
-            .with(target_url)
+            .with(target_url, {})
             .and_yield(StringIO.new(chart_html))
 
           refresh_tag_following_downloader.load_following_refresh_tags(original_url) do |file|
@@ -75,11 +88,11 @@ module Pageflow
           result = ''
 
           allow(downloader).to receive(:load)
-            .with(original_url)
+            .with(original_url, {})
             .and_yield(StringIO.new(redirect_html))
 
           allow(downloader).to receive(:load)
-            .with(target_url)
+            .with(target_url, {})
             .and_yield(StringIO.new(chart_html))
 
           refresh_tag_following_downloader.load_following_refresh_tags(original_url) do |file|
@@ -106,11 +119,11 @@ module Pageflow
           result = ''
 
           allow(downloader).to receive(:load)
-            .with(original_url)
+            .with(original_url, {})
             .and_yield(StringIO.new(redirect_html))
 
           allow(downloader).to receive(:load)
-            .with(target_url)
+            .with(target_url, {})
             .and_yield(StringIO.new(chart_html))
 
           refresh_tag_following_downloader.load_following_refresh_tags(original_url) do |file|
@@ -130,7 +143,7 @@ module Pageflow
             <html><head><meta http-equiv="REFRESH" content="0; url=#{original_url}"></head></html>
           HTML
 
-          allow(downloader).to receive(:load).with(original_url) do |&block|
+          allow(downloader).to receive(:load).with(original_url, {}) do |&block|
             block.call(StringIO.new(redirect_html))
           end
 
@@ -149,7 +162,7 @@ module Pageflow
             <html><head><meta http-equiv="REFRESH" content="something strange"></head></html>
           HTML
 
-          allow(downloader).to receive(:load).with(original_url).and_yield(StringIO.new(redirect_html))
+          allow(downloader).to receive(:load).with(original_url, {}).and_yield(StringIO.new(redirect_html))
 
           expect {
             refresh_tag_following_downloader.load_following_refresh_tags(original_url)
@@ -166,7 +179,7 @@ module Pageflow
             <html><head><meta http-equiv="REFRESH"></head></html>
           HTML
 
-          allow(downloader).to receive(:load).with(original_url).and_yield(StringIO.new(redirect_html))
+          allow(downloader).to receive(:load).with(original_url, {}).and_yield(StringIO.new(redirect_html))
 
           expect {
             refresh_tag_following_downloader.load_following_refresh_tags(original_url)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pageflow-chart
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Codevise Solutions Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-08 00:00:00.000000000 Z
+date: 2019-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pageflow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 14.x
+        version: 15.x
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 14.x
+        version: 15.x
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 14.x
+        version: 15.x
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 14.x
+        version: 15.x
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -180,28 +180,23 @@ files:
 - app/assets/javascripts/pageflow/chart.js
 - app/assets/javascripts/pageflow/chart/asset_urls.js.erb
 - app/assets/javascripts/pageflow/chart/editor.js
-- app/assets/javascripts/pageflow/chart/editor/collections/scraped_sites_collection.js
-- app/assets/javascripts/pageflow/chart/editor/initializers/setup_collections.js
+- app/assets/javascripts/pageflow/chart/editor/config.js
 - app/assets/javascripts/pageflow/chart/editor/models/scraped_site.js
 - app/assets/javascripts/pageflow/chart/editor/templates/datawrapper_ad.jst.ejs
-- app/assets/javascripts/pageflow/chart/editor/templates/scraped_site_status.jst.ejs
-- app/assets/javascripts/pageflow/chart/editor/templates/url_input.jst.ejs
 - app/assets/javascripts/pageflow/chart/editor/views/configuration_editor.js
 - app/assets/javascripts/pageflow/chart/editor/views/datawrapper_ad_view.js
 - app/assets/javascripts/pageflow/chart/editor/views/embedded/iframe_embedded_view.js
 - app/assets/javascripts/pageflow/chart/editor/views/inputs/scraped_url_input_view.js
-- app/assets/javascripts/pageflow/chart/editor/views/scraped_site_status_view.js
 - app/assets/javascripts/pageflow/chart/page_type.js
 - app/assets/stylesheets/pageflow/chart.scss
 - app/assets/stylesheets/pageflow/chart/custom.scss
 - app/assets/stylesheets/pageflow/chart/editor.scss
 - app/assets/stylesheets/pageflow/chart/themes/default.scss
 - app/assets/stylesheets/pageflow/chart/transparent_background.css.erb
-- app/controllers/pageflow/chart/application_controller.rb
-- app/controllers/pageflow/chart/scraped_sites_controller.rb
 - app/helpers/pageflow/chart/scraped_sites_helper.rb
 - app/jobs/pageflow/chart/scrape_site_job.rb
 - app/models/pageflow/chart/scraped_site.rb
+- app/views/pageflow/chart/editor/scraped_sites/_scraped_site.json.jbuilder
 - app/views/pageflow/chart/page.html.erb
 - app/views/pageflow/chart/page_type.json.jbuilder
 - bin/rails-cmd
@@ -210,9 +205,10 @@ files:
 - chart.gemspec
 - config/locales/de.yml
 - config/locales/en.yml
-- config/routes.rb
 - db/migrate/20140417112724_create_pageflow_chart_scraped_sites.rb
 - db/migrate/20160211085234_add_use_custom_theme_to_scraped_sites.rb
+- db/migrate/20190531141820_add_file_attributes_to_scraped_sites.rb
+- db/migrate/20190531145431_insert_file_usages_for_scraped_sites.rb
 - lib/generators/pageflow_chart/install/install_generator.rb
 - lib/pageflow/chart.rb
 - lib/pageflow/chart/configuration.rb
@@ -222,9 +218,13 @@ files:
 - lib/pageflow/chart/refresh_tag_following_downloader.rb
 - lib/pageflow/chart/scraper.rb
 - lib/pageflow/chart/version.rb
-- spec/controllers/pageflow/chart/scraped_sites_controller_spec.rb
 - spec/factories/scraped_sites.rb
+- spec/fixtures/all.css
+- spec/fixtures/all.js
+- spec/fixtures/data.csv
 - spec/fixtures/datawrapper.html
+- spec/fixtures/index.html
+- spec/integration/file_type_spec.rb
 - spec/integration/page_type_spec.rb
 - spec/jobs/pageflow/chart/scrape_site_job_spec.rb
 - spec/models/pageflow/chart/scraped_site_spec.rb
@@ -232,7 +232,6 @@ files:
 - spec/pageflow/chart/downloader_spec.rb
 - spec/pageflow/chart/refresh_tag_following_downloader_spec.rb
 - spec/pageflow/chart/scraper_spec.rb
-- spec/requests/scraping_site_spec.rb
 - spec/spec_helper.rb
 - spec/support/active_job.rb
 - spec/support/factory_bot.rb
@@ -264,9 +263,13 @@ signing_key:
 specification_version: 4
 summary: Pagetype for Embedded Datawrapper Charts
 test_files:
-- spec/controllers/pageflow/chart/scraped_sites_controller_spec.rb
 - spec/factories/scraped_sites.rb
+- spec/fixtures/all.css
+- spec/fixtures/all.js
+- spec/fixtures/data.csv
 - spec/fixtures/datawrapper.html
+- spec/fixtures/index.html
+- spec/integration/file_type_spec.rb
 - spec/integration/page_type_spec.rb
 - spec/jobs/pageflow/chart/scrape_site_job_spec.rb
 - spec/models/pageflow/chart/scraped_site_spec.rb
@@ -274,7 +277,6 @@ test_files:
 - spec/pageflow/chart/downloader_spec.rb
 - spec/pageflow/chart/refresh_tag_following_downloader_spec.rb
 - spec/pageflow/chart/scraper_spec.rb
-- spec/requests/scraping_site_spec.rb
 - spec/spec_helper.rb
 - spec/support/active_job.rb
 - spec/support/factory_bot.rb

data/app/assets/javascripts/pageflow/chart/editor/collections/scraped_sites_collection.js

@@ -1,23 +0,0 @@
-pageflow.chart.ScrapedSitesCollection = Backbone.Collection.extend({
-  model: pageflow.chart.ScrapedSite,
-
-  name: 'scraped_sites',
-
-  getOrFetch: function(id, options) {
-    options = options || {};
-    var model = this.get(id);
-
-    if (model) {
-      if (options.success) {
-        options.success(model);
-      }
-    }
-    else {
-      model = new pageflow.chart.ScrapedSite({id: id});
-      this.add(model);
-      model.fetch(options);
-    }
-
-    return model;
-  }
-});

data/app/assets/javascripts/pageflow/chart/editor/initializers/setup_collections.js

@@ -1 +0,0 @@
-pageflow.chart.scrapedSites = new pageflow.chart.ScrapedSitesCollection();

data/app/assets/javascripts/pageflow/chart/editor/templates/scraped_site_status.jst.ejs

@@ -1,2 +0,0 @@
-<p class="processing state validation pending"><%= I18n.t('pageflow.chart.editor.templates.scraped_site_status.pending') %></p>
-<p class="failed state validation failed"><%= I18n.t('pageflow.chart.editor.templates.scraped_site_status.failed') %></p>

data/app/assets/javascripts/pageflow/chart/editor/templates/url_input.jst.ejs

@@ -1,7 +0,0 @@
-<label>
-  <span class="name"></span>
-  <span class="inline_help"></span>
-</label>
-<input type="text" />
-<div class="validation"></div>
-<div class="status_container"></div>

data/app/assets/javascripts/pageflow/chart/editor/views/scraped_site_status_view.js

@@ -1,18 +0,0 @@
-pageflow.chart.ScrapedSiteStatusView = Backbone.Marionette.ItemView.extend({
-  template: 'pageflow/chart/editor/templates/scraped_site_status',
-  className: 'scraped_site_status',
-
-  modelEvents: {
-    change: 'update'
-  },
-
-  onRender: function() {
-    this.update();
-  },
-
-  update: function() {
-    this.$el.toggleClass('processed', this.model.isProcessed());
-    this.$el.toggleClass('failed', this.model.isFailed());
-    this.$el.toggleClass('processing', this.model.isProcessing());
-  }
-});

data/app/controllers/pageflow/chart/application_controller.rb

@@ -1,6 +0,0 @@
-module Pageflow
-  module Chart
-    class ApplicationController < ActionController::Base
-    end
-  end
-end

data/app/controllers/pageflow/chart/scraped_sites_controller.rb

@@ -1,25 +0,0 @@
-module Pageflow
-  module Chart
-    class ScrapedSitesController < Chart::ApplicationController
-      respond_to :json
-
-      def create
-        scraped_site = ScrapedSite.create!(scraped_site_params)
-        scraped_site.process!
-
-        respond_with(scraped_site)
-      end
-
-      def show
-        scraped_site = ScrapedSite.find(params[:id])
-        respond_with(scraped_site)
-      end
-
-      protected
-
-      def scraped_site_params
-        params.require(:scraped_site).permit(:url)
-      end
-    end
-  end
-end

data/config/routes.rb

@@ -1,3 +0,0 @@
-Pageflow::Chart::Engine.routes.draw do
-  resources :scraped_sites, only: [:create, :show]
-end

data/spec/controllers/pageflow/chart/scraped_sites_controller_spec.rb

@@ -1,35 +0,0 @@
-require 'spec_helper'
-
-module Pageflow
-  module Chart
-    describe ScrapedSitesController do
-      describe '#create' do
-        routes { Pageflow::Chart::Engine.routes }
-
-        it 'responds with success' do
-          post(:create, params: {scraped_site: {url: "http://example.com/chart.html"}}, format: 'json')
-
-          expect(response.status).to eq(201)
-        end
-
-        it 'creates scraped site' do
-          expect {
-            post(:create, params: {scraped_site: {url: "http://example.com/chart.html"}}, format: 'json')
-          }.to change { ScrapedSite.count }
-        end
-      end
-
-      describe '#show' do
-        routes { Pageflow::Chart::Engine.routes }
-
-        it 'responds with success' do
-          scraped_site = create(:scraped_site, state: 'unprocessed')
-
-          get(:show, params: { id: scraped_site.id }, format: 'json')
-
-          expect(response.status).to eq(200)
-        end
-      end
-    end
-  end
-end

data/spec/requests/scraping_site_spec.rb

@@ -1,23 +0,0 @@
-require 'spec_helper'
-
-module Pageflow
-  module Chart
-    describe 'scraping site', perform_jobs: true do
-      before do
-        stub_request(:get, "http://example.com/chart.html")
-          .to_return(:status => 200, :body => File.read('spec/fixtures/datawrapper.html'))
-        stub_request(:get, /js$|css$|csv$/)
-          .to_return(:status => 200, :body => 'file')
-      end
-
-      it 'downloads html and dependencies' do
-        post('/charts/scraped_sites.json', params: {scraped_site: {url: 'http://example.com/chart.html'}})
-
-        expect(ScrapedSite.first.html_file).to be_present
-        expect(ScrapedSite.first.javascript_file).to be_present
-        expect(ScrapedSite.first.stylesheet_file).to be_present
-        expect(ScrapedSite.first.csv_file).to be_present
-      end
-    end
-  end
-end