pageflow-chart 0.1.0

data/lib/pageflow/chart/downloader.rb

@@ -0,0 +1,53 @@
+require 'uri'
+require 'open-uri'
+
+module Pageflow
+  module Chart
+    class Downloader
+      attr_reader :options
+
+      def initialize(options = {})
+        @options = options
+      end
+
+      def load(url)
+        file = open(make_absolute(url))
+
+        begin
+          yield(file)
+        ensure
+          file.close
+        end
+      end
+
+      def load_all(urls, options = {})
+        file = Tempfile.new(['concatenation', options.fetch(:extension, 'txt')])
+        file.binmode
+
+        begin
+          urls.map do |url|
+            load(url) do |source|
+              while data = source.read(16 * 1024)
+                file.write(data)
+              end
+            end
+
+            file.write(options.fetch(:separator, "\n"))
+          end
+
+          file.rewind
+          yield(file)
+        ensure
+          file.close
+          file.unlink
+        end
+      end
+
+      private
+
+      def make_absolute(url)
+        options[:base_url] ? URI.join(options[:base_url], url) : URI.parse(url)
+      end
+    end
+  end
+end

data/lib/pageflow/chart/engine.rb

@@ -0,0 +1,17 @@
+module Pageflow
+  module Chart
+    class Engine < Rails::Engine
+      isolate_namespace Pageflow::Chart
+
+      config.autoload_paths << File.join(config.root, 'lib')
+      config.assets.precompile += ['pageflow/chart/custom.css']
+
+      config.generators do |g|
+        g.test_framework :rspec,:fixture => false
+        g.fixture_replacement :factory_girl, :dir => 'spec/factories'
+        g.assets false
+        g.helper false
+      end
+    end
+  end
+end

data/lib/pageflow/chart/page_type.rb

@@ -0,0 +1,15 @@
+module Pageflow
+  module Chart
+    class PageType < Pageflow::PageType
+      name 'chart'
+
+      def view_helpers
+        [ScrapedSitesHelper]
+      end
+
+      def json_seed_template
+        'pageflow/chart/page_type.json.jbuilder'
+      end
+    end
+  end
+end

data/lib/pageflow/chart/scraper.rb

@@ -0,0 +1,107 @@
+require 'nokogiri'
+
+module Pageflow
+  module Chart
+    class Scraper
+      attr_reader :document, :options, :javascript_urls, :stylesheet_urls
+
+      def initialize(html, options = {})
+        @document = Nokogiri::HTML(html)
+        @options = options
+
+        parse
+        rewrite
+      end
+
+      def html
+        document.to_s
+      end
+
+      def csv_url
+      end
+
+      private
+
+      def parse
+        parse_javascript_urls
+        parse_stylesheet_urls
+      end
+
+      def parse_javascript_urls
+        @javascript_urls = filtered_script_tags_in_head.map do |tag|
+          tag[:src]
+        end
+      end
+
+      def parse_stylesheet_urls
+        @stylesheet_urls = css_link_tags.map do |tag|
+          tag[:href]
+        end
+      end
+
+      def rewrite
+        filter_inline_scripts
+        filter_by_selectors
+        combine_script_tags_in_head
+        combine_css_link_tags
+      end
+
+      def filter_inline_scripts
+        document.css('body script').each do |tag|
+          if blacklisted_inline_script?(tag)
+            tag.remove
+          end
+        end
+      end
+
+      def blacklisted_inline_script?(tag)
+        options.fetch(:inline_script_blacklist, []).any? do |r|
+          tag.content =~ r
+        end
+      end
+
+      def filter_by_selectors
+        options.fetch(:selector_blacklist, []).each do |selector|
+          document.css(selector).each(&:remove)
+        end
+      end
+
+      def combine_script_tags_in_head
+        script_tags_in_head.each(&:remove)
+
+        all_script_tag = Nokogiri::XML::Node.new('script', document)
+        all_script_tag[:src] = 'all.js'
+        all_script_tag[:type] = 'text/javascript'
+        document.at_css('head') << all_script_tag
+      end
+
+      def combine_css_link_tags
+        css_link_tags.each(&:remove)
+
+        all_css_link_tag = Nokogiri::XML::Node.new('link', document)
+        all_css_link_tag[:href] = 'all.css'
+        all_css_link_tag[:type] = 'text/css'
+        all_css_link_tag[:rel] = 'stylesheet'
+        document.at_css('head') << all_css_link_tag
+      end
+
+      def filtered_script_tags_in_head
+        script_tags_in_head.reject do |tag|
+          options.fetch(:head_script_blacklist, []).any? do |regexp|
+            tag[:src] =~ regexp
+          end
+        end
+      end
+
+      def script_tags_in_head
+        document.css('head script[src]')
+      end
+
+      def css_link_tags
+        document.css('head link').find_all do |tag|
+          tag[:type] == 'text/css'
+        end
+      end
+    end
+  end
+end

data/spec/controllers/pageflow/chart/scraped_sites_controller_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+module Pageflow
+  module Chart
+    describe ScrapedSitesController do
+      describe '#create' do
+        routes { Pageflow::Chart::Engine.routes }
+
+        it 'responds with success' do
+          post(:create, scraped_site: {url: "http://example.com/chart.html"}, format: 'json')
+
+          expect(response.status).to eq(201)
+        end
+
+        it 'creates scraped site' do
+          expect {
+            post(:create, scraped_site: {url: "http://example.com/chart.html"}, format: 'json')
+          }.to change { ScrapedSite.count }
+        end
+      end
+
+      describe '#show' do
+        routes { Pageflow::Chart::Engine.routes }
+
+        it 'responds with success' do
+          scraped_site = create(:scraped_site, state: 'unprocessed')
+
+          get(:show, id: scraped_site.id, format: 'json')
+
+          expect(response.status).to eq(200)
+        end
+      end
+    end
+  end
+end

data/spec/dummy/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Dummy::Application.load_tasks

data/spec/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/sstephenson/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/spec/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,13 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the top of the
+ * compiled file, but it's generally better to create a new file per style scope.
+ *
+ *= require_self
+ *= require_tree .
+ */

data/spec/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,5 @@
+class ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+end

data/spec/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/spec/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Dummy</title>
+  <%= stylesheet_link_tag    "application", media: "all", "data-turbolinks-track" => true %>
+  <%= javascript_include_tag "application", "data-turbolinks-track" => true %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/spec/dummy/bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+load Gem.bin_path('bundler', 'bundle')

data/spec/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../../config/application',  __FILE__)
+require_relative '../config/boot'
+require 'rails/commands'

data/spec/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../config/boot'
+require 'rake'
+Rake.application.run

data/spec/dummy/config.ru

@@ -0,0 +1,4 @@
+# This file is used by Rack-based servers to start the application.
+
+require ::File.expand_path('../config/environment',  __FILE__)
+run Rails.application

data/spec/dummy/config/application.rb

@@ -0,0 +1,22 @@
+require File.expand_path('../boot', __FILE__)
+
+require 'rails/all'
+
+Bundler.require(*Rails.groups)
+require "pageflow/chart"
+
+module Dummy
+  class Application < Rails::Application
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration should go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded.
+
+    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
+    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
+    # config.time_zone = 'Central Time (US & Canada)'
+
+    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
+    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
+    # config.i18n.default_locale = :de
+  end
+end

data/spec/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__)
+
+require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])
+$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)

data/spec/dummy/config/database.yml

@@ -0,0 +1,25 @@
+# SQLite version 3.x
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem 'sqlite3'
+development:
+  adapter: sqlite3
+  database: db/development.sqlite3
+  pool: 5
+  timeout: 5000
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  adapter: sqlite3
+  database: db/test.sqlite3
+  pool: 5
+  timeout: 5000
+
+production:
+  adapter: sqlite3
+  database: db/production.sqlite3
+  pool: 5
+  timeout: 5000

data/spec/dummy/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the Rails application.
+require File.expand_path('../application', __FILE__)
+
+# Initialize the Rails application.
+Dummy::Application.initialize!

data/spec/dummy/config/environments/development.rb

@@ -0,0 +1,29 @@
+Dummy::Application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # In the development environment your application's code is reloaded on
+  # every request. This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = false
+
+  # Do not eager load code on boot.
+  config.eager_load = false
+
+  # Show full error reports and disable caching.
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Don't care if the mailer can't send.
+  # config.action_mailer.raise_delivery_errors = false
+
+  # Print deprecation notices to the Rails logger.
+  config.active_support.deprecation = :log
+
+  # Raise an error on page load if there are pending migrations
+  config.active_record.migration_error = :page_load
+
+  # Debug mode disables concatenation and preprocessing of assets.
+  # This option may cause significant delays in view rendering with a large
+  # number of complex assets.
+  config.assets.debug = true
+end