RubyGems - pageflow-chart - Versions diffs - 0.1.0 - Mend

pageflow-chart 0.1.0

Files changed (100) hide show

checksums.yaml +15 -0
data/.gitignore +21 -0
data/CHANGELOG.md +7 -0
data/Gemfile +10 -0
data/README.md +91 -0
data/Rakefile +20 -0
data/app/assets/images/pageflow/chart/fs_close_sprite.png +0 -0
data/app/assets/images/pageflow/chart_pictogram.png +0 -0
data/app/assets/images/pageflow/chart_pictogram_small.png +0 -0
data/app/assets/images/pageflow/chart_sprite.png +0 -0
data/app/assets/images/pageflow/ov-chart.png +0 -0
data/app/assets/javascripts/pageflow/chart.js +5 -0
data/app/assets/javascripts/pageflow/chart/asset_urls.js.erb +3 -0
data/app/assets/javascripts/pageflow/chart/editor.js +9 -0
data/app/assets/javascripts/pageflow/chart/editor/collections/scraped_sites_collection.js +23 -0
data/app/assets/javascripts/pageflow/chart/editor/initializers/setup_collections.js +1 -0
data/app/assets/javascripts/pageflow/chart/editor/models/scraped_site.js +55 -0
data/app/assets/javascripts/pageflow/chart/editor/templates/scraped_site_status.jst.ejs +2 -0
data/app/assets/javascripts/pageflow/chart/editor/templates/url_input.jst.ejs +7 -0
data/app/assets/javascripts/pageflow/chart/editor/views/configuration_editor.js +26 -0
data/app/assets/javascripts/pageflow/chart/editor/views/embedded/iframe_embedded_view.js +47 -0
data/app/assets/javascripts/pageflow/chart/editor/views/inputs/scraped_url_input_view.js +49 -0
data/app/assets/javascripts/pageflow/chart/editor/views/scraped_site_status_view.js +18 -0
data/app/assets/javascripts/pageflow/chart/page_type.js +152 -0
data/app/assets/stylesheets/pageflow/chart.css.scss +130 -0
data/app/assets/stylesheets/pageflow/chart/custom.css.scss +209 -0
data/app/assets/stylesheets/pageflow/chart/editor.css.scss +17 -0
data/app/assets/stylesheets/pageflow/chart/themes/default.css.scss +10 -0
data/app/controllers/pageflow/chart/application_controller.rb +6 -0
data/app/controllers/pageflow/chart/scraped_sites_controller.rb +25 -0
data/app/helpers/pageflow/chart/scraped_sites_helper.rb +13 -0
data/app/jobs/pageflow/chart/scrape_site_job.rb +59 -0
data/app/models/pageflow/chart/scraped_site.rb +51 -0
data/app/views/pageflow/chart/page.html +41 -0
data/app/views/pageflow/chart/page_type.json.jbuilder +2 -0
data/bin/rails +8 -0
data/chart.gemspec +30 -0
data/config/locales/de.yml +40 -0
data/config/locales/en.yml +22 -0
data/config/routes.rb +3 -0
data/db/migrate/20140417112724_create_pageflow_chart_scraped_sites.rb +14 -0
data/lib/pageflow/chart.rb +21 -0
data/lib/pageflow/chart/configuration.rb +63 -0
data/lib/pageflow/chart/downloader.rb +53 -0
data/lib/pageflow/chart/engine.rb +17 -0
data/lib/pageflow/chart/page_type.rb +15 -0
data/lib/pageflow/chart/scraper.rb +107 -0
data/spec/controllers/pageflow/chart/scraped_sites_controller_spec.rb +35 -0
data/spec/dummy/README.rdoc +28 -0
data/spec/dummy/Rakefile +6 -0
data/spec/dummy/app/assets/images/.keep +0 -0
data/spec/dummy/app/assets/javascripts/application.js +13 -0
data/spec/dummy/app/assets/stylesheets/application.css +13 -0
data/spec/dummy/app/controllers/application_controller.rb +5 -0
data/spec/dummy/app/controllers/concerns/.keep +0 -0
data/spec/dummy/app/helpers/application_helper.rb +2 -0
data/spec/dummy/app/mailers/.keep +0 -0
data/spec/dummy/app/models/.keep +0 -0
data/spec/dummy/app/models/concerns/.keep +0 -0
data/spec/dummy/app/views/layouts/application.html.erb +14 -0
data/spec/dummy/bin/bundle +3 -0
data/spec/dummy/bin/rails +4 -0
data/spec/dummy/bin/rake +4 -0
data/spec/dummy/config.ru +4 -0
data/spec/dummy/config/application.rb +22 -0
data/spec/dummy/config/boot.rb +5 -0
data/spec/dummy/config/database.yml +25 -0
data/spec/dummy/config/environment.rb +5 -0
data/spec/dummy/config/environments/development.rb +29 -0
data/spec/dummy/config/environments/production.rb +80 -0
data/spec/dummy/config/environments/test.rb +36 -0
data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
data/spec/dummy/config/initializers/inflections.rb +16 -0
data/spec/dummy/config/initializers/mime_types.rb +5 -0
data/spec/dummy/config/initializers/secret_token.rb +12 -0
data/spec/dummy/config/initializers/session_store.rb +3 -0
data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
data/spec/dummy/config/locales/en.yml +23 -0
data/spec/dummy/config/routes.rb +4 -0
data/spec/dummy/db/schema.rb +39 -0
data/spec/dummy/lib/assets/.keep +0 -0
data/spec/dummy/public/404.html +58 -0
data/spec/dummy/public/422.html +58 -0
data/spec/dummy/public/500.html +57 -0
data/spec/dummy/public/favicon.ico +0 -0
data/spec/factories/scraped_sites.rb +5 -0
data/spec/fixtures/datawrapper.html +121 -0
data/spec/jobs/pageflow/chart/scrape_site_job_spec.rb +22 -0
data/spec/models/pageflow/chart/scraped_site_spec.rb +19 -0
data/spec/pageflow/chart/downloader_spec.rb +90 -0
data/spec/pageflow/chart/scraper_spec.rb +179 -0
data/spec/requests/scraping_site_spec.rb +23 -0
data/spec/spec_helper.rb +20 -0
data/spec/support/factory_girl.rb +5 -0
data/spec/support/html_fragment.rb +13 -0
data/spec/support/paperclip.rb +11 -0
data/spec/support/resque.rb +20 -0
data/spec/support/webmock.rb +11 -0
metadata +363 -0

data/spec/dummy/public/favicon.ico ADDED Viewed

File without changes

data/spec/factories/scraped_sites.rb ADDED Viewed

@@ -0,0 +1,5 @@
+FactoryGirl.define do
+  factory :scraped_site, :class => 'Pageflow::Chart::ScrapedSite' do
+    url "MyString"
+  end
+end

data/spec/fixtures/datawrapper.html ADDED Viewed

@@ -0,0 +1,121 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge"/>
+    <title>[ Insert title here ]</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=0.95, maximum-scale=0.95, user-scalable=0" />
+    <!--[if lt IE 7]>
+    <meta http-equiv="refresh" content="0;URL='static.html'">
+    <![endif]-->
+    <!--[if lt IE 9]>
+    <script src="https://datawrapper.de//static/vendor/json-js/json2.min.js"></script>
+    <script type="text/javascript">
+        window.__ltie9 = true;
+    </script>
+    <![endif]-->
+    <link rel="stylesheet" type="text/css" href="aLrJt.all.css"></link>
+    <script type="text/javascript">
+    window.__locale = 'en';
+    </script>
+    <script type="text/javascript" src="//assets-datawrapper.s3.amazonaws.com/globalize.min.js"></script>
+    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/underscore.js/1.5.2/underscore-min.js"></script>
+    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
+    <script type="text/javascript" src="//assets-datawrapper.s3.amazonaws.com/vendor/d3-light/3.1.7/d3-light.min.js"></script>
+    <script type="text/javascript" src="//assets-datawrapper.s3.amazonaws.com/vendor/chroma-js/0.5.4/chroma.min.js"></script>
+    <script type="text/javascript" src="//assets-datawrapper.s3.amazonaws.com/vendor/raphael-js/2.1.2/raphael-min.js"></script>
+    <script type="text/javascript" src="/lib/vis/line-chart-5fb380e5bc3650f9e9661eb780af2d52.min.js"></script>
+    <script type="text/javascript" src="/lib/theme/default-5baee189e1a2f0bf680f4910cc958098.min.js"></script>
+    <script type="text/javascript" src="/lib/chart-206e22793fae102dcb0bb68278f61581.min.js"></script>
+</head>
+<body class="chart">
+<div id="header">
+        <h1 >
+        <span class="chart-title">[ Insert title here ]</span>
+    </h1>
+        <p class="chart-intro hidden"></p>
+        <a href="http://cf.datawrapper.de/aLrJt/1/?fs=1" target="_blank" class="fs-btn top-right" title="Fullscreen" data-toggle="fullscreen"></a>
+    </div>
+<div id="chart"></div>
+<div id="footer">
+            <div class="footer-left">
+             Created with <a href="https://datawrapper.de/" target="_blank" class="lgo">Datawrapper</a>
+        </div>
+        <div class="footer-right">
+            <div class="toggleable-panel">
+                    <span data-src="Source: " class="source-block">Source:                     <a target="_blank" class="source" href="http://www.census.gov/foreign-trade/balance/c4120.html">US Census Bureau</a></span>
+                                <a href="data">Get the data</a>
+            </div>
+            <a class="toggle-panel">About</a>
+        </div>
+        <div class="toggleable-panel-overlay"></div>
+    </div>
+    <div class="noscript" style="position: absolute;top: 0;left: 0;z-index: -10;">
+        <a href="static.html"><img src="nojs.png" alt="Please activate JavaScript to see the interactive chart." /></a>
+    </div>
+    <div class="noscript" style="height:95%;position:absolute;top:0;width:98%;z-index:10;overflow:hidden">
+        <div style="position:relative;top:50%;text-align:center;font-size:18px;padding:0 20px"><a style="color: rgba(0,0,0,.2); text-shadow: 0 0 7px #fff, 0 0 7px #fff, 0 0 7px #fff, 0 0 7px #fff, 0 0 7px #fff, 0 0 7px #fff, 0 0 7px #fff, 0 0 7px #fff, 0 0 7px #fff, 0 0 7px #fff, 0 0 7px #fff; text-decoration: none;" href="static.html" >Please activate JavaScript to see the interactive chart.</a></div>
+    </div>
+    <script type="text/javascript">
+    if (document.getElementsByTagName('script').length > 0) $('body').addClass('js');
+    __dw.init({
+        chartJSON: JSON.parse('{\"id\":\"aLrJt\",\"title\":\"[ Insert title here ]\",\"theme\":\"default\",\"createdAt\":\"2014-04-15 19:14:01\",\"lastModifiedAt\":\"2014-04-15 19:15:26\",\"type\":\"line-chart\",\"metadata\":{\"data\":{\"transpose\":false,\"vertical-header\":true,\"horizontal-header\":true},\"visualize\":{\"highlighted-series\":[],\"highlighted-values\":[],\"direct-labeling\":false,\"legend-position\":\"right\",\"show-grid\":false,\"fill-between\":false,\"line-mode\":\"straight\",\"invert-y-axis\":false,\"scale-y1\":\"linear\",\"user-change-scale\":false,\"sort-values\":false},\"describe\":{\"source-name\":\"US Census Bureau\",\"source-url\":\"http:\\/\\/www.census.gov\\/foreign-trade\\/balance\\/c4120.html\",\"number-format\":\"n1\",\"number-divisor\":\"3\",\"number-append\":\" Billion USD\",\"number-prepend\":\"\",\"intro\":\"\"},\"publish\":{\"embed-width\":600,\"embed-height\":400,\"background\":\"#ffffff\",\"text\":\"#000000\"}},\"authorId\":9505,\"showInGallery\":false,\"language\":\"en\",\"guestSession\":null,\"lastEditStep\":3,\"publishedAt\":null,\"publicUrl\":\"http:\\/\\/cf.datawrapper.de\\/aLrJt\\/1\\/\",\"publicVersion\":1,\"organizationId\":null,\"forkedFrom\":null,\"author\":{\"id\":9505,\"email\":\"\",\"name\":\"tf\",\"website\":\"stderr.timfischbach.de\",\"socialmedia\":\"https:\\/\\/github.com\\/tf\"}}'),
+        chartLocale: 'en',
+        themeId: 'default',
+        visId: 'line-chart',
+        visJSON: JSON.parse('\x7B\x22title\x22\x3A\x22Line\x20Chart\x22,\x22id\x22\x3A\x22line\x2Dchart\x22,\x22extends\x22\x3A\x22raphael\x2Dchart\x22,\x22dimensions\x22\x3A2,\x22order\x22\x3A40,\x22axes\x22\x3A\x7B\x22x\x22\x3A\x7B\x22accepts\x22\x3A\x5B\x22text\x22,\x22date\x22\x5D\x7D,\x22y1\x22\x3A\x7B\x22accepts\x22\x3A\x5B\x22number\x22\x5D,\x22multiple\x22\x3Atrue\x7D,\x22y2\x22\x3A\x7B\x22accepts\x22\x3A\x5B\x22number\x22\x5D,\x22multiple\x22\x3Atrue,\x22optional\x22\x3Atrue\x7D\x7D,\x22options\x22\x3A\x7B\x22base\x2Dcolor\x22\x3A\x7B\x22type\x22\x3A\x22base\x2Dcolor\x22,\x22label\x22\x3A\x22Base\x20color\x22\x7D,\x22sep\x2Dlabeling\x22\x3A\x7B\x22type\x22\x3A\x22separator\x22,\x22label\x22\x3A\x22Customize\x20labeling\x22,\x22depends\x2Don\x22\x3A\x7B\x22chart.min_columns\x5By1\x5D\x22\x3A2\x7D\x7D,\x22direct\x2Dlabeling\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22label\x22\x3A\x22Direct\x20labeling\x22,\x22default\x22\x3Afalse,\x22depends\x2Don\x22\x3A\x7B\x22chart.min_columns\x5By1\x5D\x22\x3A2,\x22chart.max_columns\x5By2\x5D\x22\x3A0\x7D,\x22help\x22\x3A\x22Show\x20the\x20labels\x20right\x20nearby\x20the\x20line\x20ends\x20instead\x20of\x20a\x20separate\x20legend\x22\x7D,\x22legend\x2Dposition\x22\x3A\x7B\x22type\x22\x3A\x22radio\x2Dleft\x22,\x22label\x22\x3A\x22Legend\x20position\x22,\x22default\x22\x3A\x22right\x22,\x22depends\x2Don\x22\x3A\x7B\x22direct\x2Dlabeling\x22\x3Afalse,\x22chart.min_columns\x5By1\x5D\x22\x3A2\x7D,\x22options\x22\x3A\x5B\x7B\x22value\x22\x3A\x22right\x22,\x22label\x22\x3A\x22right\x22\x7D,\x7B\x22value\x22\x3A\x22top\x22,\x22label\x22\x3A\x22top\x22\x7D,\x7B\x22value\x22\x3A\x22inside\x22,\x22label\x22\x3A\x22inside\x20left\x22\x7D,\x7B\x22value\x22\x3A\x22inside\x2Dright\x22,\x22label\x22\x3A\x22inside\x20right\x22\x7D\x5D\x7D,\x22sep\x2Dlines\x22\x3A\x7B\x22type\x22\x3A\x22separator\x22,\x22label\x22\x3A\x22Customize\x20lines\x22\x7D,\x22force\x2Dbanking\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22hidden\x22\x3Atrue,\x22label\x22\x3A\x22Bank\x20the\x20lines\x20to\x2045\x20degrees\x22\x7D,\x22show\x2Dgrid\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22hidden\x22\x3Atrue,\x22label\x22\x3A\x22Show\x20grid\x22,\x22default\x22\x3Afalse\x7D,\x22connect\x2Dmissing\x2Dvalues\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22label\x22\x3A\x22Connect\x20lines\x20between\x20missing\x20values\x22\x7D,\x22fill\x2Dbetween\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22label\x22\x3A\x22Fill\x20area\x20between\x20lines\x22,\x22default\x22\x3Afalse,\x22depends\x2Don\x22\x3A\x7B\x22chart.min_columns\x5By1\x5D\x22\x3A2,\x22chart.max_columns\x5By1\x5D\x22\x3A2,\x22chart.max_columns\x5By2\x5D\x22\x3A0\x7D\x7D,\x22fill\x2Dbelow\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22label\x22\x3A\x22Fill\x20area\x20below\x20line\x22,\x22defaut\x22\x3Afalse,\x22depends\x2Don\x22\x3A\x7B\x22chart.max_columns\x5By1\x5D\x22\x3A1,\x22chart.max_columns\x5By2\x5D\x22\x3A0\x7D\x7D,\x22line\x2Dmode\x22\x3A\x7B\x22type\x22\x3A\x22radio\x2Dleft\x22,\x22label\x22\x3A\x22Line\x20interpolation\x22,\x22options\x22\x3A\x5B\x7B\x22label\x22\x3A\x22Straight\x22,\x22value\x22\x3A\x22straight\x22\x7D,\x7B\x22label\x22\x3A\x22Curved\x22,\x22value\x22\x3A\x22curved\x22\x7D,\x7B\x22label\x22\x3A\x22Stepped\x22,\x22value\x22\x3A\x22stepped\x22\x7D\x5D,\x22default\x22\x3A\x22straight\x22\x7D,\x22sep\x2Dy\x2Daxis\x22\x3A\x7B\x22type\x22\x3A\x22separator\x22,\x22label\x22\x3A\x22Customize\x20y\x2DAxis\x22\x7D,\x22baseline\x2Dzero\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22label\x22\x3A\x22Extend\x20to\x20zero\x22\x7D,\x22extend\x2Drange\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22label\x22\x3A\x22Extend\x20to\x20nice\x20ticks\x22,\x22help\x22\x3A\x22Extend\x20the\x20y\x2Daxis\x20range\x20to\x20nice,\x20rounded\x20values\x20instead\x20of\x20the\x20default\x20range\x20from\x20the\x20minimum\x20to\x20maximum\x20value.\x22\x7D,\x22invert\x2Dy\x2Daxis\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22label\x22\x3A\x22Invert\x20direction\x22,\x22default\x22\x3Afalse\x7D,\x22scale\x2Dy1\x22\x3A\x7B\x22type\x22\x3A\x22radio\x2Dleft\x22,\x22label\x22\x3A\x22Scale\x20\x28y\x2Daxis\x29\x22,\x22options\x22\x3A\x5B\x7B\x22label\x22\x3A\x22linear\x22,\x22value\x22\x3A\x22linear\x22\x7D,\x7B\x22label\x22\x3A\x22logarithmic\x22,\x22value\x22\x3A\x22log\x22\x7D\x5D,\x22default\x22\x3A\x22linear\x22,\x22depends\x2Don\x22\x3A\x7B\x22chart.min_value\x5By1\x5D\x22\x3A\x22\x3E0\x22,\x22chart.magnitude_range\x5By1\x5D\x22\x3A\x22\x3E3\x22\x7D\x7D,\x22user\x2Dchange\x2Dscale\x22\x3A\x7B\x22type\x22\x3A\x22checkbox\x22,\x22label\x22\x3A\x22Let\x20user\x20change\x20scale\x22,\x22default\x22\x3Afalse,\x22depends\x2Don\x22\x3A\x7B\x22chart.min_value\x5By1\x5D\x22\x3A\x22\x3E0\x22,\x22chart.magnitude_range\x5By1\x5D\x22\x3A\x22\x3E3\x22\x7D\x7D\x7D,\x22locale\x22\x3A\x7B\x22tooManyLinesToLabel\x22\x3A\x22Your\x20chart\x20contains\x20\x3Cb\x3Emore\x20lines\x20than\x20we\x20can\x20label\x3C\x5C\x2Fb\x3E,\x20so\x20automatic\x20labeling\x20is\x20turned\x20off.\x20To\x20fix\x20this\x20\x3Cul\x3E\x3Cli\x3Efilter\x20some\x20columns\x20in\x20the\x20data\x20table\x20in\x20the\x20previous\x20step,\x20or\x3C\x5C\x2Fli\x3E\x3Cli\x3Euse\x20direct\x20labeling\x20and\x20the\x20highlight\x20feature\x20to\x20label\x20the\x20lines\x20that\x20are\x20important\x20to\x20your\x20story.\x3C\x5C\x2Fli\x3E\x3C\x5C\x2Ful\x3E\x22,\x22useLogarithmicScale\x22\x3A\x22Use\x20logarithmic\x20scale\x22,\x22couldNotParseAllDates\x22\x3A\x22Some\x20of\x20the\x20\x3Cb\x3Edates\x20in\x20your\x20x\x2Daxis\x20could\x20not\x20be\x20parsed\x3C\x5C\x2Fb\x3E,\x20hence\x20the\x20line\x20chart\x20cannot\x20display\x20a\x20proper\x20date\x20axis.\x20To\x20fix\x20this\x3Cul\x3E\x3Cli\x3Ereturn\x20to\x20the\x20previous\x20step\x20and\x20clean\x20your\x20date\x20column.\x3C\x5C\x2Fli\x3E\x3Cli\x3E\x3Ca\x20href\x3D\x27http\x3A\x5C\x2F\x5C\x2Fblog.datawrapper.de\x5C\x2F2013\x5C\x2Fcleaning\x2Dyour\x2Ddata\x2Din\x2Ddatawrapper\x5C\x2F\x27\x3ERead\x20more\x20about\x20how\x20to\x20do\x20this.\x3C\x5C\x2Fa\x3E\x3C\x5C\x2Fli\x3E\x3C\x5C\x2Ful\x3E\x22\x7D,\x22annotations\x22\x3A\x5B\x7B\x22type\x22\x3A\x22axis\x2Drange\x22,\x22axis\x22\x3A\x22x\x22\x7D,\x7B\x22type\x22\x3A\x22axis\x2Dpoint\x22,\x22axis\x22\x3A\x22x\x22\x7D,\x7B\x22type\x22\x3A\x22axis\x2Drange\x22,\x22axis\x22\x3A\x22y\x22\x7D,\x7B\x22type\x22\x3A\x22axis\x2Dpoint\x22,\x22axis\x22\x3A\x22y\x22\x7D,\x7B\x22type\x22\x3A\x22data\x2Dpoint\x22\x7D\x5D,\x22__static_path\x22\x3A\x22assets\x5C\x2Fplugins\x5C\x2Fvisualization\x2Dline\x2Dchart\x5C\x2F\x22,\x22version\x22\x3A\x221.5.0\x22,\x22icon\x22\x3A\x22\x3C\x3Fxml\x20version\x3D\x5C\x221.0\x5C\x22\x20encoding\x3D\x5C\x22utf\x2D8\x5C\x22\x3F\x3E\x5Cn\x3C\x21\x2D\x2D\x20Generator\x3A\x20Adobe\x20Illustrator\x2016.2.1,\x20SVG\x20Export\x20Plug\x2DIn\x20.\x20SVG\x20Version\x3A\x206.00\x20Build\x200\x29\x20\x20\x2D\x2D\x3E\x5Cn\x3C\x21DOCTYPE\x20svg\x20PUBLIC\x20\x5C\x22\x2D\x5C\x2F\x5C\x2FW3C\x5C\x2F\x5C\x2FDTD\x20SVG\x201.1\x20Basic\x5C\x2F\x5C\x2FEN\x5C\x22\x20\x5C\x22http\x3A\x5C\x2F\x5C\x2Fwww.w3.org\x5C\x2FGraphics\x5C\x2FSVG\x5C\x2F1.1\x5C\x2FDTD\x5C\x2Fsvg11\x2Dbasic.dtd\x5C\x22\x3E\x5Cn\x3Csvg\x20version\x3D\x5C\x221.1\x5C\x22\x20baseProfile\x3D\x5C\x22basic\x5C\x22\x20id\x3D\x5C\x22Ebene_1\x5C\x22\x5Cn\x5Ct\x20xmlns\x3D\x5C\x22http\x3A\x5C\x2F\x5C\x2Fwww.w3.org\x5C\x2F2000\x5C\x2Fsvg\x5C\x22\x20xmlns\x3Axlink\x3D\x5C\x22http\x3A\x5C\x2F\x5C\x2Fwww.w3.org\x5C\x2F1999\x5C\x2Fxlink\x5C\x22\x20x\x3D\x5C\x220px\x5C\x22\x20y\x3D\x5C\x220px\x5C\x22\x20width\x3D\x5C\x22100px\x5C\x22\x20height\x3D\x5C\x22100px\x5C\x22\x5Cn\x5Ct\x20viewBox\x3D\x5C\x220\x200\x20100\x20100\x5C\x22\x20xml\x3Aspace\x3D\x5C\x22preserve\x5C\x22\x3E\x5Cn\x3Cg\x3E\x5Cn\x5Ct\x3Cline\x20fill\x3D\x5C\x22none\x5C\x22\x20stroke\x3D\x5C\x22\x23000000\x5C\x22\x20stroke\x2Dwidth\x3D\x5C\x220.5\x5C\x22\x20stroke\x2Dmiterlimit\x3D\x5C\x2210\x5C\x22\x20x1\x3D\x5C\x229\x5C\x22\x20y1\x3D\x5C\x2279.5\x5C\x22\x20x2\x3D\x5C\x2291\x5C\x22\x20y2\x3D\x5C\x2279.5\x5C\x22\x5C\x2F\x3E\x5Cn\x5Ct\x3Cline\x20fill\x3D\x5C\x22none\x5C\x22\x20stroke\x3D\x5C\x22\x23000000\x5C\x22\x20stroke\x2Dwidth\x3D\x5C\x220.5\x5C\x22\x20stroke\x2Dmiterlimit\x3D\x5C\x2210\x5C\x22\x20x1\x3D\x5C\x229\x5C\x22\x20y1\x3D\x5C\x2259.5\x5C\x22\x20x2\x3D\x5C\x2291\x5C\x22\x20y2\x3D\x5C\x2259.5\x5C\x22\x5C\x2F\x3E\x5Cn\x5Ct\x3Cline\x20fill\x3D\x5C\x22none\x5C\x22\x20stroke\x3D\x5C\x22\x23000000\x5C\x22\x20stroke\x2Dwidth\x3D\x5C\x220.5\x5C\x22\x20stroke\x2Dmiterlimit\x3D\x5C\x2210\x5C\x22\x20x1\x3D\x5C\x229\x5C\x22\x20y1\x3D\x5C\x2239.5\x5C\x22\x20x2\x3D\x5C\x2291\x5C\x22\x20y2\x3D\x5C\x2239.5\x5C\x22\x5C\x2F\x3E\x5Cn\x5Ct\x3Cline\x20fill\x3D\x5C\x22none\x5C\x22\x20stroke\x3D\x5C\x22\x23000000\x5C\x22\x20stroke\x2Dwidth\x3D\x5C\x220.5\x5C\x22\x20stroke\x2Dmiterlimit\x3D\x5C\x2210\x5C\x22\x20x1\x3D\x5C\x229\x5C\x22\x20y1\x3D\x5C\x2219.5\x5C\x22\x20x2\x3D\x5C\x2291\x5C\x22\x20y2\x3D\x5C\x2219.5\x5C\x22\x5C\x2F\x3E\x5Cn\x3C\x5C\x2Fg\x3E\x5Cn\x3Cline\x20fill\x3D\x5C\x22none\x5C\x22\x20stroke\x3D\x5C\x22\x23000000\x5C\x22\x20stroke\x2Dwidth\x3D\x5C\x224\x5C\x22\x20stroke\x2Dlinecap\x3D\x5C\x22round\x5C\x22\x20stroke\x2Dmiterlimit\x3D\x5C\x2210\x5C\x22\x20x1\x3D\x5C\x2210\x5C\x22\x20y1\x3D\x5C\x2275.814\x5C\x22\x20x2\x3D\x5C\x2235.937\x5C\x22\x20y2\x3D\x5C\x2242.674\x5C\x22\x5C\x2F\x3E\x5Cn\x3Cline\x20fill\x3D\x5C\x22none\x5C\x22\x20stroke\x3D\x5C\x22\x23000000\x5C\x22\x20stroke\x2Dwidth\x3D\x5C\x224\x5C\x22\x20stroke\x2Dlinecap\x3D\x5C\x22round\x5C\x22\x20stroke\x2Dmiterlimit\x3D\x5C\x2210\x5C\x22\x20x1\x3D\x5C\x2235.937\x5C\x22\x20y1\x3D\x5C\x2242.674\x5C\x22\x20x2\x3D\x5C\x2262.719\x5C\x22\x20y2\x3D\x5C\x2255.584\x5C\x22\x5C\x2F\x3E\x5Cn\x3Cline\x20fill\x3D\x5C\x22none\x5C\x22\x20stroke\x3D\x5C\x22\x23000000\x5C\x22\x20stroke\x2Dwidth\x3D\x5C\x224\x5C\x22\x20stroke\x2Dlinecap\x3D\x5C\x22round\x5C\x22\x20stroke\x2Dmiterlimit\x3D\x5C\x2210\x5C\x22\x20x1\x3D\x5C\x2262.719\x5C\x22\x20y1\x3D\x5C\x2255.584\x5C\x22\x20x2\x3D\x5C\x2290.562\x5C\x22\x20y2\x3D\x5C\x2223.755\x5C\x22\x5C\x2F\x3E\x5Cn\x3C\x5C\x2Fsvg\x3E\x5Cn\x22,\x22hasCSS\x22\x3Atrue\x7D'),
+        metricPrefix: JSON.parse('\x7B\x223\x22\x3A\x22k\x22,\x226\x22\x3A\x22m\x22,\x229\x22\x3A\x22b\x22,\x2212\x22\x3A\x22t\x22\x7D'),
+        lang: 'en'
+    });
+    </script>
+    <!-- Piwik -->
+<script type="text/javascript">
+  var _paq = _paq || [];
+  _paq.push(["setDocumentTitle", document.domain + "/" + document.title]);
+  _paq.push(["setCookieDomain", "*.www.datawrapper.de"]);
+  _paq.push(["setCustomVariable", 1, "Layout", "default", "page"]);
+  _paq.push(["setCustomVariable", 2, "Author", "9505", "page"]);
+  _paq.push(["setCustomVariable", 3, "Visualization", "line-chart", "page"]);
+  _paq.push(["trackPageView"]);
+  _paq.push(["enableLinkTracking"]);
+  (function() {
+    var u=(("https:" == document.location.protocol) ? "https" : "http") + "://piwik.datawrapper.de/";
+    _paq.push(["setTrackerUrl", u+"piwik.php"]);
+    _paq.push(["setSiteId", "1"]);
+    var d=document, g=d.createElement("script"), s=d.getElementsByTagName("script")[0]; g.type="text/javascript";
+    g.defer=true; g.async=true; g.src=u+"piwik.js"; s.parentNode.insertBefore(g,s);
+  })();
+</script>
+<!-- End Piwik Code -->
+</body>
+</html>

data/spec/jobs/pageflow/chart/scrape_site_job_spec.rb ADDED Viewed

@@ -0,0 +1,22 @@
+require 'spec_helper'
+module Pageflow
+  module Chart
+    describe ScrapeSiteJob do
+      describe '#perform' do
+        it 'scrapes html' do
+          scraper = double("Scraper", html: '<html>rewritten</html>')
+          downloader = double("Downloader", load: '<html>original</html>')
+          job = ScrapeSiteJob.new(downloader)
+          scraped_site = create(:scraped_site, url: 'http://example.com')
+          allow(Scraper).to receive(:new).and_return(scraper)
+          expect(downloader).to receive(:load).with('http://example.com')
+          job.perform(scraped_site)
+        end
+      end
+    end
+  end
+end

data/spec/models/pageflow/chart/scraped_site_spec.rb ADDED Viewed

@@ -0,0 +1,19 @@
+require 'spec_helper'
+module Pageflow::Chart
+  describe ScrapedSite do
+    describe '#csv_url' do
+      it 'replaces base filename of url with data.csv' do
+        scraped_site = ScrapedSite.new(url: 'http://example.com/foo/index.html')
+        expect(scraped_site.csv_url).to eq('http://example.com/foo/data.csv')
+      end
+      it 'appends data.csv to directory url' do
+        scraped_site = ScrapedSite.new(url: 'http://example.com/foo/')
+        expect(scraped_site.csv_url).to eq('http://example.com/foo/data.csv')
+      end
+    end
+  end
+end

data/spec/pageflow/chart/downloader_spec.rb ADDED Viewed

@@ -0,0 +1,90 @@
+require 'spec_helper'
+module Pageflow
+  module Chart
+    describe Downloader do
+      describe '#load' do
+        it 'yields io containing downloaded files' do
+          downloader = Downloader.new
+          result = ''
+          stub_request(:get, "http://example.com/a").to_return(status: 200, body: 'aaa')
+          downloader.load('http://example.com/a') do |io|
+            result = io.read
+          end
+          expect(result).to eq("aaa")
+        end
+        it 'derives protocol from base_url' do
+          downloader = Downloader.new(base_url: 'http://someother.com')
+          result = ''
+          stub_request(:get, "http://example.com/a").to_return(status: 200, body: 'aaa')
+          downloader.load('//example.com/a') do |io|
+            result = io.read
+          end
+          expect(result).to eq("aaa")
+        end
+        it 'handles urls relative to base_url' do
+          downloader = Downloader.new(base_url: 'http://example.com/b/')
+          result = ''
+          stub_request(:get, "http://example.com/a").to_return(status: 200, body: 'aaa')
+          downloader.load('../a') do |io|
+            result = io.read
+          end
+          expect(result).to eq("aaa")
+        end
+      end
+      describe '#load_all' do
+        it 'yields io containing concatenation of downloaded files' do
+          downloader = Downloader.new
+          result = ''
+          stub_request(:get, "http://example.com/a").to_return(status: 200, body: 'aaa')
+          stub_request(:get, "http://example.com/b").to_return(status: 200, body: 'bbb')
+          downloader.load_all(['http://example.com/a', 'http://example.com/b']) do |io|
+            result = io.read
+          end
+        end
+        it 'allows to specify temp file extension' do
+          downloader = Downloader.new
+          path = ''
+          stub_request(:get, "http://example.com/a").to_return(status: 200, body: 'aaa')
+          downloader.load_all(['http://example.com/a'], extension: '.js') do |file|
+            path = file.path
+          end
+          expect(File.extname(path)).to eq('.js')
+        end
+        it 'allows to specify seprator string' do
+          downloader = Downloader.new
+          result = ''
+          stub_request(:get, "http://example.com/a").to_return(status: 200, body: 'aaa')
+          stub_request(:get, "http://example.com/b").to_return(status: 200, body: 'bbb')
+          downloader.load_all(['http://example.com/a', 'http://example.com/b'], separator: ';') do |io|
+            result = io.read
+          end
+          expect(result).to eq("aaa;bbb;")
+        end
+      end
+    end
+  end
+end

data/spec/pageflow/chart/scraper_spec.rb ADDED Viewed

@@ -0,0 +1,179 @@
+require 'spec_helper'
+module Pageflow
+  module Chart
+    describe Scraper do
+      describe '#html' do
+        it 'returns contents of parsed html' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head></head>
+              <body>contents</body>
+            </html>
+          HTML
+          scraper = Scraper.new(html)
+          expect(scraper.html).to include('contents')
+        end
+        it 'combines script tags in head' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head>
+                <script type="text/javascript" src="/some.js"></script>
+                <script type="text/javascript" src="/other.js"></script>
+              </head>
+              <body>
+              </body>
+            </html>
+          HTML
+          scraper = Scraper.new(html)
+          expect(HtmlFragment.new(scraper.html)).not_to have_tag('head script[src="/some.js"]')
+          expect(HtmlFragment.new(scraper.html)).to have_tag('head script[src="all.js"]')
+        end
+        it 'combines link tags in head' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head>
+                <link rel="stylesheet" type="text/css" href="/some.css">
+                <link rel="stylesheet" type="text/css" href="/other.css">
+              </head>
+              <body>
+              </body>
+            </html>
+          HTML
+          scraper = Scraper.new(html)
+          expect(HtmlFragment.new(scraper.html)).not_to have_tag('head link[href="/some.css"]')
+          expect(HtmlFragment.new(scraper.html)).to have_tag('head link[href="all.css"]')
+        end
+        it 'filters blacklisted inline scripts' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head></head>
+              <body>
+                <script id="good">window.ok = true;</script>
+                <script id="bad">alert();</script>
+              </body>
+            </html>
+          HTML
+          scraper = Scraper.new(html, inline_script_blacklist: [/alert/])
+          expect(HtmlFragment.new(scraper.html)).to have_tag('body script#good')
+          expect(HtmlFragment.new(scraper.html)).not_to have_tag('body script#bad')
+        end
+        it 'filters blacklisted selectors' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head></head>
+              <body>
+                <div id="bad" class="noscript"></div>
+                <div id="good"></div>
+              </body>
+            </html>
+          HTML
+          scraper = Scraper.new(html, selector_blacklist: ['body .noscript'])
+          expect(HtmlFragment.new(scraper.html)).to have_tag('body #good')
+          expect(HtmlFragment.new(scraper.html)).not_to have_tag('body #bad')
+        end
+      end
+      describe '#javascript_urls' do
+        it 'returns list of urls to javascript files' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head>
+                <script type="text/javascript" src="/some.js"></script>
+                <script type="text/javascript" src="/other.js"></script>
+              </head>
+              <body>
+              </body>
+            </html>
+          HTML
+          scraper = Scraper.new(html)
+          expect(scraper.javascript_urls).to eq(['/some.js', '/other.js'])
+        end
+        it 'filters by blacklist' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head>
+                <script type="text/javascript" src="/some.js"></script>
+                <script type="text/javascript" src="http://example.com/piwik.js"></script>
+              </head>
+              <body>
+              </body>
+            </html>
+          HTML
+          scraper = Scraper.new(html, head_script_blacklist: [/piwik/])
+          expect(scraper.javascript_urls).to eq(['/some.js'])
+        end
+        it 'ignores inline scripts in head' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head>
+                <script type="text/javascript"></script>
+              </head>
+              <body>
+              </body>
+            </html>
+          HTML
+          scraper = Scraper.new(html)
+          expect(scraper.javascript_urls).to eq([])
+        end
+      end
+      describe '#stylesheet_urls' do
+        it 'returns list of urls to stylesheet files' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head>
+                <link rel="stylesheet" type="text/css" href="/some.css">
+                <link rel="stylesheet" type="text/css" href="/other.css">
+              </head>
+              <body>
+              </body>
+            </html>
+          HTML
+          scraper = Scraper.new(html)
+          expect(scraper.stylesheet_urls).to eq(['/some.css', '/other.css'])
+        end
+        it 'ignores non css links' do
+          html = <<-HTML
+            <!DOCTYPE html>
+            <html>
+              <head>
+                <link rel="shortcut icon" href="/favicon.ico" />
+              </head>
+              <body>
+              </body>
+            </html>
+          HTML
+          scraper = Scraper.new(html)
+          expect(scraper.stylesheet_urls).to eq([])
+        end
+      end
+    end
+  end
+end

data/spec/requests/scraping_site_spec.rb ADDED Viewed

@@ -0,0 +1,23 @@
+require 'spec_helper'
+module Pageflow
+  module Chart
+    describe 'scraping site', :inline_resque => true do
+      before do
+        stub_request(:get, "http://example.com/chart.html")
+          .to_return(:status => 200, :body => File.read('spec/fixtures/datawrapper.html'))
+        stub_request(:get, /js$|css$|csv$/)
+          .to_return(:status => 200, :body => 'file')
+      end
+      it 'downloads html and dependencies' do
+        post('/charts/scraped_sites', scraped_site: {url: 'http://example.com/chart.html'}, format: 'json')
+        expect(ScrapedSite.first.html_file).to be_present
+        expect(ScrapedSite.first.javascript_file).to be_present
+        expect(ScrapedSite.first.stylesheet_file).to be_present
+        expect(ScrapedSite.first.csv_file).to be_present
+      end
+    end
+  end
+end