radiant-scraped_layouts-extension 1.0.0

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gem 'radiant', '~> 1.1.0'
+

data/Gemfile.lock

@@ -0,0 +1,69 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    RedCloth (4.2.9)
+    actionmailer (2.3.17)
+      actionpack (= 2.3.17)
+    actionpack (2.3.17)
+      activesupport (= 2.3.17)
+      rack (~> 1.1.0)
+    activerecord (2.3.17)
+      activesupport (= 2.3.17)
+    activeresource (2.3.17)
+      activesupport (= 2.3.17)
+    activesupport (2.3.17)
+    acts_as_tree (0.1.1)
+    chunky_png (1.2.7)
+    compass (0.12.2)
+      chunky_png (~> 1.2)
+      fssm (>= 0.2.7)
+      sass (~> 3.1)
+    compass-rails (1.0.3)
+      compass (>= 0.12.2, < 0.14)
+    delocalize (0.2.6)
+    fssm (0.2.10)
+    haml (3.1.8)
+    highline (1.6.16)
+    json (1.7.7)
+    rack (1.1.6)
+    rack-cache (1.2)
+      rack (>= 0.4)
+    radiant (1.1.3)
+      RedCloth (~> 4.2.0)
+      acts_as_tree (~> 0.1.1)
+      bundler (>= 1.0.0)
+      compass (~> 0.12.2)
+      compass-rails (~> 1.0.3)
+      delocalize (~> 0.2.3)
+      haml (~> 3.1.1)
+      highline (~> 1.6.10)
+      rack (~> 1.1.6)
+      rack-cache (~> 1.2)
+      radius (~> 0.7.3)
+      rails (~> 2.3.17)
+      rake (>= 0.8.7)
+      rdoc (>= 3.9.2)
+      stringex (~> 1.3.0)
+      tzinfo (~> 0.3.31)
+      will_paginate (~> 2.3.11)
+    radius (0.7.3)
+    rails (2.3.17)
+      actionmailer (= 2.3.17)
+      actionpack (= 2.3.17)
+      activerecord (= 2.3.17)
+      activeresource (= 2.3.17)
+      activesupport (= 2.3.17)
+      rake (>= 0.8.3)
+    rake (10.0.3)
+    rdoc (4.0.0)
+      json (~> 1.4)
+    sass (3.2.7)
+    stringex (1.3.3)
+    tzinfo (0.3.37)
+    will_paginate (2.3.16)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  radiant (~> 1.1.0)

data/README.md

@@ -0,0 +1,7 @@
+# Scraped Layouts
+
+This extension allows a Radiant designer to pull a webpage from a given URL, replace portions of it using regular expressions, and save the outcome as a standard (but uneditable) Radiant layout.
+
+---
+
+Created by Charles Hudson for YP Intellectual Property, LLC (yellowpages.com).

data/Rakefile

@@ -0,0 +1,109 @@
+# Determine where the RSpec plugin is by loading the boot
+unless defined? RADIANT_ROOT
+  ENV["RAILS_ENV"] = "test"
+  case
+  when ENV["RADIANT_ENV_FILE"]
+    require File.dirname(ENV["RADIANT_ENV_FILE"]) + "/boot"
+  when File.dirname(__FILE__) =~ %r{vendor/radiant/vendor/extensions}
+    require "#{File.expand_path(File.dirname(__FILE__) + "/../../../../../")}/config/boot"
+  else
+    require "#{File.expand_path(File.dirname(__FILE__) + "/../../../")}/config/boot"
+  end
+end
+
+require 'rake'
+require 'rdoc/task'
+require 'rake/testtask'
+
+rspec_base = File.expand_path(RADIANT_ROOT + '/vendor/plugins/rspec/lib')
+$LOAD_PATH.unshift(rspec_base) if File.exist?(rspec_base)
+require 'spec/rake/spectask'
+require 'cucumber'
+require 'cucumber/rake/task'
+
+# Cleanup the RADIANT_ROOT constant so specs will load the environment
+Object.send(:remove_const, :RADIANT_ROOT)
+
+extension_root = File.expand_path(File.dirname(__FILE__))
+
+task :default => [:spec, :features]
+task :stats => "spec:statsetup"
+
+desc "Run all specs in spec directory"
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.spec_opts = ['--options', "\"#{extension_root}/spec/spec.opts\""]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+task :features => 'spec:integration'
+
+namespace :spec do
+  desc "Run all specs in spec directory with RCov"
+  Spec::Rake::SpecTask.new(:rcov) do |t|
+    t.spec_opts = ['--options', "\"#{extension_root}/spec/spec.opts\""]
+    t.spec_files = FileList['spec/**/*_spec.rb']
+    t.rcov = true
+    t.rcov_opts = ['--exclude', 'spec', '--rails']
+  end
+  
+  desc "Print Specdoc for all specs"
+  Spec::Rake::SpecTask.new(:doc) do |t|
+    t.spec_opts = ["--format", "specdoc", "--dry-run"]
+    t.spec_files = FileList['spec/**/*_spec.rb']
+  end
+
+  [:models, :controllers, :views, :helpers].each do |sub|
+    desc "Run the specs under spec/#{sub}"
+    Spec::Rake::SpecTask.new(sub) do |t|
+      t.spec_opts = ['--options', "\"#{extension_root}/spec/spec.opts\""]
+      t.spec_files = FileList["spec/#{sub}/**/*_spec.rb"]
+    end
+  end
+  
+  desc "Run the Cucumber features"
+  Cucumber::Rake::Task.new(:integration) do |t|
+    t.fork = true
+    t.cucumber_opts = ['--format', (ENV['CUCUMBER_FORMAT'] || 'pretty')]
+    # t.feature_pattern = "#{extension_root}/features/**/*.feature"
+    t.profile = "default"
+  end
+
+  # Setup specs for stats
+  task :statsetup do
+    require 'code_statistics'
+    ::STATS_DIRECTORIES << %w(Model\ specs spec/models)
+    ::STATS_DIRECTORIES << %w(View\ specs spec/views)
+    ::STATS_DIRECTORIES << %w(Controller\ specs spec/controllers)
+    ::STATS_DIRECTORIES << %w(Helper\ specs spec/views)
+    ::CodeStatistics::TEST_TYPES << "Model specs"
+    ::CodeStatistics::TEST_TYPES << "View specs"
+    ::CodeStatistics::TEST_TYPES << "Controller specs"
+    ::CodeStatistics::TEST_TYPES << "Helper specs"
+    ::STATS_DIRECTORIES.delete_if {|a| a[0] =~ /test/}
+  end
+
+  namespace :db do
+    namespace :fixtures do
+      desc "Load fixtures (from spec/fixtures) into the current environment's database.  Load specific fixtures using FIXTURES=x,y"
+      task :load => :environment do
+        require 'active_record/fixtures'
+        ActiveRecord::Base.establish_connection(RAILS_ENV.to_sym)
+        (ENV['FIXTURES'] ? ENV['FIXTURES'].split(/,/) : Dir.glob(File.join(RAILS_ROOT, 'spec', 'fixtures', '*.{yml,csv}'))).each do |fixture_file|
+          Fixtures.create_fixtures('spec/fixtures', File.basename(fixture_file, '.*'))
+        end
+      end
+    end
+  end
+end
+
+desc 'Generate documentation for the scraped_layout extension.'
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ScrapedLayoutExtension'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+# Load any custom rakefiles for extension
+Dir[File.dirname(__FILE__) + '/tasks/*.rake'].sort.each { |f| require f }

data/app/controllers/admin/scrapers_controller.rb

@@ -0,0 +1,74 @@
+class Admin::ScrapersController < Admin::ResourceController
+
+  def new
+    @scraper = Scraper.new
+    @scraper.scraper_inserts << ScraperInsert.new
+    response_for :new
+  end
+
+  def create
+    @scraper = Scraper.new(params[:scraper])
+    @scraper.layout = Layout.new({ :name => "#{@scraper.name} (scraped)" })
+
+    save_scraper @scraper
+  end
+
+  def update
+    @scraper = Scraper.find(params[:id])
+    @scraper.update_attributes(params[:scraper])
+
+    save_scraper @scraper, :edit
+  end
+
+  private
+
+  def save_scraper(scraper, action = :new)
+    if manage_inserts?(scraper) || !scraper.save
+      unless scraper.errors.empty?
+        flash[:error] = scraper.errors.full_messages.join(' ')
+      end
+
+      render :action => action
+    else
+      scrape_site(scraper)
+
+      flash[:notice] = "Scraper successfully #{action == :new ? 'created' : 'updated'}"
+
+      redirect_to admin_scrapers_path
+    end
+  end
+
+  def manage_inserts?(scraper)
+    managed = false
+    scraper.scraper_inserts.each do |insert|
+      if insert.remove
+        scraper.scraper_inserts.delete(insert)
+        managed = true
+      end
+    end
+
+    if params[:add_insert]
+      scraper.scraper_inserts << ScraperInsert.new
+      scraper.scraper_inserts.last.errors.clear # TODO: manage this separately so Rails won't try to save it automatically.
+      managed = true
+    end
+
+    managed
+  end
+
+  def scrape_site(scraper)
+    begin
+      html = Net::HTTP.get_response(URI(scraper.url)).body
+
+      scraper.scraper_inserts.each do |insert|
+        html.gsub!(Regexp.new(insert.regex), insert.content)
+      end
+
+      scraper.layout.content = html
+      scraper.layout.generated = true
+    rescue
+      scraper.layout.content = nil
+      scraper.layout.generated = false
+    end
+  end
+end

data/app/models/scraper.rb

@@ -0,0 +1,10 @@
+class Scraper < ActiveRecord::Base
+  has_many :scraper_inserts, :autosave => true, :dependent => :destroy
+
+  belongs_to :layout, :autosave => true, :dependent => :destroy
+
+  validates_uniqueness_of :name, :case_sensitive => true
+  validates_presence_of :name, :url, :description
+
+  accepts_nested_attributes_for :scraper_inserts
+end

data/app/models/scraper_insert.rb

@@ -0,0 +1,7 @@
+class ScraperInsert < ActiveRecord::Base
+  belongs_to :scraper, :autosave => true
+
+  validates_presence_of :regex, :content
+
+  attr_accessor :remove
+end

data/app/views/admin/layouts/_actions_cell.html.haml

@@ -0,0 +1,3 @@
+%td.actions
+  - unless layout.scraped?
+    = link_to image('minus') + ' ' + t('remove'), remove_admin_layout_url(layout), :class => "action"

data/app/views/admin/layouts/_disabled_form.html.haml

@@ -0,0 +1,32 @@
+- form_for [:admin, @layout] do |f|
+  = f.hidden_field :lock_version
+  = render_region :form_top, :locals => {:f => f}
+  .form_area
+    - render_region :form, :locals => {:f => f} do |form|
+      - form.edit_title do
+        %p.title
+          = f.label :name, t('name')
+          = f.text_field :name, :maxlength => 100, :class => "disabled textbox", :disabled => true
+      - form.edit_extended_metadata do
+        .drawer
+          .drawer_contents#attributes
+            %table.fieldset
+              %tr
+                %th.label= f.label :content_type, t('content_type')
+                %td.field= f.text_field :content_type, :maxlength => 40, :class => "disabled textbox", :disabled => true
+              = render_region :extended_metadata, :locals => {:f => f}
+          .drawer_handle
+            %a.toggle{:href=>'#attributes', :rel=>"toggle[attributes]", :class=>"#{(meta_errors? ? 'less' : 'more')}"}= meta_label
+      - form.edit_content do
+        %p.content
+          = f.label :layout_content, "Body"
+          ~ f.text_area :content, :size => "40x20", :style => "width: 100%", :class => "disabled textarea large", :disabled => true
+  - render_region :form_bottom, :locals => {:f => f} do |form_bottom|
+    - form_bottom.reference_links do
+      %p{ :style => 'padding: 2ex 1em; border: 6px outset #f00; text-align: center; color: #c00; font-weight: bold; background: #fff;' }
+        This layout has been generated and may not be directly edited.
+    - form_bottom.edit_buttons do
+      .buttons
+        = link_to '« Return to Index', admin_layouts_url
+    - form_bottom.edit_timestamp do
+      = updated_stamp @layout

data/app/views/admin/layouts/_title_cell.html.haml

@@ -0,0 +1,5 @@
+%td.name
+  - if layout.scraped?
+    %em= link_to image('virtual_page', :alt => '') + ' ' + layout.name, edit_admin_layout_url(layout)
+  -else
+    = link_to image('layout', :alt => '') + ' ' + layout.name, edit_admin_layout_url(layout)

data/app/views/admin/layouts/_warning.html.haml

@@ -0,0 +1,3 @@
+- if @layout.scraped?
+  #notice
+    %p This layout has been generated and may not be modified.

data/app/views/admin/layouts/edit.html.haml

@@ -0,0 +1,15 @@
+- @page_title = @layout.name + ' ' + t('layout') + ' - ' + default_page_title
+
+- render_region :main do |main|
+  - if @layout.scraped?
+    - main.edit_header do
+      %h1= 'Scraped Layout'
+    - main.edit_form do
+      = render :partial => 'disabled_form'
+  - else
+    - main.edit_header do
+      %h1= t('edit_layout')
+    - main.edit_form do
+      = render :partial => 'form'
+
+- content_for 'page_scripts', layout_edit_javascripts

data/app/views/admin/scraper_inserts/_scraper_insert.html.haml

@@ -0,0 +1,10 @@
+%hr
+
+%p
+  = f.label :regex
+  = f.text_field :regex, :class => 'textbox', :size => 90
+%p
+  = f.label :content
+  = f.text_area :content, :class => 'textarea', :style => 'width: 90%;', :cols => 90, :rows => 10
+%p
+  = f.submit 'Remove Insert', :name => "scraper[scraper_inserts_attributes][#{f.object_name.gsub(/[^0-9]+/,'')}][remove]"

data/app/views/admin/scrapers/_form.html.haml

@@ -0,0 +1,31 @@
+- form_for [:admin, @scraper] do |f|
+  = render_region :form_top, :locals => {:f => f}
+
+  - render_region :form do |form|
+    - form.edit_title do
+      %p
+        = f.label :name
+        = f.text_field :name, :class => 'textbox', :size => 90
+    - form.edit_url do
+      %p
+        %label{:for => "scraper_url"} URL
+        = f.text_field :url, :class => 'textbox', :size => 90
+    - form.edit_description do
+      %p
+        %label{:for => "scraper_description"} Description
+        = f.text_field :description, :class => 'textbox', :size => 90
+
+    - form.edit_inserts do
+      - f.fields_for :scraper_inserts do |insert_form|
+        = render :partial => 'admin/scraper_inserts/scraper_insert', :locals => { :f => insert_form }
+
+      %hr
+
+      = submit_tag 'Add Another Insert', :name => 'add_insert'
+
+  - render_region :form_bottom do |form_bottom|
+    - form_bottom.edit_buttons do
+      %p.buttons
+        = save_model_button(@scraper)
+        or
+        = link_to "Cancel", admin_scrapers_path

data/app/views/admin/scrapers/edit.html.haml

@@ -0,0 +1,7 @@
+- body_classes << "reversed"
+
+- render_region :main do |main|
+  - main.edit_header do
+    %h1 Edit Scraper
+  - main.edit_form do
+    = render :partial => "form"

data/app/views/admin/scrapers/index.html.haml

@@ -0,0 +1,35 @@
+- @page_title = t('scraped_layouts') + ' - ' + default_page_title
+
+.outset
+  = render_region :top
+  %table.index#scrapers
+    %thead
+      %tr
+        - render_region :thead do |thead|
+          - thead.title_header do
+            %th.name
+              = t('scraper')
+          - thead.actions_header do
+            %th.actions{:style=>"width:9em"}
+              = t('modify')
+    %tbody
+      - if @scrapers.any?
+        - @scrapers.each do |scraper|
+          %tr[scraper]
+            - render_region :tbody, :locals => {:scraper => scraper} do |tbody|
+              - tbody.title_cell do
+                %td.name
+                  = link_to image('layout', :alt => '') + ' ' + scraper.name, edit_admin_scraper_url(scraper)
+              - tbody.actions_cell do
+                %td.actions
+                  = link_to image('minus') + ' ' + t('remove'), remove_admin_scraper_url(scraper), :class => "action"
+      - else
+        %tr
+          %td.empty{:colspan => admin.scrapers.index.tbody.length}= t('no_scrapers')
+
+- render_region :bottom do |bottom|
+  - bottom.new_button do
+    #actions
+      = pagination_for(@scrapers)
+      %ul
+        %li= link_to image('plus') + " " + t('new_scraper'), new_admin_scraper_url, :class => 'action_button'

data/app/views/admin/scrapers/new.html.haml

@@ -0,0 +1,4 @@
+- body_classes << "reversed"
+
+%h1 New Scraper
+= render :partial => "form"

data/app/views/admin/scrapers/remove.html.haml

@@ -0,0 +1,16 @@
+%h1= t('remove_scraper')
+
+%p= t('text.scrapers.remove_warning')
+
+%table.index#scrapers
+  %tbody
+    %tr.node.level_1
+      %td.scraper
+        = image('layout', :alt => '')
+        = h @scraper.name
+
+- form_for [:admin, @scraper], :html => { :method => 'delete', 'data-onsubmit_status'=>"Removing scraper…" } do
+  .buttons
+    %input.button{:type => "submit", :value => t('delete_scraper')  }/
+    = t('or')
+    = link_to t('cancel'), admin_scrapers_url

data/config/initializers/radiant_config.rb

@@ -0,0 +1,3 @@
+Radiant.config do |config|
+  # config.define "setting.name", :default => 'value', :select_from => ['foo', 'bar']
+end

data/config/locales/en.yml

@@ -0,0 +1,12 @@
+---
+en:
+  scraped_layout: Scraped Layout
+  scraped_layouts: Scraped Layouts
+  scraper: Scraper
+  no_scrapers: No Scraped Layouts
+  new_scraper: New Scraped Layout
+  delete_scraper: Delete Scraper
+  remove_scraper: Remove Scraper
+  text:
+    scrapers:
+      remove_warning: 'Are you sure you want to <strong class="warning">permanently remove</strong> the following layout?'

data/config/routes.rb

@@ -0,0 +1,5 @@
+ActionController::Routing::Routes.draw do |map|
+  map.namespace :admin, :member => { :remove => :get } do |admin|
+    admin.resources :scrapers
+  end
+end

data/cucumber.yml

@@ -0,0 +1 @@
+default: --format progress features --tags ~@proposed,~@in_progress

data/db/migrate/20121015160755_create_scrapers.rb

@@ -0,0 +1,15 @@
+class CreateScrapers < ActiveRecord::Migration
+  def self.up
+    create_table :scrapers do |t|
+      t.string :name, :unique => true
+      t.string :url
+      t.text :description
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :scrapers
+  end
+end

data/db/migrate/20121015160953_create_scraper_inserts.rb

@@ -0,0 +1,16 @@
+class CreateScraperInserts < ActiveRecord::Migration
+  def self.up
+    create_table :scraper_inserts do |t|
+      t.string :regex
+      t.text :content
+
+      t.references :scraper
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :scraper_inserts
+  end
+end

data/db/migrate/20121101183044_integrate_scraper_and_layout.rb

@@ -0,0 +1,11 @@
+class IntegrateScraperAndLayout < ActiveRecord::Migration
+  def self.up
+    change_table :scrapers do |t|
+      t.references :layout
+    end
+  end
+
+  def self.down
+    remove_column :scrapers, :layout_id
+  end
+end

data/features/support/env.rb

@@ -0,0 +1,11 @@
+# Sets up the Rails environment for Cucumber
+ENV["RAILS_ENV"] = "test"
+# Extension root
+extension_env = File.expand_path(File.dirname(__FILE__) + '/../../../../../config/environment')
+require extension_env+'.rb'
+
+Dir.glob(File.join(RADIANT_ROOT, "features", "**", "*.rb")).each {|step| require step unless step =~ /datasets_loader\.rb$/}
+ 
+Cucumber::Rails::World.class_eval do
+  dataset :scraped_layout
+end

data/features/support/paths.rb

@@ -0,0 +1,22 @@
+module NavigationHelpers
+  
+  # Extend the standard PathMatchers with your own paths
+  # to be used in your features.
+  # 
+  # The keys and values here may be used in your standard web steps
+  # Using:
+  #
+  #   When I go to the "scraped_layout" admin page
+  # 
+  # would direct the request to the path you provide in the value:
+  # 
+  #   admin_scraped_layout_path
+  # 
+  PathMatchers = {} unless defined?(PathMatchers)
+  PathMatchers.merge!({
+    # /scraped_layout/i => 'admin_scraped_layout_path'
+  })
+  
+end
+
+World(NavigationHelpers)

data/lib/radiant-scraped_layouts-extension.rb

@@ -0,0 +1,8 @@
+module RadiantScrapedLayoutsExtension
+  VERSION     = "1.0.0"
+  SUMMARY     = "Scraped Layouts for Radiant CMS"
+  DESCRIPTION = "Generates scraped layouts, allowing content to be inserted via regular expressions."
+  URL         = "https://github.com/yp-engineering/radiant-scraped_layouts-extension"
+  AUTHORS     = ["Charles Hudson"]
+  EMAIL       = ["charles.h.hudson@phobetron.com"]
+end

data/lib/tasks/scraped_layouts_extension_tasks.rake

@@ -0,0 +1,47 @@
+namespace :radiant do
+  namespace :extensions do
+    namespace :scraped_layouts do
+      
+      desc "Runs the migration of the Scraped Layout extension"
+      task :migrate => :environment do
+        require 'radiant/extension_migrator'
+        if ENV["VERSION"]
+          ScrapedLayoutsExtension.migrator.migrate(ENV["VERSION"].to_i)
+          Rake::Task['db:schema:dump'].invoke
+        else
+          ScrapedLayoutsExtension.migrator.migrate
+          Rake::Task['db:schema:dump'].invoke
+        end
+      end
+      
+      desc "Copies public assets of the Scraped Layout to the instance public/ directory."
+      task :update => :environment do
+        is_svn_or_dir = proc {|path| path =~ /\.svn/ || File.directory?(path) }
+        puts "Copying assets from ScrapedLayoutsExtension"
+        Dir[ScrapedLayoutsExtension.root + "/public/**/*"].reject(&is_svn_or_dir).each do |file|
+          path = file.sub(ScrapedLayoutsExtension.root, '')
+          directory = File.dirname(path)
+          mkdir_p RAILS_ROOT + directory, :verbose => false
+          cp file, RAILS_ROOT + path, :verbose => false
+        end
+      end  
+      
+      desc "Syncs all available translations for this ext to the English ext master"
+      task :sync => :environment do
+        # The main translation root, basically where English is kept
+        language_root = ScrapedLayoutsExtension.root + "/config/locales"
+        words = TranslationSupport.get_translation_keys(language_root)
+        
+        Dir["#{language_root}/*.yml"].each do |filename|
+          next if filename.match('_available_tags')
+          basename = File.basename(filename, '.yml')
+          puts "Syncing #{basename}"
+          (comments, other) = TranslationSupport.read_file(filename, basename)
+          words.each { |k,v| other[k] ||= words[k] }  # Initializing hash variable as empty if it does not exist
+          other.delete_if { |k,v| !words[k] }         # Remove if not defined in en.yml
+          TranslationSupport.write_file(filename, basename, comments, other)
+        end
+      end
+    end
+  end
+end

data/radiant-scraped_layouts-extension.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "radiant-scraped_layouts-extension"
+
+Gem::Specification.new do |s|
+  s.name        = "radiant-scraped_layouts-extension"
+  s.version     = RadiantScrapedLayoutsExtension::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = RadiantScrapedLayoutsExtension::AUTHORS
+  s.email       = RadiantScrapedLayoutsExtension::EMAIL
+  s.homepage    = RadiantScrapedLayoutsExtension::URL
+  s.summary     = RadiantScrapedLayoutsExtension::SUMMARY
+  s.description = RadiantScrapedLayoutsExtension::DESCRIPTION
+
+  s.add_dependency "radiant", "~> 1.1.0"
+
+  ignores = if File.exist?('.gitignore')
+    File.read('.gitignore').split("\n").collect { |a| d = Dir["./#{a}/**/*"]; (d.empty? ? a : d) }.flatten
+  else
+    []
+  end
+
+  s.files         = Dir['./**/*'] - ignores
+  s.test_files    = Dir['test/**/*','spec/**/*','features/**/*'] - ignores
+  s.require_paths = ["lib"]
+end

data/scraped_layouts_extension.rb

@@ -0,0 +1,47 @@
+# Uncomment this if you reference any of your controllers in activate
+# require_dependency "application_controller"
+require "radiant-scraped_layouts-extension"
+
+class ScrapedLayoutsExtension < Radiant::Extension
+  version     RadiantScrapedLayoutsExtension::VERSION
+  description RadiantScrapedLayoutsExtension::DESCRIPTION
+  url         RadiantScrapedLayoutsExtension::URL
+
+  # See your config/routes.rb file in this extension to define custom routes
+
+  extension_config do |config|
+    # config is the Radiant.configuration object
+  end
+
+  def activate
+    Layout.class_eval do
+      has_one :scraper
+
+      attr_accessor :generated
+      validates_presence_of :generated, :if => :scraped?, :message => 'layouts may only be created or edited within the Scraped Layouts section, and when supplied with a working URL.'
+
+      def scraped?
+        !scraper.nil?
+      end
+    end
+
+    tab 'Design' do
+      add_item "Scraped Layouts", "/admin/scrapers", :after => "Layouts"
+    end
+
+    Radiant::AdminUI.class_eval { attr_accessor :scrapers }
+    admin.scrapers = OpenStruct.new.tap do |scrapers|
+      scrapers.index = Radiant::AdminUI::RegionSet.new do |index|
+        index.thead.concat %w{title_header actions_header}
+        index.tbody.concat %w{title_cell actions_cell}
+        index.bottom.concat %w{new_button}
+      end
+      scrapers.edit = Radiant::AdminUI::RegionSet.new do |edit|
+        edit.main.concat %w{edit_header edit_form}
+        edit.form.concat %w{edit_title edit_url edit_description edit_inserts}
+        edit.form_bottom.concat %w{edit_buttons}
+      end
+      scrapers.new = scrapers.edit
+    end
+  end
+end

data/spec/controllers/admin/scrapers_controller_spec.rb

@@ -0,0 +1,255 @@
+require 'spec_helper'
+
+describe Admin::ScrapersController do
+  dataset :users
+
+  before :each do
+    login_as :designer
+
+    @scraper_params = {
+      :name => 'YP.com',
+      :url => 'http://www.yellowpages.com/pages/partner_template',
+      :description => 'YP.com Template',
+      :scraper_inserts_attributes => []
+    }
+    @scraper_insert_params = {
+      :regex => '^.*$',
+      :content => 'New'
+    }
+    @scraper_params[:scraper_inserts_attributes] << @scraper_insert_params
+
+    @scraper = mock_model(Scraper)
+    @scraper.stub!(:url).and_return(@scraper_params[:url])
+    @scraper.stub!(:name).and_return(@scraper_params[:name])
+    @scraper.stub!(:description).and_return(@scraper_params[:description])
+  end
+
+  describe :new do
+    it 'initializes a new Scraper' do
+      get :new
+
+      assigns(:scraper).should be_new_record
+    end
+
+    it 'adds a new ScraperInsert to the new Scraper' do
+      get :new
+
+      assigns(:scraper).scraper_inserts[0].should be_new_record
+    end
+  end
+
+  describe :create do
+    before :each do
+      @layout = mock_model(Layout)
+      Layout.stub!(:new).and_return(@layout)
+
+      @scraper.stub!(:layout=).and_return(@layout)
+      Scraper.stub!(:new).and_return(@scraper)
+
+      controller.stub!(:save_scraper).and_return(true)
+    end
+
+    it 'creates a new associated Layout with a name based on the Scraper\'s name' do
+      Layout.should_receive(:new).with(:name => 'YP.com (scraped)')
+
+      post :create, :scraper => @scraper_params
+    end
+
+    it 'calls save_scraper method' do
+      controller.should_receive(:save_scraper)
+
+      post :create, :scraper => @scraper_params
+    end
+  end
+
+  describe :update do
+    before :each do
+      @scraper.stub!(:update_attributes).and_return(true)
+      Scraper.stub!(:find).and_return(@scraper)
+      Scraper.stub!(:update_attributes).and_return(true)
+
+      controller.stub!(:save_scraper).and_return(true)
+    end
+
+    it 'updates the existing Scraper with form params' do
+      @scraper.should_receive(:update_attributes).with(anything)
+
+      put :update, :id => 0, :scraper => @scraper_params
+    end
+
+    it 'calls save_scraper method with previous action name' do
+      controller.should_receive(:save_scraper).with(@scraper, :edit)
+
+      put :update, :id => 0, :scraper => @scraper_params
+    end
+  end
+
+  describe :save_scraper do
+    before :each do
+      controller.stub!(:scrape_site).and_return(true)
+      controller.stub!(:redirect_to)
+      controller.stub!(:render)
+      controller.stub!(:admin_scrapers_path)
+    end
+
+    it 'calls manage_inserts?' do
+      @scraper.stub!(:save).and_return(true)
+
+      controller.should_receive(:manage_inserts?)
+
+      controller.send(:save_scraper, @scraper)
+    end
+
+    it 'adds flash notice on successful save or update' do
+      @scraper.stub!(:save).and_return(true)
+      controller.stub!(:manage_inserts?).and_return(false)
+
+      controller.send(:save_scraper, @scraper)
+
+      flash[:error].should be_nil
+      flash[:notice].should == 'Scraper successfully created'
+    end
+
+    it 'redirects to Scraped Layouts index on successful save or update' do
+      controller.stub!(:manage_inserts?).and_return(false)
+      @scraper.stub!(:save).and_return(true)
+
+      controller.should_receive(:redirect_to)
+      controller.should_receive(:admin_scrapers_path)
+
+      controller.send(:save_scraper, @scraper)
+    end
+
+    it 'renders to previous action when inserts are managed (added or removed)' do
+      controller.stub!(:manage_inserts?).and_return(true)
+
+      @scraper.stub!(:errors).and_return({})
+      errors = @scraper.errors
+      errors.stub!(:empty?).and_return(true)
+
+      controller.should_receive(:render).with(:action => :new)
+
+      controller.send(:save_scraper, @scraper)
+    end
+
+    it 'adds flash error when save is unsuccessful' do
+      controller.stub!(:manage_inserts?).and_return(false)
+      @scraper.stub!(:save).and_return(false)
+
+      @scraper.stub!(:errors).and_return({})
+      errors = @scraper.errors
+      errors.stub!(:empty?).and_return(false)
+      errors.stub!(:full_messages).and_return(['Error'])
+
+      controller.send(:save_scraper, @scraper)
+
+      flash[:notice].should be_nil
+      flash[:error].should == 'Error'
+    end
+
+    it 'renders previous action when save is unsuccessful' do
+      controller.stub!(:manage_inserts?).and_return(false)
+      @scraper.stub!(:save).and_return(false)
+
+      @scraper.stub!(:errors).and_return({})
+      errors = @scraper.errors
+      errors.stub!(:empty?).and_return(true)
+
+      controller.should_receive(:render).with(:action => :edit)
+
+      controller.send(:save_scraper, @scraper, :edit)
+    end
+  end
+
+  describe :manage_inserts? do
+    before :each do
+      @scraper_insert = mock_model(ScraperInsert)
+      @scraper_insert.stub!(:regex).and_return(@scraper_insert_params[:regex])
+      @scraper_insert.stub!(:content).and_return(@scraper_insert_params[:content])
+
+      @scraper_inserts = [@scraper_insert]
+      @scraper.stub!(:scraper_inserts).and_return(@scraper_inserts)
+
+      controller.stub!(:scrape_site).and_return(true)
+    end
+
+    it 'removes inserts that are marked for removal' do
+      @scraper_insert.stub!(:remove).and_return(true)
+
+      @scraper_inserts.should_receive(:delete).with(@scraper_insert)
+
+      controller.send(:manage_inserts?, @scraper).should be_true
+    end
+
+    it 'adds inserts when requested' do
+      @scraper_insert.stub!(:remove).and_return(false)
+      @scraper_insert.stub!(:errors).and_return({})
+      errors = @scraper_insert.errors
+      errors.stub!(:clear).and_return(true)
+
+      @scraper_inserts.should_receive(:<<)
+
+      controller.params[:add_insert] = true
+      controller.send(:manage_inserts?, @scraper).should be_true
+    end
+
+    it 'returns false when inserts have not been managed' do
+      @scraper_insert.stub!(:remove).and_return(false)
+
+      controller.send(:manage_inserts?, @scraper).should be_false
+    end
+  end
+
+  describe :scrape_site do
+    before :each do
+      @scraper_insert = mock_model(ScraperInsert)
+      @scraper_insert.stub!(:regex).and_return(@scraper_insert_params[:regex])
+      @scraper_insert.stub!(:content).and_return(@scraper_insert_params[:content])
+
+      @scraper_inserts = [@scraper_insert]
+      @scraper.stub!(:scraper_inserts).and_return(@scraper_inserts)
+
+      @layout = mock_model(Layout)
+      @layout.stub!(:content=)
+      @layout.stub!(:generated=)
+      @scraper.stub!(:layout).and_return(@layout)
+
+      @uri = URI(@scraper.url)
+      @response = mock(Object)
+      @response.stub!(:body).and_return('Text')
+
+      @regex = Regexp.new(@scraper_insert.regex)
+      Regexp.stub!(:new).and_return(@regex)
+
+      controller.stub!(:URI).and_return(@uri)
+    end
+
+    it 'stuffs content from an external URL into a Scraper\'s associalte Layout' do
+      Net::HTTP.should_receive(:get_response).with(@uri).and_return(@response)
+
+      @layout.should_receive(:content=).with(anything)
+      @layout.should_receive(:generated=).with(true)
+
+      controller.send(:scrape_site, @scraper)
+    end
+
+    it 'inserts content via the given regex into the HTML returned by the URL' do
+      Net::HTTP.should_receive(:get_response).with(@uri).and_return(@response)
+
+      @layout.should_receive(:content=).with('New')
+      @layout.should_receive(:generated=).with(true)
+
+      controller.send(:scrape_site, @scraper)
+    end
+
+    it 'places the Scraper\'s associated Layout into an invalid state when the URL can not be reached' do
+      Net::HTTP.should_receive(:get_response).with(@uri).and_raise('Random exception')
+
+      @layout.should_receive(:content=).with(nil)
+      @layout.should_receive(:generated=).with(false)
+
+      controller.send(:scrape_site, @scraper)
+    end
+  end
+
+end

data/spec/fixtures/scraper_inserts.yml

@@ -0,0 +1,3 @@
+yellowpages_canvas:
+  regex: <!-- CANVAS INSERTION POINT -->
+  content: <div id="canvas">Content</div>

data/spec/fixtures/scrapers.yml

@@ -0,0 +1,4 @@
+yellowpages:
+  url: http://stage-www.yellowpages.com
+  name: YP.com
+  description: YP.com Layout

data/spec/models/layout_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+describe Layout do
+  dataset :layouts
+  test_helper :validations
+
+  before :each do
+    @layout = @model = Layout.new(layout_params)
+  end
+
+  describe :scraped_layout do
+    before :each do
+      @layout.stub!(:scraped?).and_return(true)
+    end
+
+    it 'should be valid when flagged as generated' do
+      @layout.generated = true
+      @layout.should be_valid
+    end
+
+    it 'should not be valid when not flagged as generated' do
+      @layout.generated = false
+      @layout.should_not be_valid
+    end
+  end
+
+  describe :normal_layout do
+    before :each do
+      @layout.stub!(:scraped?).and_return(false)
+    end
+
+    it 'should be valid regardless of being generated' do
+      @layout.generated = true
+      @layout.should be_valid
+    end
+
+    it 'should be valid regardless of not being generated' do
+      @layout.generated = false
+      @layout.should be_valid
+    end
+  end
+
+end
+

data/spec/models/scraper_insert_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe ScraperInsert do
+  fixtures :scraper_inserts
+
+  before(:each) do
+    @scraper_insert = scraper_inserts(:yellowpages_canvas)
+  end
+
+  it "should be valid" do
+    @scraper_insert.should be_valid
+  end
+
+  it "should validate required fields" do
+    @scraper_insert.regex = nil
+    @scraper_insert.content = nil
+
+    @scraper_insert.should_not be_valid
+  end
+end

data/spec/models/scraper_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe Scraper do
+  fixtures :scrapers
+
+  before(:each) do
+    @scraper = scrapers(:yellowpages)
+  end
+
+  it "should be valid" do
+    @scraper.should be_valid
+  end
+
+  it "should validate required fields" do
+    @scraper.name = nil
+    @scraper.url = nil
+    @scraper.description = nil
+
+    @scraper.should_not be_valid
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,6 @@
+--colour
+--format
+progress
+--loadby
+mtime
+--reverse

data/spec/spec_helper.rb

@@ -0,0 +1,36 @@
+unless defined? RADIANT_ROOT
+  ENV["RAILS_ENV"] = "test"
+  case
+  when ENV["RADIANT_ENV_FILE"]
+    require ENV["RADIANT_ENV_FILE"]
+  when File.dirname(__FILE__) =~ %r{vendor/radiant/vendor/extensions}
+    require "#{File.expand_path(File.dirname(__FILE__) + "/../../../../../../")}/config/environment"
+  else
+    require "#{File.expand_path(File.dirname(__FILE__) + "/../../../../")}/config/environment"
+  end
+end
+require "#{RADIANT_ROOT}/spec/spec_helper"
+
+Dataset::Resolver.default << (File.dirname(__FILE__) + "/datasets")
+
+if File.directory?(File.dirname(__FILE__) + "/matchers")
+  Dir[File.dirname(__FILE__) + "/matchers/*.rb"].each {|file| require file }
+end
+
+Spec::Runner.configure do |config|
+  config.use_transactional_fixtures = true
+  config.use_instantiated_fixtures  = false
+  config.fixture_path = 'spec/fixtures'
+
+  # You can declare fixtures for each behaviour like this:
+  #   describe "...." do
+  #     fixtures :table_a, :table_b
+  #
+  # Alternatively, if you prefer to declare them only once, you can
+  # do so here, like so ...
+  #
+  #   config.global_fixtures = :table_a, :table_b
+  #
+  # If you declare global fixtures, be aware that they will be declared
+  # for all of your examples, even those that don't use them.
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: radiant-scraped_layouts-extension
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Charles Hudson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: radiant
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+description: Generates scraped layouts, allowing content to be inserted via regular
+  expressions.
+email:
+- charles.h.hudson@phobetron.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ./app/controllers/admin/scrapers_controller.rb
+- ./app/models/scraper.rb
+- ./app/models/scraper_insert.rb
+- ./app/views/admin/layouts/_actions_cell.html.haml
+- ./app/views/admin/layouts/_disabled_form.html.haml
+- ./app/views/admin/layouts/_title_cell.html.haml
+- ./app/views/admin/layouts/_warning.html.haml
+- ./app/views/admin/layouts/edit.html.haml
+- ./app/views/admin/scraper_inserts/_scraper_insert.html.haml
+- ./app/views/admin/scrapers/_form.html.haml
+- ./app/views/admin/scrapers/edit.html.haml
+- ./app/views/admin/scrapers/index.html.haml
+- ./app/views/admin/scrapers/new.html.haml
+- ./app/views/admin/scrapers/remove.html.haml
+- ./config/initializers/radiant_config.rb
+- ./config/locales/en.yml
+- ./config/routes.rb
+- ./cucumber.yml
+- ./db/migrate/20121015160755_create_scrapers.rb
+- ./db/migrate/20121015160953_create_scraper_inserts.rb
+- ./db/migrate/20121101183044_integrate_scraper_and_layout.rb
+- ./features/support/env.rb
+- ./features/support/paths.rb
+- ./Gemfile
+- ./Gemfile.lock
+- ./lib/radiant-scraped_layouts-extension.rb
+- ./lib/tasks/scraped_layouts_extension_tasks.rake
+- ./radiant-scraped_layouts-extension.gemspec
+- ./Rakefile
+- ./README.md
+- ./scraped_layouts_extension.rb
+- ./spec/controllers/admin/scrapers_controller_spec.rb
+- ./spec/fixtures/scraper_inserts.yml
+- ./spec/fixtures/scrapers.yml
+- ./spec/models/layout_spec.rb
+- ./spec/models/scraper_insert_spec.rb
+- ./spec/models/scraper_spec.rb
+- ./spec/spec.opts
+- ./spec/spec_helper.rb
+- spec/controllers/admin/scrapers_controller_spec.rb
+- spec/fixtures/scraper_inserts.yml
+- spec/fixtures/scrapers.yml
+- spec/models/layout_spec.rb
+- spec/models/scraper_insert_spec.rb
+- spec/models/scraper_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- features/support/env.rb
+- features/support/paths.rb
+homepage: https://github.com/yp-engineering/radiant-scraped_layouts-extension
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Scraped Layouts for Radiant CMS
+test_files:
+- spec/controllers/admin/scrapers_controller_spec.rb
+- spec/fixtures/scraper_inserts.yml
+- spec/fixtures/scrapers.yml
+- spec/models/layout_spec.rb
+- spec/models/scraper_insert_spec.rb
+- spec/models/scraper_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- features/support/env.rb
+- features/support/paths.rb
+has_rdoc: