radiant-scraped_layouts-extension 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +4 -0
- data/Gemfile.lock +69 -0
- data/README.md +7 -0
- data/Rakefile +109 -0
- data/app/controllers/admin/scrapers_controller.rb +74 -0
- data/app/models/scraper.rb +10 -0
- data/app/models/scraper_insert.rb +7 -0
- data/app/views/admin/layouts/_actions_cell.html.haml +3 -0
- data/app/views/admin/layouts/_disabled_form.html.haml +32 -0
- data/app/views/admin/layouts/_title_cell.html.haml +5 -0
- data/app/views/admin/layouts/_warning.html.haml +3 -0
- data/app/views/admin/layouts/edit.html.haml +15 -0
- data/app/views/admin/scraper_inserts/_scraper_insert.html.haml +10 -0
- data/app/views/admin/scrapers/_form.html.haml +31 -0
- data/app/views/admin/scrapers/edit.html.haml +7 -0
- data/app/views/admin/scrapers/index.html.haml +35 -0
- data/app/views/admin/scrapers/new.html.haml +4 -0
- data/app/views/admin/scrapers/remove.html.haml +16 -0
- data/config/initializers/radiant_config.rb +3 -0
- data/config/locales/en.yml +12 -0
- data/config/routes.rb +5 -0
- data/cucumber.yml +1 -0
- data/db/migrate/20121015160755_create_scrapers.rb +15 -0
- data/db/migrate/20121015160953_create_scraper_inserts.rb +16 -0
- data/db/migrate/20121101183044_integrate_scraper_and_layout.rb +11 -0
- data/features/support/env.rb +11 -0
- data/features/support/paths.rb +22 -0
- data/lib/radiant-scraped_layouts-extension.rb +8 -0
- data/lib/tasks/scraped_layouts_extension_tasks.rake +47 -0
- data/radiant-scraped_layouts-extension.gemspec +26 -0
- data/scraped_layouts_extension.rb +47 -0
- data/spec/controllers/admin/scrapers_controller_spec.rb +255 -0
- data/spec/fixtures/scraper_inserts.yml +3 -0
- data/spec/fixtures/scrapers.yml +4 -0
- data/spec/models/layout_spec.rb +44 -0
- data/spec/models/scraper_insert_spec.rb +20 -0
- data/spec/models/scraper_spec.rb +21 -0
- data/spec/spec.opts +6 -0
- data/spec/spec_helper.rb +36 -0
- metadata +122 -0
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
GEM
|
2
|
+
remote: https://rubygems.org/
|
3
|
+
specs:
|
4
|
+
RedCloth (4.2.9)
|
5
|
+
actionmailer (2.3.17)
|
6
|
+
actionpack (= 2.3.17)
|
7
|
+
actionpack (2.3.17)
|
8
|
+
activesupport (= 2.3.17)
|
9
|
+
rack (~> 1.1.0)
|
10
|
+
activerecord (2.3.17)
|
11
|
+
activesupport (= 2.3.17)
|
12
|
+
activeresource (2.3.17)
|
13
|
+
activesupport (= 2.3.17)
|
14
|
+
activesupport (2.3.17)
|
15
|
+
acts_as_tree (0.1.1)
|
16
|
+
chunky_png (1.2.7)
|
17
|
+
compass (0.12.2)
|
18
|
+
chunky_png (~> 1.2)
|
19
|
+
fssm (>= 0.2.7)
|
20
|
+
sass (~> 3.1)
|
21
|
+
compass-rails (1.0.3)
|
22
|
+
compass (>= 0.12.2, < 0.14)
|
23
|
+
delocalize (0.2.6)
|
24
|
+
fssm (0.2.10)
|
25
|
+
haml (3.1.8)
|
26
|
+
highline (1.6.16)
|
27
|
+
json (1.7.7)
|
28
|
+
rack (1.1.6)
|
29
|
+
rack-cache (1.2)
|
30
|
+
rack (>= 0.4)
|
31
|
+
radiant (1.1.3)
|
32
|
+
RedCloth (~> 4.2.0)
|
33
|
+
acts_as_tree (~> 0.1.1)
|
34
|
+
bundler (>= 1.0.0)
|
35
|
+
compass (~> 0.12.2)
|
36
|
+
compass-rails (~> 1.0.3)
|
37
|
+
delocalize (~> 0.2.3)
|
38
|
+
haml (~> 3.1.1)
|
39
|
+
highline (~> 1.6.10)
|
40
|
+
rack (~> 1.1.6)
|
41
|
+
rack-cache (~> 1.2)
|
42
|
+
radius (~> 0.7.3)
|
43
|
+
rails (~> 2.3.17)
|
44
|
+
rake (>= 0.8.7)
|
45
|
+
rdoc (>= 3.9.2)
|
46
|
+
stringex (~> 1.3.0)
|
47
|
+
tzinfo (~> 0.3.31)
|
48
|
+
will_paginate (~> 2.3.11)
|
49
|
+
radius (0.7.3)
|
50
|
+
rails (2.3.17)
|
51
|
+
actionmailer (= 2.3.17)
|
52
|
+
actionpack (= 2.3.17)
|
53
|
+
activerecord (= 2.3.17)
|
54
|
+
activeresource (= 2.3.17)
|
55
|
+
activesupport (= 2.3.17)
|
56
|
+
rake (>= 0.8.3)
|
57
|
+
rake (10.0.3)
|
58
|
+
rdoc (4.0.0)
|
59
|
+
json (~> 1.4)
|
60
|
+
sass (3.2.7)
|
61
|
+
stringex (1.3.3)
|
62
|
+
tzinfo (0.3.37)
|
63
|
+
will_paginate (2.3.16)
|
64
|
+
|
65
|
+
PLATFORMS
|
66
|
+
ruby
|
67
|
+
|
68
|
+
DEPENDENCIES
|
69
|
+
radiant (~> 1.1.0)
|
data/README.md
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
# Scraped Layouts
|
2
|
+
|
3
|
+
This extension allows a Radiant designer to pull a webpage from a given URL, replace portions of it using regular expressions, and save the outcome as a standard (but uneditable) Radiant layout.
|
4
|
+
|
5
|
+
---
|
6
|
+
|
7
|
+
Created by Charles Hudson for YP Intellectual Property, LLC (yellowpages.com).
|
data/Rakefile
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
# Determine where the RSpec plugin is by loading the boot
|
2
|
+
unless defined? RADIANT_ROOT
|
3
|
+
ENV["RAILS_ENV"] = "test"
|
4
|
+
case
|
5
|
+
when ENV["RADIANT_ENV_FILE"]
|
6
|
+
require File.dirname(ENV["RADIANT_ENV_FILE"]) + "/boot"
|
7
|
+
when File.dirname(__FILE__) =~ %r{vendor/radiant/vendor/extensions}
|
8
|
+
require "#{File.expand_path(File.dirname(__FILE__) + "/../../../../../")}/config/boot"
|
9
|
+
else
|
10
|
+
require "#{File.expand_path(File.dirname(__FILE__) + "/../../../")}/config/boot"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
require 'rake'
|
15
|
+
require 'rdoc/task'
|
16
|
+
require 'rake/testtask'
|
17
|
+
|
18
|
+
rspec_base = File.expand_path(RADIANT_ROOT + '/vendor/plugins/rspec/lib')
|
19
|
+
$LOAD_PATH.unshift(rspec_base) if File.exist?(rspec_base)
|
20
|
+
require 'spec/rake/spectask'
|
21
|
+
require 'cucumber'
|
22
|
+
require 'cucumber/rake/task'
|
23
|
+
|
24
|
+
# Cleanup the RADIANT_ROOT constant so specs will load the environment
|
25
|
+
Object.send(:remove_const, :RADIANT_ROOT)
|
26
|
+
|
27
|
+
extension_root = File.expand_path(File.dirname(__FILE__))
|
28
|
+
|
29
|
+
task :default => [:spec, :features]
|
30
|
+
task :stats => "spec:statsetup"
|
31
|
+
|
32
|
+
desc "Run all specs in spec directory"
|
33
|
+
Spec::Rake::SpecTask.new(:spec) do |t|
|
34
|
+
t.spec_opts = ['--options', "\"#{extension_root}/spec/spec.opts\""]
|
35
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
36
|
+
end
|
37
|
+
|
38
|
+
task :features => 'spec:integration'
|
39
|
+
|
40
|
+
namespace :spec do
|
41
|
+
desc "Run all specs in spec directory with RCov"
|
42
|
+
Spec::Rake::SpecTask.new(:rcov) do |t|
|
43
|
+
t.spec_opts = ['--options', "\"#{extension_root}/spec/spec.opts\""]
|
44
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
45
|
+
t.rcov = true
|
46
|
+
t.rcov_opts = ['--exclude', 'spec', '--rails']
|
47
|
+
end
|
48
|
+
|
49
|
+
desc "Print Specdoc for all specs"
|
50
|
+
Spec::Rake::SpecTask.new(:doc) do |t|
|
51
|
+
t.spec_opts = ["--format", "specdoc", "--dry-run"]
|
52
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
53
|
+
end
|
54
|
+
|
55
|
+
[:models, :controllers, :views, :helpers].each do |sub|
|
56
|
+
desc "Run the specs under spec/#{sub}"
|
57
|
+
Spec::Rake::SpecTask.new(sub) do |t|
|
58
|
+
t.spec_opts = ['--options', "\"#{extension_root}/spec/spec.opts\""]
|
59
|
+
t.spec_files = FileList["spec/#{sub}/**/*_spec.rb"]
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
desc "Run the Cucumber features"
|
64
|
+
Cucumber::Rake::Task.new(:integration) do |t|
|
65
|
+
t.fork = true
|
66
|
+
t.cucumber_opts = ['--format', (ENV['CUCUMBER_FORMAT'] || 'pretty')]
|
67
|
+
# t.feature_pattern = "#{extension_root}/features/**/*.feature"
|
68
|
+
t.profile = "default"
|
69
|
+
end
|
70
|
+
|
71
|
+
# Setup specs for stats
|
72
|
+
task :statsetup do
|
73
|
+
require 'code_statistics'
|
74
|
+
::STATS_DIRECTORIES << %w(Model\ specs spec/models)
|
75
|
+
::STATS_DIRECTORIES << %w(View\ specs spec/views)
|
76
|
+
::STATS_DIRECTORIES << %w(Controller\ specs spec/controllers)
|
77
|
+
::STATS_DIRECTORIES << %w(Helper\ specs spec/views)
|
78
|
+
::CodeStatistics::TEST_TYPES << "Model specs"
|
79
|
+
::CodeStatistics::TEST_TYPES << "View specs"
|
80
|
+
::CodeStatistics::TEST_TYPES << "Controller specs"
|
81
|
+
::CodeStatistics::TEST_TYPES << "Helper specs"
|
82
|
+
::STATS_DIRECTORIES.delete_if {|a| a[0] =~ /test/}
|
83
|
+
end
|
84
|
+
|
85
|
+
namespace :db do
|
86
|
+
namespace :fixtures do
|
87
|
+
desc "Load fixtures (from spec/fixtures) into the current environment's database. Load specific fixtures using FIXTURES=x,y"
|
88
|
+
task :load => :environment do
|
89
|
+
require 'active_record/fixtures'
|
90
|
+
ActiveRecord::Base.establish_connection(RAILS_ENV.to_sym)
|
91
|
+
(ENV['FIXTURES'] ? ENV['FIXTURES'].split(/,/) : Dir.glob(File.join(RAILS_ROOT, 'spec', 'fixtures', '*.{yml,csv}'))).each do |fixture_file|
|
92
|
+
Fixtures.create_fixtures('spec/fixtures', File.basename(fixture_file, '.*'))
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
desc 'Generate documentation for the scraped_layout extension.'
|
100
|
+
RDoc::Task.new(:rdoc) do |rdoc|
|
101
|
+
rdoc.rdoc_dir = 'rdoc'
|
102
|
+
rdoc.title = 'ScrapedLayoutExtension'
|
103
|
+
rdoc.options << '--line-numbers' << '--inline-source'
|
104
|
+
rdoc.rdoc_files.include('README')
|
105
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
106
|
+
end
|
107
|
+
|
108
|
+
# Load any custom rakefiles for extension
|
109
|
+
Dir[File.dirname(__FILE__) + '/tasks/*.rake'].sort.each { |f| require f }
|
@@ -0,0 +1,74 @@
|
|
1
|
+
class Admin::ScrapersController < Admin::ResourceController
|
2
|
+
|
3
|
+
def new
|
4
|
+
@scraper = Scraper.new
|
5
|
+
@scraper.scraper_inserts << ScraperInsert.new
|
6
|
+
response_for :new
|
7
|
+
end
|
8
|
+
|
9
|
+
def create
|
10
|
+
@scraper = Scraper.new(params[:scraper])
|
11
|
+
@scraper.layout = Layout.new({ :name => "#{@scraper.name} (scraped)" })
|
12
|
+
|
13
|
+
save_scraper @scraper
|
14
|
+
end
|
15
|
+
|
16
|
+
def update
|
17
|
+
@scraper = Scraper.find(params[:id])
|
18
|
+
@scraper.update_attributes(params[:scraper])
|
19
|
+
|
20
|
+
save_scraper @scraper, :edit
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def save_scraper(scraper, action = :new)
|
26
|
+
if manage_inserts?(scraper) || !scraper.save
|
27
|
+
unless scraper.errors.empty?
|
28
|
+
flash[:error] = scraper.errors.full_messages.join(' ')
|
29
|
+
end
|
30
|
+
|
31
|
+
render :action => action
|
32
|
+
else
|
33
|
+
scrape_site(scraper)
|
34
|
+
|
35
|
+
flash[:notice] = "Scraper successfully #{action == :new ? 'created' : 'updated'}"
|
36
|
+
|
37
|
+
redirect_to admin_scrapers_path
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def manage_inserts?(scraper)
|
42
|
+
managed = false
|
43
|
+
scraper.scraper_inserts.each do |insert|
|
44
|
+
if insert.remove
|
45
|
+
scraper.scraper_inserts.delete(insert)
|
46
|
+
managed = true
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
if params[:add_insert]
|
51
|
+
scraper.scraper_inserts << ScraperInsert.new
|
52
|
+
scraper.scraper_inserts.last.errors.clear # TODO: manage this separately so Rails won't try to save it automatically.
|
53
|
+
managed = true
|
54
|
+
end
|
55
|
+
|
56
|
+
managed
|
57
|
+
end
|
58
|
+
|
59
|
+
def scrape_site(scraper)
|
60
|
+
begin
|
61
|
+
html = Net::HTTP.get_response(URI(scraper.url)).body
|
62
|
+
|
63
|
+
scraper.scraper_inserts.each do |insert|
|
64
|
+
html.gsub!(Regexp.new(insert.regex), insert.content)
|
65
|
+
end
|
66
|
+
|
67
|
+
scraper.layout.content = html
|
68
|
+
scraper.layout.generated = true
|
69
|
+
rescue
|
70
|
+
scraper.layout.content = nil
|
71
|
+
scraper.layout.generated = false
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
class Scraper < ActiveRecord::Base
|
2
|
+
has_many :scraper_inserts, :autosave => true, :dependent => :destroy
|
3
|
+
|
4
|
+
belongs_to :layout, :autosave => true, :dependent => :destroy
|
5
|
+
|
6
|
+
validates_uniqueness_of :name, :case_sensitive => true
|
7
|
+
validates_presence_of :name, :url, :description
|
8
|
+
|
9
|
+
accepts_nested_attributes_for :scraper_inserts
|
10
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
- form_for [:admin, @layout] do |f|
|
2
|
+
= f.hidden_field :lock_version
|
3
|
+
= render_region :form_top, :locals => {:f => f}
|
4
|
+
.form_area
|
5
|
+
- render_region :form, :locals => {:f => f} do |form|
|
6
|
+
- form.edit_title do
|
7
|
+
%p.title
|
8
|
+
= f.label :name, t('name')
|
9
|
+
= f.text_field :name, :maxlength => 100, :class => "disabled textbox", :disabled => true
|
10
|
+
- form.edit_extended_metadata do
|
11
|
+
.drawer
|
12
|
+
.drawer_contents#attributes
|
13
|
+
%table.fieldset
|
14
|
+
%tr
|
15
|
+
%th.label= f.label :content_type, t('content_type')
|
16
|
+
%td.field= f.text_field :content_type, :maxlength => 40, :class => "disabled textbox", :disabled => true
|
17
|
+
= render_region :extended_metadata, :locals => {:f => f}
|
18
|
+
.drawer_handle
|
19
|
+
%a.toggle{:href=>'#attributes', :rel=>"toggle[attributes]", :class=>"#{(meta_errors? ? 'less' : 'more')}"}= meta_label
|
20
|
+
- form.edit_content do
|
21
|
+
%p.content
|
22
|
+
= f.label :layout_content, "Body"
|
23
|
+
~ f.text_area :content, :size => "40x20", :style => "width: 100%", :class => "disabled textarea large", :disabled => true
|
24
|
+
- render_region :form_bottom, :locals => {:f => f} do |form_bottom|
|
25
|
+
- form_bottom.reference_links do
|
26
|
+
%p{ :style => 'padding: 2ex 1em; border: 6px outset #f00; text-align: center; color: #c00; font-weight: bold; background: #fff;' }
|
27
|
+
This layout has been generated and may not be directly edited.
|
28
|
+
- form_bottom.edit_buttons do
|
29
|
+
.buttons
|
30
|
+
= link_to '« Return to Index', admin_layouts_url
|
31
|
+
- form_bottom.edit_timestamp do
|
32
|
+
= updated_stamp @layout
|
@@ -0,0 +1,15 @@
|
|
1
|
+
- @page_title = @layout.name + ' ' + t('layout') + ' - ' + default_page_title
|
2
|
+
|
3
|
+
- render_region :main do |main|
|
4
|
+
- if @layout.scraped?
|
5
|
+
- main.edit_header do
|
6
|
+
%h1= 'Scraped Layout'
|
7
|
+
- main.edit_form do
|
8
|
+
= render :partial => 'disabled_form'
|
9
|
+
- else
|
10
|
+
- main.edit_header do
|
11
|
+
%h1= t('edit_layout')
|
12
|
+
- main.edit_form do
|
13
|
+
= render :partial => 'form'
|
14
|
+
|
15
|
+
- content_for 'page_scripts', layout_edit_javascripts
|
@@ -0,0 +1,10 @@
|
|
1
|
+
%hr
|
2
|
+
|
3
|
+
%p
|
4
|
+
= f.label :regex
|
5
|
+
= f.text_field :regex, :class => 'textbox', :size => 90
|
6
|
+
%p
|
7
|
+
= f.label :content
|
8
|
+
= f.text_area :content, :class => 'textarea', :style => 'width: 90%;', :cols => 90, :rows => 10
|
9
|
+
%p
|
10
|
+
= f.submit 'Remove Insert', :name => "scraper[scraper_inserts_attributes][#{f.object_name.gsub(/[^0-9]+/,'')}][remove]"
|
@@ -0,0 +1,31 @@
|
|
1
|
+
- form_for [:admin, @scraper] do |f|
|
2
|
+
= render_region :form_top, :locals => {:f => f}
|
3
|
+
|
4
|
+
- render_region :form do |form|
|
5
|
+
- form.edit_title do
|
6
|
+
%p
|
7
|
+
= f.label :name
|
8
|
+
= f.text_field :name, :class => 'textbox', :size => 90
|
9
|
+
- form.edit_url do
|
10
|
+
%p
|
11
|
+
%label{:for => "scraper_url"} URL
|
12
|
+
= f.text_field :url, :class => 'textbox', :size => 90
|
13
|
+
- form.edit_description do
|
14
|
+
%p
|
15
|
+
%label{:for => "scraper_description"} Description
|
16
|
+
= f.text_field :description, :class => 'textbox', :size => 90
|
17
|
+
|
18
|
+
- form.edit_inserts do
|
19
|
+
- f.fields_for :scraper_inserts do |insert_form|
|
20
|
+
= render :partial => 'admin/scraper_inserts/scraper_insert', :locals => { :f => insert_form }
|
21
|
+
|
22
|
+
%hr
|
23
|
+
|
24
|
+
= submit_tag 'Add Another Insert', :name => 'add_insert'
|
25
|
+
|
26
|
+
- render_region :form_bottom do |form_bottom|
|
27
|
+
- form_bottom.edit_buttons do
|
28
|
+
%p.buttons
|
29
|
+
= save_model_button(@scraper)
|
30
|
+
or
|
31
|
+
= link_to "Cancel", admin_scrapers_path
|
@@ -0,0 +1,35 @@
|
|
1
|
+
- @page_title = t('scraped_layouts') + ' - ' + default_page_title
|
2
|
+
|
3
|
+
.outset
|
4
|
+
= render_region :top
|
5
|
+
%table.index#scrapers
|
6
|
+
%thead
|
7
|
+
%tr
|
8
|
+
- render_region :thead do |thead|
|
9
|
+
- thead.title_header do
|
10
|
+
%th.name
|
11
|
+
= t('scraper')
|
12
|
+
- thead.actions_header do
|
13
|
+
%th.actions{:style=>"width:9em"}
|
14
|
+
= t('modify')
|
15
|
+
%tbody
|
16
|
+
- if @scrapers.any?
|
17
|
+
- @scrapers.each do |scraper|
|
18
|
+
%tr[scraper]
|
19
|
+
- render_region :tbody, :locals => {:scraper => scraper} do |tbody|
|
20
|
+
- tbody.title_cell do
|
21
|
+
%td.name
|
22
|
+
= link_to image('layout', :alt => '') + ' ' + scraper.name, edit_admin_scraper_url(scraper)
|
23
|
+
- tbody.actions_cell do
|
24
|
+
%td.actions
|
25
|
+
= link_to image('minus') + ' ' + t('remove'), remove_admin_scraper_url(scraper), :class => "action"
|
26
|
+
- else
|
27
|
+
%tr
|
28
|
+
%td.empty{:colspan => admin.scrapers.index.tbody.length}= t('no_scrapers')
|
29
|
+
|
30
|
+
- render_region :bottom do |bottom|
|
31
|
+
- bottom.new_button do
|
32
|
+
#actions
|
33
|
+
= pagination_for(@scrapers)
|
34
|
+
%ul
|
35
|
+
%li= link_to image('plus') + " " + t('new_scraper'), new_admin_scraper_url, :class => 'action_button'
|
@@ -0,0 +1,16 @@
|
|
1
|
+
%h1= t('remove_scraper')
|
2
|
+
|
3
|
+
%p= t('text.scrapers.remove_warning')
|
4
|
+
|
5
|
+
%table.index#scrapers
|
6
|
+
%tbody
|
7
|
+
%tr.node.level_1
|
8
|
+
%td.scraper
|
9
|
+
= image('layout', :alt => '')
|
10
|
+
= h @scraper.name
|
11
|
+
|
12
|
+
- form_for [:admin, @scraper], :html => { :method => 'delete', 'data-onsubmit_status'=>"Removing scraper…" } do
|
13
|
+
.buttons
|
14
|
+
%input.button{:type => "submit", :value => t('delete_scraper') }/
|
15
|
+
= t('or')
|
16
|
+
= link_to t('cancel'), admin_scrapers_url
|
@@ -0,0 +1,12 @@
|
|
1
|
+
---
|
2
|
+
en:
|
3
|
+
scraped_layout: Scraped Layout
|
4
|
+
scraped_layouts: Scraped Layouts
|
5
|
+
scraper: Scraper
|
6
|
+
no_scrapers: No Scraped Layouts
|
7
|
+
new_scraper: New Scraped Layout
|
8
|
+
delete_scraper: Delete Scraper
|
9
|
+
remove_scraper: Remove Scraper
|
10
|
+
text:
|
11
|
+
scrapers:
|
12
|
+
remove_warning: 'Are you sure you want to <strong class="warning">permanently remove</strong> the following layout?'
|
data/config/routes.rb
ADDED
data/cucumber.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
default: --format progress features --tags ~@proposed,~@in_progress
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class CreateScrapers < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
create_table :scrapers do |t|
|
4
|
+
t.string :name, :unique => true
|
5
|
+
t.string :url
|
6
|
+
t.text :description
|
7
|
+
|
8
|
+
t.timestamps
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.down
|
13
|
+
drop_table :scrapers
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
class CreateScraperInserts < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
create_table :scraper_inserts do |t|
|
4
|
+
t.string :regex
|
5
|
+
t.text :content
|
6
|
+
|
7
|
+
t.references :scraper
|
8
|
+
|
9
|
+
t.timestamps
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.down
|
14
|
+
drop_table :scraper_inserts
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# Sets up the Rails environment for Cucumber
|
2
|
+
ENV["RAILS_ENV"] = "test"
|
3
|
+
# Extension root
|
4
|
+
extension_env = File.expand_path(File.dirname(__FILE__) + '/../../../../../config/environment')
|
5
|
+
require extension_env+'.rb'
|
6
|
+
|
7
|
+
Dir.glob(File.join(RADIANT_ROOT, "features", "**", "*.rb")).each {|step| require step unless step =~ /datasets_loader\.rb$/}
|
8
|
+
|
9
|
+
Cucumber::Rails::World.class_eval do
|
10
|
+
dataset :scraped_layout
|
11
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module NavigationHelpers
|
2
|
+
|
3
|
+
# Extend the standard PathMatchers with your own paths
|
4
|
+
# to be used in your features.
|
5
|
+
#
|
6
|
+
# The keys and values here may be used in your standard web steps
|
7
|
+
# Using:
|
8
|
+
#
|
9
|
+
# When I go to the "scraped_layout" admin page
|
10
|
+
#
|
11
|
+
# would direct the request to the path you provide in the value:
|
12
|
+
#
|
13
|
+
# admin_scraped_layout_path
|
14
|
+
#
|
15
|
+
PathMatchers = {} unless defined?(PathMatchers)
|
16
|
+
PathMatchers.merge!({
|
17
|
+
# /scraped_layout/i => 'admin_scraped_layout_path'
|
18
|
+
})
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
World(NavigationHelpers)
|
@@ -0,0 +1,8 @@
|
|
1
|
+
module RadiantScrapedLayoutsExtension
|
2
|
+
VERSION = "1.0.0"
|
3
|
+
SUMMARY = "Scraped Layouts for Radiant CMS"
|
4
|
+
DESCRIPTION = "Generates scraped layouts, allowing content to be inserted via regular expressions."
|
5
|
+
URL = "https://github.com/yp-engineering/radiant-scraped_layouts-extension"
|
6
|
+
AUTHORS = ["Charles Hudson"]
|
7
|
+
EMAIL = ["charles.h.hudson@phobetron.com"]
|
8
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
namespace :radiant do
|
2
|
+
namespace :extensions do
|
3
|
+
namespace :scraped_layouts do
|
4
|
+
|
5
|
+
desc "Runs the migration of the Scraped Layout extension"
|
6
|
+
task :migrate => :environment do
|
7
|
+
require 'radiant/extension_migrator'
|
8
|
+
if ENV["VERSION"]
|
9
|
+
ScrapedLayoutsExtension.migrator.migrate(ENV["VERSION"].to_i)
|
10
|
+
Rake::Task['db:schema:dump'].invoke
|
11
|
+
else
|
12
|
+
ScrapedLayoutsExtension.migrator.migrate
|
13
|
+
Rake::Task['db:schema:dump'].invoke
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
desc "Copies public assets of the Scraped Layout to the instance public/ directory."
|
18
|
+
task :update => :environment do
|
19
|
+
is_svn_or_dir = proc {|path| path =~ /\.svn/ || File.directory?(path) }
|
20
|
+
puts "Copying assets from ScrapedLayoutsExtension"
|
21
|
+
Dir[ScrapedLayoutsExtension.root + "/public/**/*"].reject(&is_svn_or_dir).each do |file|
|
22
|
+
path = file.sub(ScrapedLayoutsExtension.root, '')
|
23
|
+
directory = File.dirname(path)
|
24
|
+
mkdir_p RAILS_ROOT + directory, :verbose => false
|
25
|
+
cp file, RAILS_ROOT + path, :verbose => false
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
desc "Syncs all available translations for this ext to the English ext master"
|
30
|
+
task :sync => :environment do
|
31
|
+
# The main translation root, basically where English is kept
|
32
|
+
language_root = ScrapedLayoutsExtension.root + "/config/locales"
|
33
|
+
words = TranslationSupport.get_translation_keys(language_root)
|
34
|
+
|
35
|
+
Dir["#{language_root}/*.yml"].each do |filename|
|
36
|
+
next if filename.match('_available_tags')
|
37
|
+
basename = File.basename(filename, '.yml')
|
38
|
+
puts "Syncing #{basename}"
|
39
|
+
(comments, other) = TranslationSupport.read_file(filename, basename)
|
40
|
+
words.each { |k,v| other[k] ||= words[k] } # Initializing hash variable as empty if it does not exist
|
41
|
+
other.delete_if { |k,v| !words[k] } # Remove if not defined in en.yml
|
42
|
+
TranslationSupport.write_file(filename, basename, comments, other)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "radiant-scraped_layouts-extension"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "radiant-scraped_layouts-extension"
|
7
|
+
s.version = RadiantScrapedLayoutsExtension::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = RadiantScrapedLayoutsExtension::AUTHORS
|
10
|
+
s.email = RadiantScrapedLayoutsExtension::EMAIL
|
11
|
+
s.homepage = RadiantScrapedLayoutsExtension::URL
|
12
|
+
s.summary = RadiantScrapedLayoutsExtension::SUMMARY
|
13
|
+
s.description = RadiantScrapedLayoutsExtension::DESCRIPTION
|
14
|
+
|
15
|
+
s.add_dependency "radiant", "~> 1.1.0"
|
16
|
+
|
17
|
+
ignores = if File.exist?('.gitignore')
|
18
|
+
File.read('.gitignore').split("\n").collect { |a| d = Dir["./#{a}/**/*"]; (d.empty? ? a : d) }.flatten
|
19
|
+
else
|
20
|
+
[]
|
21
|
+
end
|
22
|
+
|
23
|
+
s.files = Dir['./**/*'] - ignores
|
24
|
+
s.test_files = Dir['test/**/*','spec/**/*','features/**/*'] - ignores
|
25
|
+
s.require_paths = ["lib"]
|
26
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# Uncomment this if you reference any of your controllers in activate
|
2
|
+
# require_dependency "application_controller"
|
3
|
+
require "radiant-scraped_layouts-extension"
|
4
|
+
|
5
|
+
class ScrapedLayoutsExtension < Radiant::Extension
|
6
|
+
version RadiantScrapedLayoutsExtension::VERSION
|
7
|
+
description RadiantScrapedLayoutsExtension::DESCRIPTION
|
8
|
+
url RadiantScrapedLayoutsExtension::URL
|
9
|
+
|
10
|
+
# See your config/routes.rb file in this extension to define custom routes
|
11
|
+
|
12
|
+
extension_config do |config|
|
13
|
+
# config is the Radiant.configuration object
|
14
|
+
end
|
15
|
+
|
16
|
+
def activate
|
17
|
+
Layout.class_eval do
|
18
|
+
has_one :scraper
|
19
|
+
|
20
|
+
attr_accessor :generated
|
21
|
+
validates_presence_of :generated, :if => :scraped?, :message => 'layouts may only be created or edited within the Scraped Layouts section, and when supplied with a working URL.'
|
22
|
+
|
23
|
+
def scraped?
|
24
|
+
!scraper.nil?
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
tab 'Design' do
|
29
|
+
add_item "Scraped Layouts", "/admin/scrapers", :after => "Layouts"
|
30
|
+
end
|
31
|
+
|
32
|
+
Radiant::AdminUI.class_eval { attr_accessor :scrapers }
|
33
|
+
admin.scrapers = OpenStruct.new.tap do |scrapers|
|
34
|
+
scrapers.index = Radiant::AdminUI::RegionSet.new do |index|
|
35
|
+
index.thead.concat %w{title_header actions_header}
|
36
|
+
index.tbody.concat %w{title_cell actions_cell}
|
37
|
+
index.bottom.concat %w{new_button}
|
38
|
+
end
|
39
|
+
scrapers.edit = Radiant::AdminUI::RegionSet.new do |edit|
|
40
|
+
edit.main.concat %w{edit_header edit_form}
|
41
|
+
edit.form.concat %w{edit_title edit_url edit_description edit_inserts}
|
42
|
+
edit.form_bottom.concat %w{edit_buttons}
|
43
|
+
end
|
44
|
+
scrapers.new = scrapers.edit
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,255 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Admin::ScrapersController do
|
4
|
+
dataset :users
|
5
|
+
|
6
|
+
before :each do
|
7
|
+
login_as :designer
|
8
|
+
|
9
|
+
@scraper_params = {
|
10
|
+
:name => 'YP.com',
|
11
|
+
:url => 'http://www.yellowpages.com/pages/partner_template',
|
12
|
+
:description => 'YP.com Template',
|
13
|
+
:scraper_inserts_attributes => []
|
14
|
+
}
|
15
|
+
@scraper_insert_params = {
|
16
|
+
:regex => '^.*$',
|
17
|
+
:content => 'New'
|
18
|
+
}
|
19
|
+
@scraper_params[:scraper_inserts_attributes] << @scraper_insert_params
|
20
|
+
|
21
|
+
@scraper = mock_model(Scraper)
|
22
|
+
@scraper.stub!(:url).and_return(@scraper_params[:url])
|
23
|
+
@scraper.stub!(:name).and_return(@scraper_params[:name])
|
24
|
+
@scraper.stub!(:description).and_return(@scraper_params[:description])
|
25
|
+
end
|
26
|
+
|
27
|
+
describe :new do
|
28
|
+
it 'initializes a new Scraper' do
|
29
|
+
get :new
|
30
|
+
|
31
|
+
assigns(:scraper).should be_new_record
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'adds a new ScraperInsert to the new Scraper' do
|
35
|
+
get :new
|
36
|
+
|
37
|
+
assigns(:scraper).scraper_inserts[0].should be_new_record
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
describe :create do
|
42
|
+
before :each do
|
43
|
+
@layout = mock_model(Layout)
|
44
|
+
Layout.stub!(:new).and_return(@layout)
|
45
|
+
|
46
|
+
@scraper.stub!(:layout=).and_return(@layout)
|
47
|
+
Scraper.stub!(:new).and_return(@scraper)
|
48
|
+
|
49
|
+
controller.stub!(:save_scraper).and_return(true)
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'creates a new associated Layout with a name based on the Scraper\'s name' do
|
53
|
+
Layout.should_receive(:new).with(:name => 'YP.com (scraped)')
|
54
|
+
|
55
|
+
post :create, :scraper => @scraper_params
|
56
|
+
end
|
57
|
+
|
58
|
+
it 'calls save_scraper method' do
|
59
|
+
controller.should_receive(:save_scraper)
|
60
|
+
|
61
|
+
post :create, :scraper => @scraper_params
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
describe :update do
|
66
|
+
before :each do
|
67
|
+
@scraper.stub!(:update_attributes).and_return(true)
|
68
|
+
Scraper.stub!(:find).and_return(@scraper)
|
69
|
+
Scraper.stub!(:update_attributes).and_return(true)
|
70
|
+
|
71
|
+
controller.stub!(:save_scraper).and_return(true)
|
72
|
+
end
|
73
|
+
|
74
|
+
it 'updates the existing Scraper with form params' do
|
75
|
+
@scraper.should_receive(:update_attributes).with(anything)
|
76
|
+
|
77
|
+
put :update, :id => 0, :scraper => @scraper_params
|
78
|
+
end
|
79
|
+
|
80
|
+
it 'calls save_scraper method with previous action name' do
|
81
|
+
controller.should_receive(:save_scraper).with(@scraper, :edit)
|
82
|
+
|
83
|
+
put :update, :id => 0, :scraper => @scraper_params
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
describe :save_scraper do
|
88
|
+
before :each do
|
89
|
+
controller.stub!(:scrape_site).and_return(true)
|
90
|
+
controller.stub!(:redirect_to)
|
91
|
+
controller.stub!(:render)
|
92
|
+
controller.stub!(:admin_scrapers_path)
|
93
|
+
end
|
94
|
+
|
95
|
+
it 'calls manage_inserts?' do
|
96
|
+
@scraper.stub!(:save).and_return(true)
|
97
|
+
|
98
|
+
controller.should_receive(:manage_inserts?)
|
99
|
+
|
100
|
+
controller.send(:save_scraper, @scraper)
|
101
|
+
end
|
102
|
+
|
103
|
+
it 'adds flash notice on successful save or update' do
|
104
|
+
@scraper.stub!(:save).and_return(true)
|
105
|
+
controller.stub!(:manage_inserts?).and_return(false)
|
106
|
+
|
107
|
+
controller.send(:save_scraper, @scraper)
|
108
|
+
|
109
|
+
flash[:error].should be_nil
|
110
|
+
flash[:notice].should == 'Scraper successfully created'
|
111
|
+
end
|
112
|
+
|
113
|
+
it 'redirects to Scraped Layouts index on successful save or update' do
|
114
|
+
controller.stub!(:manage_inserts?).and_return(false)
|
115
|
+
@scraper.stub!(:save).and_return(true)
|
116
|
+
|
117
|
+
controller.should_receive(:redirect_to)
|
118
|
+
controller.should_receive(:admin_scrapers_path)
|
119
|
+
|
120
|
+
controller.send(:save_scraper, @scraper)
|
121
|
+
end
|
122
|
+
|
123
|
+
it 'renders to previous action when inserts are managed (added or removed)' do
|
124
|
+
controller.stub!(:manage_inserts?).and_return(true)
|
125
|
+
|
126
|
+
@scraper.stub!(:errors).and_return({})
|
127
|
+
errors = @scraper.errors
|
128
|
+
errors.stub!(:empty?).and_return(true)
|
129
|
+
|
130
|
+
controller.should_receive(:render).with(:action => :new)
|
131
|
+
|
132
|
+
controller.send(:save_scraper, @scraper)
|
133
|
+
end
|
134
|
+
|
135
|
+
it 'adds flash error when save is unsuccessful' do
|
136
|
+
controller.stub!(:manage_inserts?).and_return(false)
|
137
|
+
@scraper.stub!(:save).and_return(false)
|
138
|
+
|
139
|
+
@scraper.stub!(:errors).and_return({})
|
140
|
+
errors = @scraper.errors
|
141
|
+
errors.stub!(:empty?).and_return(false)
|
142
|
+
errors.stub!(:full_messages).and_return(['Error'])
|
143
|
+
|
144
|
+
controller.send(:save_scraper, @scraper)
|
145
|
+
|
146
|
+
flash[:notice].should be_nil
|
147
|
+
flash[:error].should == 'Error'
|
148
|
+
end
|
149
|
+
|
150
|
+
it 'renders previous action when save is unsuccessful' do
|
151
|
+
controller.stub!(:manage_inserts?).and_return(false)
|
152
|
+
@scraper.stub!(:save).and_return(false)
|
153
|
+
|
154
|
+
@scraper.stub!(:errors).and_return({})
|
155
|
+
errors = @scraper.errors
|
156
|
+
errors.stub!(:empty?).and_return(true)
|
157
|
+
|
158
|
+
controller.should_receive(:render).with(:action => :edit)
|
159
|
+
|
160
|
+
controller.send(:save_scraper, @scraper, :edit)
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
describe :manage_inserts? do
|
165
|
+
before :each do
|
166
|
+
@scraper_insert = mock_model(ScraperInsert)
|
167
|
+
@scraper_insert.stub!(:regex).and_return(@scraper_insert_params[:regex])
|
168
|
+
@scraper_insert.stub!(:content).and_return(@scraper_insert_params[:content])
|
169
|
+
|
170
|
+
@scraper_inserts = [@scraper_insert]
|
171
|
+
@scraper.stub!(:scraper_inserts).and_return(@scraper_inserts)
|
172
|
+
|
173
|
+
controller.stub!(:scrape_site).and_return(true)
|
174
|
+
end
|
175
|
+
|
176
|
+
it 'removes inserts that are marked for removal' do
|
177
|
+
@scraper_insert.stub!(:remove).and_return(true)
|
178
|
+
|
179
|
+
@scraper_inserts.should_receive(:delete).with(@scraper_insert)
|
180
|
+
|
181
|
+
controller.send(:manage_inserts?, @scraper).should be_true
|
182
|
+
end
|
183
|
+
|
184
|
+
it 'adds inserts when requested' do
|
185
|
+
@scraper_insert.stub!(:remove).and_return(false)
|
186
|
+
@scraper_insert.stub!(:errors).and_return({})
|
187
|
+
errors = @scraper_insert.errors
|
188
|
+
errors.stub!(:clear).and_return(true)
|
189
|
+
|
190
|
+
@scraper_inserts.should_receive(:<<)
|
191
|
+
|
192
|
+
controller.params[:add_insert] = true
|
193
|
+
controller.send(:manage_inserts?, @scraper).should be_true
|
194
|
+
end
|
195
|
+
|
196
|
+
it 'returns false when inserts have not been managed' do
|
197
|
+
@scraper_insert.stub!(:remove).and_return(false)
|
198
|
+
|
199
|
+
controller.send(:manage_inserts?, @scraper).should be_false
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
describe :scrape_site do
|
204
|
+
before :each do
|
205
|
+
@scraper_insert = mock_model(ScraperInsert)
|
206
|
+
@scraper_insert.stub!(:regex).and_return(@scraper_insert_params[:regex])
|
207
|
+
@scraper_insert.stub!(:content).and_return(@scraper_insert_params[:content])
|
208
|
+
|
209
|
+
@scraper_inserts = [@scraper_insert]
|
210
|
+
@scraper.stub!(:scraper_inserts).and_return(@scraper_inserts)
|
211
|
+
|
212
|
+
@layout = mock_model(Layout)
|
213
|
+
@layout.stub!(:content=)
|
214
|
+
@layout.stub!(:generated=)
|
215
|
+
@scraper.stub!(:layout).and_return(@layout)
|
216
|
+
|
217
|
+
@uri = URI(@scraper.url)
|
218
|
+
@response = mock(Object)
|
219
|
+
@response.stub!(:body).and_return('Text')
|
220
|
+
|
221
|
+
@regex = Regexp.new(@scraper_insert.regex)
|
222
|
+
Regexp.stub!(:new).and_return(@regex)
|
223
|
+
|
224
|
+
controller.stub!(:URI).and_return(@uri)
|
225
|
+
end
|
226
|
+
|
227
|
+
it 'stuffs content from an external URL into a Scraper\'s associalte Layout' do
|
228
|
+
Net::HTTP.should_receive(:get_response).with(@uri).and_return(@response)
|
229
|
+
|
230
|
+
@layout.should_receive(:content=).with(anything)
|
231
|
+
@layout.should_receive(:generated=).with(true)
|
232
|
+
|
233
|
+
controller.send(:scrape_site, @scraper)
|
234
|
+
end
|
235
|
+
|
236
|
+
it 'inserts content via the given regex into the HTML returned by the URL' do
|
237
|
+
Net::HTTP.should_receive(:get_response).with(@uri).and_return(@response)
|
238
|
+
|
239
|
+
@layout.should_receive(:content=).with('New')
|
240
|
+
@layout.should_receive(:generated=).with(true)
|
241
|
+
|
242
|
+
controller.send(:scrape_site, @scraper)
|
243
|
+
end
|
244
|
+
|
245
|
+
it 'places the Scraper\'s associated Layout into an invalid state when the URL can not be reached' do
|
246
|
+
Net::HTTP.should_receive(:get_response).with(@uri).and_raise('Random exception')
|
247
|
+
|
248
|
+
@layout.should_receive(:content=).with(nil)
|
249
|
+
@layout.should_receive(:generated=).with(false)
|
250
|
+
|
251
|
+
controller.send(:scrape_site, @scraper)
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Layout do
|
4
|
+
dataset :layouts
|
5
|
+
test_helper :validations
|
6
|
+
|
7
|
+
before :each do
|
8
|
+
@layout = @model = Layout.new(layout_params)
|
9
|
+
end
|
10
|
+
|
11
|
+
describe :scraped_layout do
|
12
|
+
before :each do
|
13
|
+
@layout.stub!(:scraped?).and_return(true)
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should be valid when flagged as generated' do
|
17
|
+
@layout.generated = true
|
18
|
+
@layout.should be_valid
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'should not be valid when not flagged as generated' do
|
22
|
+
@layout.generated = false
|
23
|
+
@layout.should_not be_valid
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
describe :normal_layout do
|
28
|
+
before :each do
|
29
|
+
@layout.stub!(:scraped?).and_return(false)
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'should be valid regardless of being generated' do
|
33
|
+
@layout.generated = true
|
34
|
+
@layout.should be_valid
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'should be valid regardless of not being generated' do
|
38
|
+
@layout.generated = false
|
39
|
+
@layout.should be_valid
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe ScraperInsert do
|
4
|
+
fixtures :scraper_inserts
|
5
|
+
|
6
|
+
before(:each) do
|
7
|
+
@scraper_insert = scraper_inserts(:yellowpages_canvas)
|
8
|
+
end
|
9
|
+
|
10
|
+
it "should be valid" do
|
11
|
+
@scraper_insert.should be_valid
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should validate required fields" do
|
15
|
+
@scraper_insert.regex = nil
|
16
|
+
@scraper_insert.content = nil
|
17
|
+
|
18
|
+
@scraper_insert.should_not be_valid
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Scraper do
|
4
|
+
fixtures :scrapers
|
5
|
+
|
6
|
+
before(:each) do
|
7
|
+
@scraper = scrapers(:yellowpages)
|
8
|
+
end
|
9
|
+
|
10
|
+
it "should be valid" do
|
11
|
+
@scraper.should be_valid
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should validate required fields" do
|
15
|
+
@scraper.name = nil
|
16
|
+
@scraper.url = nil
|
17
|
+
@scraper.description = nil
|
18
|
+
|
19
|
+
@scraper.should_not be_valid
|
20
|
+
end
|
21
|
+
end
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
unless defined? RADIANT_ROOT
|
2
|
+
ENV["RAILS_ENV"] = "test"
|
3
|
+
case
|
4
|
+
when ENV["RADIANT_ENV_FILE"]
|
5
|
+
require ENV["RADIANT_ENV_FILE"]
|
6
|
+
when File.dirname(__FILE__) =~ %r{vendor/radiant/vendor/extensions}
|
7
|
+
require "#{File.expand_path(File.dirname(__FILE__) + "/../../../../../../")}/config/environment"
|
8
|
+
else
|
9
|
+
require "#{File.expand_path(File.dirname(__FILE__) + "/../../../../")}/config/environment"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
require "#{RADIANT_ROOT}/spec/spec_helper"
|
13
|
+
|
14
|
+
Dataset::Resolver.default << (File.dirname(__FILE__) + "/datasets")
|
15
|
+
|
16
|
+
if File.directory?(File.dirname(__FILE__) + "/matchers")
|
17
|
+
Dir[File.dirname(__FILE__) + "/matchers/*.rb"].each {|file| require file }
|
18
|
+
end
|
19
|
+
|
20
|
+
Spec::Runner.configure do |config|
|
21
|
+
config.use_transactional_fixtures = true
|
22
|
+
config.use_instantiated_fixtures = false
|
23
|
+
config.fixture_path = 'spec/fixtures'
|
24
|
+
|
25
|
+
# You can declare fixtures for each behaviour like this:
|
26
|
+
# describe "...." do
|
27
|
+
# fixtures :table_a, :table_b
|
28
|
+
#
|
29
|
+
# Alternatively, if you prefer to declare them only once, you can
|
30
|
+
# do so here, like so ...
|
31
|
+
#
|
32
|
+
# config.global_fixtures = :table_a, :table_b
|
33
|
+
#
|
34
|
+
# If you declare global fixtures, be aware that they will be declared
|
35
|
+
# for all of your examples, even those that don't use them.
|
36
|
+
end
|
metadata
ADDED
@@ -0,0 +1,122 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: radiant-scraped_layouts-extension
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Charles Hudson
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-03-20 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: radiant
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 1.1.0
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 1.1.0
|
30
|
+
description: Generates scraped layouts, allowing content to be inserted via regular
|
31
|
+
expressions.
|
32
|
+
email:
|
33
|
+
- charles.h.hudson@phobetron.com
|
34
|
+
executables: []
|
35
|
+
extensions: []
|
36
|
+
extra_rdoc_files: []
|
37
|
+
files:
|
38
|
+
- ./app/controllers/admin/scrapers_controller.rb
|
39
|
+
- ./app/models/scraper.rb
|
40
|
+
- ./app/models/scraper_insert.rb
|
41
|
+
- ./app/views/admin/layouts/_actions_cell.html.haml
|
42
|
+
- ./app/views/admin/layouts/_disabled_form.html.haml
|
43
|
+
- ./app/views/admin/layouts/_title_cell.html.haml
|
44
|
+
- ./app/views/admin/layouts/_warning.html.haml
|
45
|
+
- ./app/views/admin/layouts/edit.html.haml
|
46
|
+
- ./app/views/admin/scraper_inserts/_scraper_insert.html.haml
|
47
|
+
- ./app/views/admin/scrapers/_form.html.haml
|
48
|
+
- ./app/views/admin/scrapers/edit.html.haml
|
49
|
+
- ./app/views/admin/scrapers/index.html.haml
|
50
|
+
- ./app/views/admin/scrapers/new.html.haml
|
51
|
+
- ./app/views/admin/scrapers/remove.html.haml
|
52
|
+
- ./config/initializers/radiant_config.rb
|
53
|
+
- ./config/locales/en.yml
|
54
|
+
- ./config/routes.rb
|
55
|
+
- ./cucumber.yml
|
56
|
+
- ./db/migrate/20121015160755_create_scrapers.rb
|
57
|
+
- ./db/migrate/20121015160953_create_scraper_inserts.rb
|
58
|
+
- ./db/migrate/20121101183044_integrate_scraper_and_layout.rb
|
59
|
+
- ./features/support/env.rb
|
60
|
+
- ./features/support/paths.rb
|
61
|
+
- ./Gemfile
|
62
|
+
- ./Gemfile.lock
|
63
|
+
- ./lib/radiant-scraped_layouts-extension.rb
|
64
|
+
- ./lib/tasks/scraped_layouts_extension_tasks.rake
|
65
|
+
- ./radiant-scraped_layouts-extension.gemspec
|
66
|
+
- ./Rakefile
|
67
|
+
- ./README.md
|
68
|
+
- ./scraped_layouts_extension.rb
|
69
|
+
- ./spec/controllers/admin/scrapers_controller_spec.rb
|
70
|
+
- ./spec/fixtures/scraper_inserts.yml
|
71
|
+
- ./spec/fixtures/scrapers.yml
|
72
|
+
- ./spec/models/layout_spec.rb
|
73
|
+
- ./spec/models/scraper_insert_spec.rb
|
74
|
+
- ./spec/models/scraper_spec.rb
|
75
|
+
- ./spec/spec.opts
|
76
|
+
- ./spec/spec_helper.rb
|
77
|
+
- spec/controllers/admin/scrapers_controller_spec.rb
|
78
|
+
- spec/fixtures/scraper_inserts.yml
|
79
|
+
- spec/fixtures/scrapers.yml
|
80
|
+
- spec/models/layout_spec.rb
|
81
|
+
- spec/models/scraper_insert_spec.rb
|
82
|
+
- spec/models/scraper_spec.rb
|
83
|
+
- spec/spec.opts
|
84
|
+
- spec/spec_helper.rb
|
85
|
+
- features/support/env.rb
|
86
|
+
- features/support/paths.rb
|
87
|
+
homepage: https://github.com/yp-engineering/radiant-scraped_layouts-extension
|
88
|
+
licenses: []
|
89
|
+
post_install_message:
|
90
|
+
rdoc_options: []
|
91
|
+
require_paths:
|
92
|
+
- lib
|
93
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
94
|
+
none: false
|
95
|
+
requirements:
|
96
|
+
- - ! '>='
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
|
+
none: false
|
101
|
+
requirements:
|
102
|
+
- - ! '>='
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
requirements: []
|
106
|
+
rubyforge_project:
|
107
|
+
rubygems_version: 1.8.24
|
108
|
+
signing_key:
|
109
|
+
specification_version: 3
|
110
|
+
summary: Scraped Layouts for Radiant CMS
|
111
|
+
test_files:
|
112
|
+
- spec/controllers/admin/scrapers_controller_spec.rb
|
113
|
+
- spec/fixtures/scraper_inserts.yml
|
114
|
+
- spec/fixtures/scrapers.yml
|
115
|
+
- spec/models/layout_spec.rb
|
116
|
+
- spec/models/scraper_insert_spec.rb
|
117
|
+
- spec/models/scraper_spec.rb
|
118
|
+
- spec/spec.opts
|
119
|
+
- spec/spec_helper.rb
|
120
|
+
- features/support/env.rb
|
121
|
+
- features/support/paths.rb
|
122
|
+
has_rdoc:
|