wco_models 3.1.0.37 → 3.1.0.38
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +3 -13
- data/app/assets/config/wco_models_manifest.js +2 -0
- data/app/assets/javascripts/wco/application.js +43 -5
- data/app/assets/javascripts/wco/collapse-expand.js +1 -0
- data/app/assets/javascripts/wco/file_upload.js +32 -0
- data/app/assets/javascripts/wco/shared.js +22 -0
- data/app/assets/stylesheets/wco/application.css +23 -8
- data/app/assets/stylesheets/wco/utils.scss +57 -1
- data/app/controllers/wco/application_controller.rb +9 -0
- data/app/controllers/wco/galleries_controller.rb +143 -0
- data/app/controllers/wco/headlines_controller.rb +71 -0
- data/app/controllers/wco/leads_controller.rb +59 -0
- data/app/controllers/wco/leadsets_controller.rb +90 -0
- data/app/controllers/wco/logs_controller.rb +64 -0
- data/app/controllers/wco/office_action_templates_controller.rb +80 -0
- data/app/controllers/wco/office_actions_controller.rb +71 -0
- data/app/controllers/wco/photos_controller.rb +78 -0
- data/app/controllers/wco/profiles_controller.rb +26 -0
- data/app/controllers/wco/publishers_controller.rb +60 -0
- data/app/controllers/wco/reports_controller.rb +75 -0
- data/app/controllers/wco/sites_controller.rb +52 -0
- data/app/controllers/wco/tags_controller.rb +57 -0
- data/app/helpers/wco/application_helper.rb +42 -0
- data/app/models/wco/asset.rb +26 -0
- data/app/models/wco/gallery.rb +3 -3
- data/app/models/wco/headline.rb +19 -0
- data/app/models/wco/invoice.rb +127 -0
- data/app/models/wco/lead.rb +42 -0
- data/app/models/wco/leadset.rb +8 -2
- data/app/models/wco/log.rb +18 -0
- data/app/models/wco/newsitem.rb +1 -0
- data/app/models/wco/office_action.rb +17 -18
- data/app/models/wco/office_action_template.rb +41 -0
- data/app/models/wco/office_action_template_tie.rb +20 -0
- data/app/models/wco/photo.rb +10 -8
- data/app/models/wco/price.rb +2 -0
- data/app/models/wco/product.rb +2 -0
- data/app/models/wco/profile.rb +1 -0
- data/app/models/wco/publisher.rb +68 -3
- data/app/models/wco/report.rb +48 -0
- data/app/models/wco/site.rb +12 -1
- data/app/models/wco/subscription.rb +1 -0
- data/app/models/wco/tag.rb +17 -4
- data/app/models/wco_email/campaign.rb +7 -11
- data/app/models/wco_email/context.rb +15 -14
- data/app/models/wco_email/conversation.rb +13 -27
- data/app/models/wco_email/{scheduled_email_action.rb → email_action.rb} +24 -20
- data/app/models/wco_email/email_action_template.rb +35 -0
- data/app/models/wco_email/email_filter.rb +9 -12
- data/app/models/wco_email/email_template.rb +124 -0
- data/app/models/wco_email/message.rb +228 -0
- data/app/models/wco_email/message_stub.rb +107 -0
- data/app/models/wco_email/obfuscated_redirect.rb +13 -0
- data/app/models/wco_email/unsubscribe.rb +19 -0
- data/app/models/wco_hosting/appliance.rb +1 -0
- data/app/models/wco_hosting/appliance_tmpl.rb +1 -0
- data/app/models/wco_hosting/domain.rb +1 -0
- data/app/models/wco_hosting/serverhost.rb +1 -0
- data/app/views/layouts/wco/application.haml +30 -17
- data/app/views/wco/_main_footer.haml +43 -0
- data/app/views/wco/_main_header.haml +37 -0
- data/app/views/wco/_main_header.haml-bk +109 -0
- data/app/views/wco/_paginate.haml +8 -0
- data/app/views/wco/_search.haml +8 -0
- data/app/views/wco/application/_debug.haml +0 -0
- data/app/views/wco/galleries/_form.haml +24 -0
- data/app/views/wco/galleries/_header.haml +8 -0
- data/app/views/wco/galleries/_index.haml +40 -0
- data/app/views/wco/galleries/_menu.haml-trash +10 -0
- data/app/views/wco/galleries/_menu_secondary.haml +12 -0
- data/app/views/wco/galleries/_thumbs.haml +9 -0
- data/app/views/wco/galleries/_title.haml +16 -0
- data/app/views/wco/galleries/edit.haml +6 -0
- data/app/views/wco/galleries/new.haml +6 -0
- data/app/views/wco/galleries/show.haml +43 -0
- data/app/views/wco/headlines/_form.haml +17 -0
- data/app/views/wco/headlines/_header.haml +6 -0
- data/app/views/wco/headlines/_index.haml +20 -0
- data/app/views/wco/headlines/edit.haml +6 -0
- data/app/views/wco/headlines/new.haml +2 -0
- data/app/views/wco/invoices/_form.haml-trash +29 -0
- data/app/views/wco/invoices/_header.haml +6 -0
- data/app/views/wco/invoices/_index_list.haml +13 -0
- data/app/views/wco/invoices/_index_table.haml +20 -0
- data/app/views/wco/invoices/index.haml +3 -0
- data/app/views/wco/invoices/new_pdf.haml +18 -0
- data/app/views/wco/invoices/new_stripe.haml +50 -0
- data/app/views/wco/invoices/show.haml +23 -0
- data/app/views/wco/kaminari/_first_page.html.erb +11 -0
- data/app/views/wco/kaminari/_gap.html.erb +8 -0
- data/app/views/wco/kaminari/_last_page.html.erb +11 -0
- data/app/views/wco/kaminari/_next_page.html.erb +11 -0
- data/app/views/wco/kaminari/_page.html.erb +13 -0
- data/app/views/wco/kaminari/_paginator.html.erb +25 -0
- data/app/views/wco/kaminari/_prev_page.html.erb +11 -0
- data/app/views/wco/leads/_form.haml +45 -0
- data/app/views/wco/leads/_form_import.haml +11 -0
- data/app/views/wco/leads/_header.haml +7 -0
- data/app/views/wco/leads/_index.haml +75 -0
- data/app/views/wco/leads/_index_rows.haml +11 -0
- data/app/views/wco/leads/edit.haml +22 -0
- data/app/views/wco/leads/index.haml +7 -0
- data/app/views/wco/leads/new.haml +9 -0
- data/app/views/wco/leads/show.haml +85 -0
- data/app/views/wco/leadsets/_form.haml +28 -0
- data/app/views/wco/leadsets/_header.haml +6 -0
- data/app/views/wco/leadsets/edit.haml +4 -0
- data/app/views/wco/leadsets/index.haml +45 -0
- data/app/views/wco/leadsets/index_dataTables.haml +33 -0
- data/app/views/wco/leadsets/new.haml +6 -0
- data/app/views/wco/leadsets/show.haml +52 -0
- data/app/views/wco/logs/_form.haml +17 -0
- data/app/views/wco/logs/_header.haml +6 -0
- data/app/views/wco/logs/_index.haml +23 -0
- data/app/views/wco/logs/edit.haml +6 -0
- data/app/views/wco/logs/new.haml +2 -0
- data/app/views/wco/office_action_templates/_form.haml +46 -0
- data/app/views/wco/office_action_templates/_header.haml +8 -0
- data/app/views/wco/office_action_templates/_ties.haml +18 -0
- data/app/views/wco/office_action_templates/edit.haml +6 -0
- data/app/views/wco/office_action_templates/index.haml +17 -0
- data/app/views/wco/office_action_templates/new.haml +5 -0
- data/app/views/wco/office_action_templates/show.haml +22 -0
- data/app/views/wco/office_actions/_form.haml +22 -0
- data/app/views/wco/office_actions/_header.haml +6 -0
- data/app/views/wco/office_actions/_index.haml +15 -0
- data/app/views/wco/office_actions/edit.haml +5 -0
- data/app/views/wco/office_actions/index.haml +10 -0
- data/app/views/wco/office_actions/new.haml +4 -0
- data/app/views/wco/office_actions/show.haml +5 -0
- data/app/views/wco/photos/_form.haml +0 -0
- data/app/views/wco/photos/_index_thumbs.haml +7 -0
- data/app/views/wco/photos/_meta.haml +7 -0
- data/app/views/wco/photos/_meta_manager.haml +7 -0
- data/app/views/wco/photos/_multinew.haml +6 -0
- data/app/views/wco/photos/index.haml +3 -0
- data/app/views/wco/photos/new.haml +0 -0
- data/app/views/wco/photos/show.haml +6 -0
- data/app/views/wco/photos/without_gallery.haml +11 -0
- data/app/views/wco/prices/_form.haml +1 -1
- data/app/views/wco/products/_header.haml +6 -0
- data/app/views/wco/publishers/_form.haml +28 -0
- data/app/views/wco/publishers/_header.haml +6 -0
- data/app/views/wco/publishers/edit.haml +4 -0
- data/app/views/wco/publishers/index.haml +37 -0
- data/app/views/wco/publishers/new.haml +4 -0
- data/app/views/wco/reports/_form.haml +28 -0
- data/app/views/wco/reports/_header.haml +14 -0
- data/app/views/wco/reports/edit.haml +4 -0
- data/app/views/wco/reports/index.haml +17 -0
- data/app/views/wco/reports/index_table.haml +17 -0
- data/app/views/wco/reports/new.haml +4 -0
- data/app/views/wco/reports/show.haml +8 -0
- data/app/views/wco/sites/_form.haml +21 -0
- data/app/views/wco/sites/_header.haml +6 -0
- data/app/views/wco/sites/edit.haml +4 -0
- data/app/views/wco/sites/index.haml +24 -0
- data/app/views/wco/sites/new.haml +4 -0
- data/app/views/wco/tags/_form.haml +10 -0
- data/app/views/wco/tags/_header.haml +6 -0
- data/app/views/wco/tags/_index.haml +10 -0
- data/app/views/wco/tags/_index_inline.haml +8 -0
- data/app/views/wco/tags/edit.haml +0 -0
- data/app/views/wco/tags/index.haml +9 -0
- data/app/views/wco/tags/new.haml +0 -0
- data/app/views/wco/tags/show.haml +15 -0
- data/config/routes.rb +42 -0
- data/lib/tasks/db_tasks.rake +13 -0
- data/lib/tasks/office_tasks.rake +36 -0
- data/lib/tasks/scrape_tasks.rake-trash +22 -0
- data/lib/wco/ai_writer.rb +55 -0
- data/lib/wco/engine.rb +0 -1
- data/lib/wco/office_worker.rb +4 -0
- data/lib/wco/scrape_wsj.rb +29 -0
- data/lib/wco/scrape_wsj_capy.rb +44 -0
- data/lib/wco_models.rb +15 -2
- metadata +241 -46
- data/app/helpers/ish_models/application_helper.rb +0 -4
- data/app/models/wco_email/message_template.rb +0 -6
- data/lib/tasks/ish_models_tasks.rake +0 -4
- /data/app/views/wco/{application/_alerts_notices.haml → _alerts_notices.haml} +0 -0
@@ -0,0 +1,17 @@
|
|
1
|
+
|
2
|
+
.reports-index.maxwidth
|
3
|
+
%h5
|
4
|
+
Reports
|
5
|
+
= link_to '[+]', new_report_path
|
6
|
+
|
7
|
+
%table.bordered
|
8
|
+
%tr
|
9
|
+
%th
|
10
|
+
%th slug
|
11
|
+
- @reports.each do |ttt|
|
12
|
+
%tr
|
13
|
+
%td
|
14
|
+
.flex-row
|
15
|
+
= button_to '[x]', report_path(ttt), method: :delete, data: { confirm: 'Are you sure?' }
|
16
|
+
= link_to '[~]', edit_report_path(ttt)
|
17
|
+
%td= link_to ttt.slug, report_path(ttt)
|
@@ -0,0 +1,21 @@
|
|
1
|
+
|
2
|
+
.sites--form
|
3
|
+
= form_for site do |f|
|
4
|
+
.field
|
5
|
+
%label Slug
|
6
|
+
= f.text_field :slug
|
7
|
+
|
8
|
+
= f.label :kind
|
9
|
+
= f.select :kind, options_for_select( Wco::Site.kinds_list, selected: site.kind )
|
10
|
+
.field
|
11
|
+
= f.label :origin
|
12
|
+
= f.text_field :origin
|
13
|
+
= f.label :post_path
|
14
|
+
= f.text_field :post_path
|
15
|
+
.field
|
16
|
+
= f.label :username
|
17
|
+
= f.text_field :username
|
18
|
+
= f.label :password
|
19
|
+
= f.text_field :password
|
20
|
+
.actions
|
21
|
+
= f.submit
|
@@ -0,0 +1,24 @@
|
|
1
|
+
|
2
|
+
.sites-index.padded
|
3
|
+
%h5
|
4
|
+
Sites
|
5
|
+
= link_to '[+]', new_site_path
|
6
|
+
|
7
|
+
%table.bordered
|
8
|
+
%tr
|
9
|
+
%th
|
10
|
+
%th slug
|
11
|
+
%th kind
|
12
|
+
%th origin
|
13
|
+
%th post_path
|
14
|
+
- @sites.each do |ttt|
|
15
|
+
%tr
|
16
|
+
%td
|
17
|
+
.flex-row
|
18
|
+
= button_to '[x]', site_path(ttt), method: :delete, data: { confirm: 'Are you sure?' }
|
19
|
+
= link_to '[~]', edit_site_path(ttt)
|
20
|
+
= ttt.id
|
21
|
+
%td= ttt.slug
|
22
|
+
%td= ttt.kind
|
23
|
+
%td= ttt.origin
|
24
|
+
%td= ttt.post_path
|
File without changes
|
File without changes
|
@@ -0,0 +1,15 @@
|
|
1
|
+
|
2
|
+
.tags-show.maxwidth
|
3
|
+
|
4
|
+
%h5 Tag `#{@tag.slug}`
|
5
|
+
|
6
|
+
%ul
|
7
|
+
%li.d-flex
|
8
|
+
<b>Headlines (#{@tag.headlines.length}):</b>
|
9
|
+
= render '/wco/headlines/index', headlines: @tag.headlines
|
10
|
+
%li
|
11
|
+
<b>N reports:</b> #{@tag.reports.length}
|
12
|
+
%li
|
13
|
+
<b>N stubs:</b> #{@tag.message_stubs.length}
|
14
|
+
= render '/wco_email/message_stubs/index', stubs: @tag.message_stubs
|
15
|
+
|
data/config/routes.rb
CHANGED
@@ -2,9 +2,51 @@
|
|
2
2
|
Wco::Engine.routes.draw do
|
3
3
|
root to: 'application#home'
|
4
4
|
|
5
|
+
get 'application/tinymce', to: 'application#tinymce'
|
6
|
+
|
7
|
+
resources :assets
|
8
|
+
# get 'assets/:id', to: 'assets#show', as: :asset
|
9
|
+
|
10
|
+
resources :galleries do
|
11
|
+
post 'multiadd', :to => 'photos#j_create', :as => :multiadd
|
12
|
+
end
|
13
|
+
|
14
|
+
resources :headlines
|
15
|
+
|
16
|
+
post 'invoices/send/:id', to: 'invoices#email_send', as: :send_invoice
|
17
|
+
post 'invoices/cr-m/:leadset_id', to: 'invoices#create_monthly_pdf', as: :create_monthly_invoice_for_leadset
|
18
|
+
post 'invoices/create-pdf', to: 'invoices#create_pdf', as: :create_invoice_pdf
|
19
|
+
post 'invoices/create-stripe', to: 'invoices#create_stripe', as: :create_invoice_stripe
|
20
|
+
get 'invoices/new_pdf', to: 'invoices#new_pdf', as: :new_invoice_pdf
|
21
|
+
get 'invoices/new_stripe', to: 'invoices#new_stripe', as: :new_invoice_stripe
|
22
|
+
post 'invoices/:id/send-stripe', to: 'invoices#send_stripe', as: :send_invoice_stripe
|
23
|
+
resources :invoices
|
24
|
+
|
25
|
+
get 'leads/:id', to: 'leads#show', id: /[^\/]+/
|
26
|
+
post 'leads/bulkop', to: 'leads#bulkop'
|
27
|
+
post 'leads/import', to: 'leads#import', as: :leads_import
|
28
|
+
resources :leads
|
29
|
+
resources :leadsets
|
30
|
+
delete 'logs/bulkop', to: 'logs#bulkop', as: :logs_bulkop
|
31
|
+
resources :logs
|
32
|
+
|
33
|
+
post 'office_action_templates', to: 'office_action_templates#update'
|
34
|
+
resources :office_action_templates
|
35
|
+
resources :office_actions
|
36
|
+
|
5
37
|
resources :prices
|
6
38
|
resources :products
|
39
|
+
resources :profiles
|
40
|
+
post 'publishers/:id/do-run', to: 'publishers#do_run', as: :do_run_publisher
|
41
|
+
resources :publishers
|
42
|
+
resources :photos
|
7
43
|
|
44
|
+
get 'reports/deleted', to: 'reports#index', as: :deleted_reports, defaults: { deleted: true }
|
45
|
+
resources :reports
|
46
|
+
|
47
|
+
resources :sites
|
8
48
|
resources :subscriptions
|
9
49
|
|
50
|
+
resources :tags
|
51
|
+
|
10
52
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
namespace :db do
|
4
|
+
|
5
|
+
desc 'seed'
|
6
|
+
task :seed do
|
7
|
+
leadset = Wco::Leadset.find_or_create_by({ company_url: 'poxlovi@gmail.com' })
|
8
|
+
leadset.persisted?
|
9
|
+
lead = Wco::Lead.find_or_create_by({ email: 'poxlovi@gmail.com', leadset: leadset })
|
10
|
+
lead.persisted?
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
|
2
|
+
require 'business_time'
|
3
|
+
require 'httparty'
|
4
|
+
|
5
|
+
namespace :office do
|
6
|
+
|
7
|
+
desc 'run_office_actions'
|
8
|
+
task run_office_actions: :environment do
|
9
|
+
while true do
|
10
|
+
|
11
|
+
OA = Wco::OfficeAction
|
12
|
+
OAT = Wco::OfficeActionTemplate
|
13
|
+
|
14
|
+
OA.active.where( :perform_at.lte => Time.now ).each do |oa|
|
15
|
+
puts "+++ +++ Office Action: #{oa}"
|
16
|
+
|
17
|
+
oa.update({ status: INACTIVE })
|
18
|
+
oa.tmpl.do_run
|
19
|
+
oa.tmpl.ties.each do |tie|
|
20
|
+
next_oa = OA.find_or_initialize_by({
|
21
|
+
office_action_template_id: tie.next_oat.id,
|
22
|
+
})
|
23
|
+
next_oa.perform_at = eval( tie.next_at_exe )
|
24
|
+
next_oa.status = ACTIVE
|
25
|
+
next_oa.save!
|
26
|
+
end
|
27
|
+
|
28
|
+
print '^'
|
29
|
+
end
|
30
|
+
|
31
|
+
print '.'
|
32
|
+
sleep Rails.env.production? ? 60 : 5 # seconds
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
|
2
|
+
require 'wco/scrape_test'
|
3
|
+
require 'wco/scrape_test_2'
|
4
|
+
|
5
|
+
|
6
|
+
RSpec::Core::RakeTask.new(:spec)
|
7
|
+
namespace :scrape do
|
8
|
+
|
9
|
+
desc 'test'
|
10
|
+
task :test do
|
11
|
+
Wco::ScrapeTest.new
|
12
|
+
end
|
13
|
+
|
14
|
+
desc 'test 2'
|
15
|
+
task :test_2 do
|
16
|
+
|
17
|
+
site = Wco::Site.find_by slug: 'wsj'
|
18
|
+
Wco::ScrapeTest2.new( site: site )
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
|
2
|
+
# require 'httparty'
|
3
|
+
|
4
|
+
##
|
5
|
+
## OpenAI GPT3 GPT3.5 GPT4
|
6
|
+
##
|
7
|
+
class Wco::AiWriter
|
8
|
+
|
9
|
+
def self.run_prompt prompt
|
10
|
+
out = HTTParty.post("https://api.openai.com/v1/chat/completions", {
|
11
|
+
headers: {
|
12
|
+
"Content-Type": "application/json",
|
13
|
+
"Authorization": "Bearer #{OPENAI_API_KEY}",
|
14
|
+
},
|
15
|
+
body: {
|
16
|
+
model: 'gpt-3.5-turbo',
|
17
|
+
messages: [
|
18
|
+
{ role: "system", content: "You are a knowledgable office assistant." },
|
19
|
+
{ role: 'user', content: prompt },
|
20
|
+
] }.to_json
|
21
|
+
})
|
22
|
+
out = JSON.parse out.response.body
|
23
|
+
out.deep_symbolize_keys!
|
24
|
+
out = out[:choices][0][:message][:content]
|
25
|
+
return out
|
26
|
+
end
|
27
|
+
def run_prompt p; self.class.run_prompt p; end
|
28
|
+
|
29
|
+
|
30
|
+
def self.run_headline headline
|
31
|
+
prompt = "Rephrase the following article title using one sentence: #{headline.name}"
|
32
|
+
new_title = self.run_prompt prompt
|
33
|
+
puts! new_title, 'new_title'
|
34
|
+
|
35
|
+
prompt = "Write an article about the following topic: #{headline.name}"
|
36
|
+
new_body = self.run_prompt prompt
|
37
|
+
new_body.gsub!("\r", '')
|
38
|
+
new_body = new_body.split("\n\n").map { |ppp| "<p>#{ppp}</p>" }.join
|
39
|
+
new_body = new_body.gsub("\n", "<br />")
|
40
|
+
puts! new_body[0...200], 'new_body'
|
41
|
+
|
42
|
+
report = Wco::Report.create!({
|
43
|
+
title: new_title,
|
44
|
+
slug: new_title,
|
45
|
+
body: new_body,
|
46
|
+
})
|
47
|
+
|
48
|
+
# headline.delete
|
49
|
+
|
50
|
+
return report
|
51
|
+
end
|
52
|
+
def run_headline h; self.class.run_headline h; end
|
53
|
+
|
54
|
+
|
55
|
+
end
|
data/lib/wco/engine.rb
CHANGED
@@ -0,0 +1,29 @@
|
|
1
|
+
|
2
|
+
require 'httparty'
|
3
|
+
|
4
|
+
class Wco::ScrapeWsj
|
5
|
+
|
6
|
+
def initialize site: nil, max_count: 2
|
7
|
+
site ||= Wco::Site.find_by slug: 'wsj'
|
8
|
+
|
9
|
+
out = HTTParty.get( site.origin ).response.body
|
10
|
+
out = Nokogiri::HTML(out)
|
11
|
+
out.css('h3').each_with_index do |headline, idx|
|
12
|
+
if idx < max_count
|
13
|
+
puts! headline.text, 'headline'
|
14
|
+
h = Wco::Headline.create({
|
15
|
+
date: Time.now.to_date,
|
16
|
+
site: site,
|
17
|
+
name: headline.text,
|
18
|
+
})
|
19
|
+
if h.persisted?
|
20
|
+
print '.'
|
21
|
+
else
|
22
|
+
puts! h.errors.full_messages
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
|
@@ -0,0 +1,44 @@
|
|
1
|
+
|
2
|
+
# require 'capybara/rspec'
|
3
|
+
# require 'byebug'
|
4
|
+
require 'capybara/rails'
|
5
|
+
|
6
|
+
Capybara.register_driver :selenium_chrome do |app|
|
7
|
+
# options = Selenium::WebDriver::Chrome::Options.new
|
8
|
+
# options.add_argument '--remote-debugging-port=4444'
|
9
|
+
# Capybara::Selenium::Driver.new(app,
|
10
|
+
# browser: :chrome,
|
11
|
+
# options: options )
|
12
|
+
|
13
|
+
Capybara::Selenium::Driver.new app, browser: :chrome
|
14
|
+
end
|
15
|
+
|
16
|
+
Capybara.default_driver = :selenium_chrome
|
17
|
+
Capybara.default_max_wait_time = 100 # seconds
|
18
|
+
Capybara.server = :webrick
|
19
|
+
|
20
|
+
class Wco::ScrapeTest
|
21
|
+
include Capybara::DSL
|
22
|
+
|
23
|
+
def initialize
|
24
|
+
|
25
|
+
visit 'https://www.wsj.com/'
|
26
|
+
|
27
|
+
# all('h6').each_with_index do |h6, idx|
|
28
|
+
(1...2).each do |idx|
|
29
|
+
headline = find(:xpath, "(//h3)[#{idx}]").text
|
30
|
+
# puts "+++ Title: #{title}"
|
31
|
+
|
32
|
+
Wco::Headline.create!({
|
33
|
+
name: headline,
|
34
|
+
site: wsj,
|
35
|
+
date: Time.now.to_date,
|
36
|
+
})
|
37
|
+
print '.'
|
38
|
+
end
|
39
|
+
|
40
|
+
# byebug
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
data/lib/wco_models.rb
CHANGED
@@ -1,18 +1,31 @@
|
|
1
1
|
|
2
|
+
require 'aws-sdk-s3'
|
3
|
+
|
4
|
+
|
2
5
|
require 'cancancan'
|
3
6
|
require 'devise'
|
4
7
|
require 'haml'
|
8
|
+
require 'httparty'
|
5
9
|
|
6
|
-
|
7
|
-
## wco_hosting_rb/test/dummy/app/models/user.rb:3:in `<class:User>': uninitialized constant User::Mongoid (NameError)
|
10
|
+
require 'mail'
|
8
11
|
require 'mongoid'
|
12
|
+
require 'mongoid_paranoia'
|
9
13
|
|
10
14
|
require 'stripe'
|
11
15
|
|
12
16
|
require "wco/engine"
|
13
17
|
|
18
|
+
ACTIVE = 'active'
|
19
|
+
INACTIVE = 'inactive'
|
20
|
+
STATUSES = [ nil, ACTIVE, INACTIVE ]
|
21
|
+
|
14
22
|
module Wco; end
|
15
23
|
module WcoEmail; end
|
16
24
|
module WcoHosting; end
|
17
25
|
|
26
|
+
class Wco::HTTParty
|
27
|
+
include HTTParty
|
28
|
+
debug_output STDOUT
|
29
|
+
end
|
18
30
|
|
31
|
+
ActiveSupport.escape_html_entities_in_json = true
|