aggregator-gem 0.1.8 → 0.1.9
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/aggregator-gem.gemspec +11 -1
- data/lib/cj_advertisers.rb +57 -0
- data/lib/coupon_aggregator.rb +102 -1
- data/lib/coupon_factory.rb +127 -0
- data/lib/feed_mapper.rb +34 -0
- data/lib/feed_parser.rb +57 -0
- data/lib/helpers/advertiser_factory.rb +50 -0
- data/lib/helpers/advertiser_parser.rb +41 -0
- data/lib/html_connection/.html_connector.rb.swp +0 -0
- data/lib/html_connection/html_connection_helper.rb +11 -0
- data/lib/tasks/.gitkeep +0 -0
- data/lib/tasks/coupon_feeds.rake +33 -0
- metadata +26 -16
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.9
|
data/aggregator-gem.gemspec
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "aggregator-gem"
|
8
|
-
s.version = "0.1.
|
8
|
+
s.version = "0.1.9"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Allen Tsai"]
|
@@ -59,7 +59,17 @@ Gem::Specification.new do |s|
|
|
59
59
|
"lib/app/models/sites_vendors.rb",
|
60
60
|
"lib/app/models/vendor.rb",
|
61
61
|
"lib/app/views/layouts/application.html.erb",
|
62
|
+
"lib/cj_advertisers.rb",
|
62
63
|
"lib/coupon_aggregator.rb",
|
64
|
+
"lib/coupon_factory.rb",
|
65
|
+
"lib/feed_mapper.rb",
|
66
|
+
"lib/feed_parser.rb",
|
67
|
+
"lib/helpers/advertiser_factory.rb",
|
68
|
+
"lib/helpers/advertiser_parser.rb",
|
69
|
+
"lib/html_connection/.html_connector.rb.swp",
|
70
|
+
"lib/html_connection/html_connection_helper.rb",
|
71
|
+
"lib/tasks/.gitkeep",
|
72
|
+
"lib/tasks/coupon_feeds.rake",
|
63
73
|
"tasks/coupon_aggregator_tasks.rake",
|
64
74
|
"test/helper.rb",
|
65
75
|
"test/test_coupon_aggregator.rb"
|
@@ -0,0 +1,57 @@
|
|
1
|
+
#
|
2
|
+
#Go get all of the advertisers we work with on CJ
|
3
|
+
#
|
4
|
+
|
5
|
+
require 'helpers/advertiser_parser.rb'
|
6
|
+
class CJAdvertisers
|
7
|
+
|
8
|
+
def initialize(params, feed = {})
|
9
|
+
#I think we will probably move towards a place where params can be replaced by
|
10
|
+
# a db object
|
11
|
+
@params = params
|
12
|
+
@feed = feed
|
13
|
+
end
|
14
|
+
|
15
|
+
def collect_advertisers()
|
16
|
+
#Setup the models we'll need
|
17
|
+
advert_parser = AdvertiserParser.new
|
18
|
+
|
19
|
+
uri = URI("https://advertiser-lookup.api.cj.com/v3/advertiser-lookup") #This can go into vendors table
|
20
|
+
headers = {"authorization" => "00b0786d8cc156bc642d3b3531b858efbf445f0b9fac809ce739506268f388170e14eb47e411c5a7fd2b6eebfc333e09821e6a4748fc068e7ebe545f7a43171029/009d71a493cb3f3e6fc627280b8724da29f4af33e8aef84484cb8edb6e35b0fa97592c51e20eea0f63a36101cbe92f47073a28acbb29a293ed27c5884ef3d5a09d"} #this might go into some headings table
|
21
|
+
|
22
|
+
require 'net/https'
|
23
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
24
|
+
http.use_ssl = true if uri.port == 443
|
25
|
+
request = HtmlConnectionHelper.build_request(uri,@params,headers)
|
26
|
+
response = http.request(request)
|
27
|
+
|
28
|
+
total_num = calculate_total_pages(response)
|
29
|
+
|
30
|
+
puts "Total pages to grab: " + total_num.to_s
|
31
|
+
|
32
|
+
#Go through each page
|
33
|
+
|
34
|
+
(1..total_num).each_with_index do |n, index|
|
35
|
+
puts "Working on page: " + n.to_s
|
36
|
+
|
37
|
+
page_number = {"page-number" => n}
|
38
|
+
@params.merge!(page_number)
|
39
|
+
|
40
|
+
request = HtmlConnectionHelper.build_request(uri,@params,headers)
|
41
|
+
response = http.request(request)
|
42
|
+
response = CGI.unescapeHTML(response.body)
|
43
|
+
#feed_parser.create_coupons_from_xml(response)
|
44
|
+
|
45
|
+
#Go parse the page!
|
46
|
+
advert_parser.create_cj_advertiser_from_xml(response)
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
def calculate_total_pages(response)
|
52
|
+
doc = Nokogiri::XML(CGI.unescapeHTML(response.body))
|
53
|
+
records_per_page = doc.xpath("//advertisers/@records-returned")[0].to_s.to_i
|
54
|
+
return (doc.xpath("//advertisers/@total-matched")[0].to_s.to_i / records_per_page) + 1
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
data/lib/coupon_aggregator.rb
CHANGED
@@ -1 +1,102 @@
|
|
1
|
-
|
1
|
+
|
2
|
+
require 'app/models/advertiser_category_link'
|
3
|
+
require 'app/models/advertiser'
|
4
|
+
require 'app/models/advertiser_vendor_link'
|
5
|
+
require 'app/models/category_coupon_link'
|
6
|
+
require 'app/models/category'
|
7
|
+
require 'app/models/cj_coupon'
|
8
|
+
require 'app/models/coupon_product_link'
|
9
|
+
require 'app/models/coupon'
|
10
|
+
require 'app/models/product'
|
11
|
+
require 'app/models/site'
|
12
|
+
require 'app/models/site_vendor_link'
|
13
|
+
require 'app/models/vendor'
|
14
|
+
|
15
|
+
class CouponAggregator
|
16
|
+
#apparently there are no enums in ruby
|
17
|
+
CJ = 1
|
18
|
+
FMTC = 2
|
19
|
+
=begin
|
20
|
+
params for fmtc is just a "key" => "api key"
|
21
|
+
params for cj involve website-id (ours, assigned by CJ), records-per-page (100 max), promotion-type (coupon), and page-number
|
22
|
+
=end
|
23
|
+
def initialize(params, feed)
|
24
|
+
#I think we will probably move towards a place where params can be replaced by
|
25
|
+
# a db object
|
26
|
+
@params = params
|
27
|
+
@feed = feed
|
28
|
+
end
|
29
|
+
|
30
|
+
#@params site_id int Our site id for the site we are parsing coupons for.
|
31
|
+
def aggregate(site_id)
|
32
|
+
case @feed
|
33
|
+
when CouponAggregator::CJ
|
34
|
+
aggregate_cj_coupons(site_id)
|
35
|
+
when CouponAggregator::FMTC
|
36
|
+
aggregate_fmtc_coupons()
|
37
|
+
else
|
38
|
+
aggregate_all_coupons_for_site(site_id)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
def aggregate_cj_coupons(site_id)
|
44
|
+
|
45
|
+
vendor_id = Vendor.find_by_nice_name("commission-junction").id
|
46
|
+
feed_parser = FeedParser.new(site_id, vendor_id)
|
47
|
+
|
48
|
+
#Find out what site ID commission junction has assigned the site we are pulling for
|
49
|
+
relationship_id = SiteVendorLink.find_by_site_id_and_vendor_id(site_id,vendor_id).relationship_id
|
50
|
+
temp_params = {"website-id" => relationship_id}
|
51
|
+
@params.merge!(temp_params)
|
52
|
+
|
53
|
+
uri = URI("https://linksearch.api.cj.com/v2/link-search") #This can go into vendors table
|
54
|
+
headers = {"authorization" => "00b0786d8cc156bc642d3b3531b858efbf445f0b9fac809ce739506268f388170e14eb47e411c5a7fd2b6eebfc333e09821e6a4748fc068e7ebe545f7a43171029/009d71a493cb3f3e6fc627280b8724da29f4af33e8aef84484cb8edb6e35b0fa97592c51e20eea0f63a36101cbe92f47073a28acbb29a293ed27c5884ef3d5a09d"} #this might go into some headings table
|
55
|
+
|
56
|
+
require 'net/https'
|
57
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
58
|
+
http.use_ssl = true if uri.port == 443
|
59
|
+
request = HtmlConnectionHelper.build_request(uri,@params,headers)
|
60
|
+
response = http.request(request)
|
61
|
+
|
62
|
+
total_num = calculate_total_pages(response)
|
63
|
+
|
64
|
+
(1..total_num).each do |n|
|
65
|
+
|
66
|
+
puts "Working on page: " + n.to_s
|
67
|
+
|
68
|
+
page_number = {"page-number" => n}
|
69
|
+
@params.merge!(page_number)
|
70
|
+
request = HtmlConnectionHelper.build_request(uri,@params,headers)
|
71
|
+
response = http.request(request)
|
72
|
+
response = CGI.unescapeHTML(response.body)
|
73
|
+
feed_parser.create_coupons_from_xml(response)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def aggregate_fmtc_coupons()
|
78
|
+
|
79
|
+
vendor_id = Vendor.find_by_vendor_name("for me to coupon").id
|
80
|
+
site = Site.find_by_site_name("All");
|
81
|
+
feed_parser = FeedParser.new(site.id,vendor_id)
|
82
|
+
|
83
|
+
#need to open the file?
|
84
|
+
uri = URI("http://www.formetocoupon.com/services/getDeals") #this can go into vendors table
|
85
|
+
headers = nil
|
86
|
+
request = HtmlConnectionHelper.build_request(uri,@params,headers)
|
87
|
+
response = http.request(request)
|
88
|
+
|
89
|
+
feed_parser.create_coupons_from_xml(response.body)
|
90
|
+
end
|
91
|
+
|
92
|
+
def aggregate_all_coupons_for_site(site_id)
|
93
|
+
#this will call each of the aggregate methods.
|
94
|
+
end
|
95
|
+
|
96
|
+
def calculate_total_pages(response)
|
97
|
+
doc = Nokogiri::XML(CGI.unescapeHTML(response.body))
|
98
|
+
records_per_page = doc.xpath("//links/@records-returned")[0].to_s.to_i
|
99
|
+
return (doc.xpath("//links/@total-matched")[0].to_s.to_i / records_per_page) + 1
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
class CouponFactory
|
2
|
+
|
3
|
+
def self.update_coupon_with_item(coupon,item,site_id,vendor_id)
|
4
|
+
|
5
|
+
site = Site.find(site_id)
|
6
|
+
vendor = Vendor.find(vendor_id)
|
7
|
+
|
8
|
+
advertiser_name = item.xpath("./advertiser-name/text()").to_s
|
9
|
+
advertiser_id = item.xpath("./advertiser-id/text()").to_s
|
10
|
+
advertiser = self.check_advertiser(advertiser_id, advertiser_name, site.id, vendor.id)
|
11
|
+
|
12
|
+
coupon.site_id = site.id
|
13
|
+
coupon.vendor_id = vendor.id
|
14
|
+
coupon.advertiser_id = advertiser.id
|
15
|
+
coupon.code = item.xpath("./couponcode/text()").to_s
|
16
|
+
coupon.description = item.xpath("./description/text()").to_s
|
17
|
+
coupon.link = item.xpath("./link/*").to_s
|
18
|
+
coupon.save
|
19
|
+
|
20
|
+
|
21
|
+
#sort out categories
|
22
|
+
coupon.categories.clear
|
23
|
+
coupon = self.add_categories_to_coupon(coupon,item)
|
24
|
+
#sort out products
|
25
|
+
coupon.products.clear
|
26
|
+
coupon = self.add_products_to_coupon(coupon,item)
|
27
|
+
return coupon
|
28
|
+
=begin
|
29
|
+
coupon.advertiser_name = item.xpath("./advertiser-name/text()").to_s
|
30
|
+
coupon.category = item.xpath("./category/text()").to_s
|
31
|
+
coupon.language = item.xpath("./language/text()").to_s
|
32
|
+
coupon.click_commission = item.xpath("./click-commission/text()").to_s
|
33
|
+
coupon.lead_commission = item.xpath("./lead-commission/text()").to_s
|
34
|
+
coupon.item_code_html = item.xpath("./item-code-html/*").to_s
|
35
|
+
coupon.item_code_description = item.xpath("./item-code-description/text()").to_s
|
36
|
+
coupon.item_code_destination = item.xpath("./item-code-destination/text()").to_s
|
37
|
+
coupon.item_name = item.xpath("./item-name/text()").to_s
|
38
|
+
coupon.item_type = item.xpath("./item-type/text()").to_s
|
39
|
+
coupon.performance_incentive = item.xpath("./performance-incentive/text()").to_s
|
40
|
+
coupon.start_date = item.xpath("./startdate/text()").to_s
|
41
|
+
coupon.end_date = item.xpath("./enddate/text()").to_s
|
42
|
+
coupon.promotion_type = item.xpath("./promotion-type/text()").to_s
|
43
|
+
coupon.ad_relationship = item.xpath("./relationship-status/text()").to_s
|
44
|
+
coupon.sale_commission = item.xpath("./sale-commission/text()").to_s
|
45
|
+
coupon.seven_day_epc = item.xpath("./seven-day-epc/text()").to_s.to_f
|
46
|
+
coupon.three_month_epc = item.xpath("./three-month-epc/text()").to_s.to_f
|
47
|
+
=end
|
48
|
+
end
|
49
|
+
|
50
|
+
###################
|
51
|
+
# Deprecated
|
52
|
+
###################
|
53
|
+
def self.create_coupon_from_item(item, site_id, vendor_id)
|
54
|
+
|
55
|
+
site = Site.find(site_id)
|
56
|
+
vendor = Vendor.find(vendor_id)
|
57
|
+
|
58
|
+
advertiser_name = item.xpath("./advertiser-name/text()").to_s
|
59
|
+
advertiser = self.check_advertiser(advertiser_name, site.id, vendor.id)
|
60
|
+
|
61
|
+
coupon = Coupon.new()
|
62
|
+
coupon.site_id = site.id
|
63
|
+
coupon.vendor_id = vendor.id
|
64
|
+
coupon.advertiser_id = advertiser.id
|
65
|
+
coupon.code = item.xpath("./couponcode/text()").to_s
|
66
|
+
coupon.description = item.xpath("./description/text()").to_s
|
67
|
+
coupon.link = item.xpath("./link/*").to_s
|
68
|
+
coupon.save
|
69
|
+
#set up the categories
|
70
|
+
coupon = self.add_categories_to_coupon(coupon,item)
|
71
|
+
coupon = self.add_products_to_coupon(coupon,item)
|
72
|
+
return coupon
|
73
|
+
#set up the products
|
74
|
+
=begin
|
75
|
+
coupon.item_id = item.xpath("./item-id/text()").to_s.to_i
|
76
|
+
coupon.category = item.xpath("./category/text()").to_s
|
77
|
+
coupon.language = item.xpath("./language/text()").to_s
|
78
|
+
coupon.click_commission = item.xpath("./click-commission/text()").to_s
|
79
|
+
coupon.lead_commission = item.xpath("./lead-commission/text()").to_s
|
80
|
+
coupon.item_code_html = item.xpath("./item-code-html/*").to_s
|
81
|
+
coupon.item_code_description = item.xpath("./item-code-description/text()").to_s
|
82
|
+
coupon.item_code_destination = item.xpath("./item-code-destination/text()").to_s
|
83
|
+
coupon.item_name = item.xpath("./item-name/text()").to_s
|
84
|
+
coupon.item_type = item.xpath("./item-type/text()").to_s
|
85
|
+
coupon.performance_incentive = item.xpath("./performance-incentive/text()").to_s
|
86
|
+
coupon.promotion_start = item.xpath("./promotion-start-date/text()").to_s
|
87
|
+
coupon.promotion_end = item.xpath("./promotion-end-date/text()").to_s
|
88
|
+
coupon.promotion_type = item.xpath("./promotion-type/text()").to_s
|
89
|
+
coupon.ad_relationship = item.xpath("./relationship-status/text()").to_s
|
90
|
+
coupon.sale_commission = item.xpath("./sale-commission/text()").to_s
|
91
|
+
coupon.seven_day_epc = item.xpath("./seven-day-epc/text()").to_s.to_f
|
92
|
+
coupon.three_month_epc = item.xpath("./three-month-epc/text()").to_s.to_f
|
93
|
+
=end
|
94
|
+
end
|
95
|
+
|
96
|
+
private
|
97
|
+
def self.add_categories_to_coupon(coupon,item)
|
98
|
+
item.xpath("./category").each do |category_xml|
|
99
|
+
category_name = category_xml.xpath("./text()").to_s
|
100
|
+
category_nice_name = category_name.gsub(/[^A-Za-z0-9]+/, '-').strip.gsub(/\ +/, '-').downcase
|
101
|
+
|
102
|
+
#Update/Create category
|
103
|
+
category = Category.find_or_create_by_nice_name(category_nice_name, :name => category_name)
|
104
|
+
|
105
|
+
#Create the relationship
|
106
|
+
CategoryCouponLink.find_or_create_by_category_id_and_coupon_id(category.id, coupon.id)
|
107
|
+
end
|
108
|
+
return coupon
|
109
|
+
end
|
110
|
+
|
111
|
+
#We don't actually have any products yet.
|
112
|
+
def self.add_products_to_coupon(coupon, item)
|
113
|
+
item.xpath("./product").each do |product|
|
114
|
+
coupon.products.find_or_create_by_name(product.xpath("./text()").to_s)
|
115
|
+
end
|
116
|
+
return coupon
|
117
|
+
end
|
118
|
+
|
119
|
+
def self.check_advertiser(advertiser_id, advertiser_name, site_id, vendor_id)
|
120
|
+
advertiser = Advertiser.find_or_create_by_nice_name(
|
121
|
+
:nice_name => advertiser_name.gsub(/[^A-Za-z0-9]+/, '-').strip.gsub(/\ +/, '-').downcase,
|
122
|
+
:name => advertiser_name, :advertiser_id => advertiser_id)
|
123
|
+
AdvertiserVendorLink.find_or_create_by_vendor_id_and_advertiser_id(:vendor_id => vendor_id, :advertiser_id => advertiser.id)
|
124
|
+
AdvertiserSiteLink.find_or_create_by_site_id_and_advertiser_id(:site_id => site_id, :advertiser_id => advertiser.id)
|
125
|
+
return advertiser
|
126
|
+
end
|
127
|
+
end
|
data/lib/feed_mapper.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
class FeedMapper
|
2
|
+
|
3
|
+
@@find_to_replace =
|
4
|
+
{ #CommissionJunction
|
5
|
+
"//links/link" => "item",
|
6
|
+
"//links/item/link-id" => "item-id",
|
7
|
+
"//root/links/item/promotion-start-date" => "startdate",
|
8
|
+
"//root/links/item/promotion-end-date" => "enddate",
|
9
|
+
"//root/links/item/link-code-html" => "link",
|
10
|
+
#ForMeToCoupon
|
11
|
+
"//root/item/dealtypes/type" => "dealtype",
|
12
|
+
"//root/item/merchantname" => "advertiser-name",
|
13
|
+
"//root/item/merchantid" => "advertiser-id",
|
14
|
+
"//root/item/label" => "description",
|
15
|
+
"//root/item/couponid" => "item-id"
|
16
|
+
}
|
17
|
+
|
18
|
+
def self.massage_feed(xml)
|
19
|
+
|
20
|
+
doc = Nokogiri::XML(xml)
|
21
|
+
|
22
|
+
doc.xpath("/").first.children.first.name="root"
|
23
|
+
|
24
|
+
@@find_to_replace.each_pair do |key, value|
|
25
|
+
unless doc.xpath(key).first.nil? then
|
26
|
+
doc.xpath(key).each do |child|
|
27
|
+
child.name=(value)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
return doc
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
data/lib/feed_parser.rb
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
class FeedParser
|
2
|
+
|
3
|
+
attr_reader :uri, :vendor_id, :site_id, :params, :headers
|
4
|
+
|
5
|
+
def initialize(site_id, vendor_id)
|
6
|
+
@vendor_id = vendor_id
|
7
|
+
@site_id = site_id
|
8
|
+
end
|
9
|
+
|
10
|
+
def create_coupons_from_xml(xml)
|
11
|
+
doc = FeedMapper.massage_feed(xml)
|
12
|
+
|
13
|
+
(doc.xpath("//item")).each do |item|
|
14
|
+
advertiser_id = item.xpath("./advertiser-id/text()")
|
15
|
+
item_id = item.xpath("./item-id/text()")
|
16
|
+
|
17
|
+
unless(item_id.to_s.empty?)
|
18
|
+
coupon = Coupon.find_or_create_by_advertiser_id_and_item_id(advertiser_id.to_s, item_id.to_s)
|
19
|
+
coupon = CouponFactory.update_coupon_with_item(coupon,item, @site_id, @vendor_id)
|
20
|
+
coupon.save
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
=begin
|
26
|
+
#START: The things in this block don't belong here
|
27
|
+
# This downloads the request.
|
28
|
+
def download_feed()
|
29
|
+
request = HtmlConnectionHelper.build_request(uri,params,headers)
|
30
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
31
|
+
http.use_ssl = true if uri.port == 443
|
32
|
+
response = http.request(request)
|
33
|
+
|
34
|
+
return CGI.unescapeHTML(response.body)
|
35
|
+
end
|
36
|
+
|
37
|
+
# This takes a pre-massaged text, and creates coupons from it.
|
38
|
+
def parse_feed(xml)
|
39
|
+
doc = Nokogiri::XML(xml)
|
40
|
+
total_num = doc.xpath("/root/total_num_pages/text()").to_s.to_i
|
41
|
+
if(total_num > 1 && !massager.nil?)
|
42
|
+
#need a way to solve for pagination universally
|
43
|
+
(1..total_num/records_per_page).each do |n|
|
44
|
+
page_number = {"page-number" => n}
|
45
|
+
params.merge(page_number)
|
46
|
+
feed = download_feed(uri, params, headers)
|
47
|
+
feed = massager.massage_feed(feed)
|
48
|
+
create_coupons_from_xml(feed)
|
49
|
+
end
|
50
|
+
else
|
51
|
+
create_coupons_from_xml(xml)
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
# END: The things in this block don't belong here.
|
56
|
+
=end
|
57
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
class AdvertiserFactory
|
2
|
+
|
3
|
+
FIELDS = [
|
4
|
+
{:xml => './advertiser-name/text()', :db => 'name'},
|
5
|
+
{:xml => './advertiser-name/text()', :db => 'nice_name'},
|
6
|
+
{:xml => './advertiser-id/text()', :db => 'advertiser_id'},
|
7
|
+
{:xml => './account-status/text()', :db => 'account_status'},
|
8
|
+
{:xml => './seven-day-epc/text()', :db => 'seven_day_epc'},
|
9
|
+
{:xml => './three-month-epc/text()', :db => 'three_month_epc'},
|
10
|
+
{:xml => './language/text()', :db => 'language'},
|
11
|
+
{:xml => './program-url/text()', :db => 'program_url'},
|
12
|
+
{:xml => './relationship-status/text()', :db => 'relationship_status'},
|
13
|
+
{:xml => './network-rank/text()', :db => 'network_rank'},
|
14
|
+
{:xml => './performance-incentives/text()', :db => 'performance_incentives'},
|
15
|
+
].freeze
|
16
|
+
|
17
|
+
def self.update_fields(advertiser, advertiser_xml)
|
18
|
+
#puts advertiser_xml.xpath(FIELDS[0][:xml]).to_s
|
19
|
+
|
20
|
+
#Iterate over every field and attempt to insert
|
21
|
+
FIELDS.each_with_index do |field, index|
|
22
|
+
|
23
|
+
xml_field = field[:xml]
|
24
|
+
db_field = field[:db]
|
25
|
+
|
26
|
+
#Magic baby!
|
27
|
+
unless db_field == 'nice_name'
|
28
|
+
advertiser[db_field] = advertiser_xml.xpath(xml_field).to_s
|
29
|
+
else
|
30
|
+
advertiser[db_field] = (advertiser_xml.xpath(xml_field).to_s).gsub(/[^A-Za-z0-9]+/, '-').strip.gsub(/\ +/, '-').downcase
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
#Update categories & Relationships
|
36
|
+
advertiser_xml.xpath('./primary-category').children.each_with_index do |children, index|
|
37
|
+
category_name = children.child.to_s
|
38
|
+
category_nice_name = category_name.gsub(/[^A-Za-z0-9]+/, '-').strip.gsub(/\ +/, '-').downcase
|
39
|
+
|
40
|
+
|
41
|
+
category = Category.find_or_create_by_nice_name(category_nice_name, :name => category_name)
|
42
|
+
AdvertiserCategoryLink.find_or_create_by_advertiser_id_and_category_id(
|
43
|
+
:advertiser_id => advertiser.id,
|
44
|
+
:category_id => category.id)
|
45
|
+
end
|
46
|
+
|
47
|
+
advertiser.save
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
#load teh helperzzz
|
2
|
+
require 'helpers/advertiser_factory.rb'
|
3
|
+
|
4
|
+
class AdvertiserParser
|
5
|
+
|
6
|
+
def create_cj_advertiser_from_xml(xml)
|
7
|
+
|
8
|
+
doc = Nokogiri::XML(xml)
|
9
|
+
|
10
|
+
doc.xpath("//advertiser").each do |advertiser_xml|
|
11
|
+
|
12
|
+
advertiser_id = advertiser_xml.xpath("./advertiser-id/text()").to_s.to_i
|
13
|
+
|
14
|
+
advertiser = Advertiser.find_or_create_by_advertiser_id(advertiser_id)
|
15
|
+
|
16
|
+
AdvertiserFactory.update_fields(advertiser, advertiser_xml)
|
17
|
+
|
18
|
+
# unless advertiser.nil?
|
19
|
+
# advertiser = AdvertiserFactory.update_advertiser(advertiser_xml)
|
20
|
+
# else
|
21
|
+
# advertiser = AdvertiserFactory.create_advertiser(advertiser_xml)
|
22
|
+
# end
|
23
|
+
end
|
24
|
+
# doc = FeedMapper.massage_feed(xml)
|
25
|
+
# puts "create coupons from xml====="
|
26
|
+
# (doc.xpath("//item")).each do |item|
|
27
|
+
# advertiser_id = item.xpath("./advertiser-id/text()")
|
28
|
+
# item_id = item.xpath("./item-id/text()")
|
29
|
+
#
|
30
|
+
# coupon = Coupon.find_by_advertiser_id_and_item_id(advertiser_id.to_s.to_i, item_id.to_s.to_i)
|
31
|
+
#
|
32
|
+
# if(!coupon.nil?)
|
33
|
+
# coupon = CouponFactory.update_coupon_with_item(coupon,item)
|
34
|
+
# else
|
35
|
+
# coupon = CouponFactory.create_coupon_from_item(item, @site_id, @vendor_id)
|
36
|
+
# end
|
37
|
+
# coupon.save
|
38
|
+
# end
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
Binary file
|
data/lib/tasks/.gitkeep
ADDED
File without changes
|
@@ -0,0 +1,33 @@
|
|
1
|
+
namespace :coupon_feeds do
|
2
|
+
desc "Populate Commission Junction Data"
|
3
|
+
task :download_cj_data => :environment do
|
4
|
+
website_id = Site.find_by_nice_name("amplified-media").id
|
5
|
+
records_per_page = 100
|
6
|
+
type_of_promotion = "coupon"
|
7
|
+
params = {"website-id" => website_id,
|
8
|
+
"advertiser-ids" => "joined",
|
9
|
+
"records-per-page" => records_per_page,
|
10
|
+
"promotion-type" => type_of_promotion,
|
11
|
+
"page-number" => 1}
|
12
|
+
ca = CouponAggregator.new(params, CouponAggregator::CJ)
|
13
|
+
ca.aggregate(website_id)
|
14
|
+
end
|
15
|
+
|
16
|
+
desc "Populate For Me To Coupon Data"
|
17
|
+
task :download_fmtc_data => :environment do
|
18
|
+
ca = CouponAggregator.new({}, CouponAggregator::FMTC)
|
19
|
+
ca.aggregate(nil)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
namespace :cj do
|
24
|
+
desc "Work on Commission Junction's Feeds"
|
25
|
+
task :get_advertisers => :environment do
|
26
|
+
params = {
|
27
|
+
"advertiser-ids"=> "joined"
|
28
|
+
}
|
29
|
+
cj_advertisers = CJAdvertisers.new(params)
|
30
|
+
|
31
|
+
cj_advertisers.collect_advertisers
|
32
|
+
end
|
33
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aggregator-gem
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.9
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -13,7 +13,7 @@ date: 2011-10-14 00:00:00.000000000Z
|
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
16
|
-
requirement: &
|
16
|
+
requirement: &15966360 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - =
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 3.0.9
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *15966360
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: mysql2
|
27
|
-
requirement: &
|
27
|
+
requirement: &15965540 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - <
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: '0.3'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *15965540
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: nokogiri
|
38
|
-
requirement: &
|
38
|
+
requirement: &15964140 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - =
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: 1.5.0
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *15964140
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: shoulda
|
49
|
-
requirement: &
|
49
|
+
requirement: &15963460 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :development
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *15963460
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: bundler
|
60
|
-
requirement: &
|
60
|
+
requirement: &15962660 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ~>
|
@@ -65,10 +65,10 @@ dependencies:
|
|
65
65
|
version: 1.0.0
|
66
66
|
type: :development
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *15962660
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: jeweler
|
71
|
-
requirement: &
|
71
|
+
requirement: &15962100 !ruby/object:Gem::Requirement
|
72
72
|
none: false
|
73
73
|
requirements:
|
74
74
|
- - ~>
|
@@ -76,10 +76,10 @@ dependencies:
|
|
76
76
|
version: 1.6.4
|
77
77
|
type: :development
|
78
78
|
prerelease: false
|
79
|
-
version_requirements: *
|
79
|
+
version_requirements: *15962100
|
80
80
|
- !ruby/object:Gem::Dependency
|
81
81
|
name: rcov
|
82
|
-
requirement: &
|
82
|
+
requirement: &15961600 !ruby/object:Gem::Requirement
|
83
83
|
none: false
|
84
84
|
requirements:
|
85
85
|
- - ! '>='
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
version: '0'
|
88
88
|
type: :development
|
89
89
|
prerelease: false
|
90
|
-
version_requirements: *
|
90
|
+
version_requirements: *15961600
|
91
91
|
description: Cut media coupon aggregator
|
92
92
|
email: allentsai87@gmail.com
|
93
93
|
executables: []
|
@@ -138,7 +138,17 @@ files:
|
|
138
138
|
- lib/app/models/sites_vendors.rb
|
139
139
|
- lib/app/models/vendor.rb
|
140
140
|
- lib/app/views/layouts/application.html.erb
|
141
|
+
- lib/cj_advertisers.rb
|
141
142
|
- lib/coupon_aggregator.rb
|
143
|
+
- lib/coupon_factory.rb
|
144
|
+
- lib/feed_mapper.rb
|
145
|
+
- lib/feed_parser.rb
|
146
|
+
- lib/helpers/advertiser_factory.rb
|
147
|
+
- lib/helpers/advertiser_parser.rb
|
148
|
+
- lib/html_connection/.html_connector.rb.swp
|
149
|
+
- lib/html_connection/html_connection_helper.rb
|
150
|
+
- lib/tasks/.gitkeep
|
151
|
+
- lib/tasks/coupon_feeds.rake
|
142
152
|
- tasks/coupon_aggregator_tasks.rake
|
143
153
|
- test/helper.rb
|
144
154
|
- test/test_coupon_aggregator.rb
|
@@ -157,7 +167,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
157
167
|
version: '0'
|
158
168
|
segments:
|
159
169
|
- 0
|
160
|
-
hash:
|
170
|
+
hash: -165068521969160248
|
161
171
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
162
172
|
none: false
|
163
173
|
requirements:
|