primal 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.project +13 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +29 -0
- data/Rakefile +1 -0
- data/lib/primal.rb +5 -0
- data/lib/primal/AlchemyAPIWrapper.rb +340 -0
- data/lib/primal/PrimalAccess.rb +204 -0
- data/lib/primal/version.rb +3 -0
- data/primal.gemspec +28 -0
- metadata +111 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: dfe6776b27a2665fb3bf680618ca23dcbeb623d4
|
4
|
+
data.tar.gz: c3e4f6c2e0757efa8af11ce30df9fe7842c91cdd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6a1c0414e1d069298d3d30973483fac9b55d80644dcbe2917da33375012108da4e8472e8cdd211a0cfcd0dd41062f1ee291f0b6de33a381e69769f3f3b0cd9cd
|
7
|
+
data.tar.gz: 7e211be3602d8f589aa7bb7697482986c23cc97dbc78984d627b69bd060ae2baac1bc0072e56dc14343d7f1efc2e2b509b0f16d2910b8fe67fe393ebac405109
|
data/.gitignore
ADDED
data/.project
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<projectDescription>
|
3
|
+
<name>primal</name>
|
4
|
+
<comment></comment>
|
5
|
+
<projects>
|
6
|
+
</projects>
|
7
|
+
<buildSpec>
|
8
|
+
</buildSpec>
|
9
|
+
<natures>
|
10
|
+
<nature>com.aptana.projects.webnature</nature>
|
11
|
+
<nature>com.aptana.ruby.core.rubynature</nature>
|
12
|
+
</natures>
|
13
|
+
</projectDescription>
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 w2davids@gmail.com
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# Primal
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'primal'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install primal
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/lib/primal.rb
ADDED
@@ -0,0 +1,340 @@
|
|
1
|
+
# We need to use some ruby gems
|
2
|
+
require 'rubygems'
|
3
|
+
require 'uri'
|
4
|
+
require 'httparty'
|
5
|
+
require 'json'
|
6
|
+
|
7
|
+
module Primal
|
8
|
+
|
9
|
+
|
10
|
+
#
|
11
|
+
# The InputTermExtraction class abstracts accessing AlchemyAPI
|
12
|
+
# (http://www.alchemyapi.com/) to extract important information from Web
|
13
|
+
# pages/News articles/blog posts/plain text.
|
14
|
+
#
|
15
|
+
# The main function is getPrimalRequest, which accepts a string that represents
|
16
|
+
# a Web page URL or some text, and build a Primal topic URI that will direct the
|
17
|
+
# user to the Primal Web App.
|
18
|
+
#
|
19
|
+
class InputTermExtraction
|
20
|
+
include HTTParty
|
21
|
+
|
22
|
+
# Uncomment this next line to see what HTTParty is doing
|
23
|
+
# debug_output $stderr
|
24
|
+
|
25
|
+
# Set this to true/false in order to turn on/off debugging of this class
|
26
|
+
@@debugMe = true
|
27
|
+
|
28
|
+
@@alchemyRoot = "http://access.alchemyapi.com/calls"
|
29
|
+
@@alchemyURL = "#{@@alchemyRoot}/url"
|
30
|
+
@@alchemyText = "#{@@alchemyRoot}/text"
|
31
|
+
|
32
|
+
# change these variables to modify how the Primal request is built
|
33
|
+
@@entitiesLimit = 1
|
34
|
+
@@keywordsLimit = 3
|
35
|
+
|
36
|
+
# We ignore keywords that intersect with entities of the following types:
|
37
|
+
@@categoryIgnores = {
|
38
|
+
'person' => 1,
|
39
|
+
'organization' => 1,
|
40
|
+
'city' => 1,
|
41
|
+
'company' => 1,
|
42
|
+
'continent' => 1,
|
43
|
+
'country' => 1,
|
44
|
+
'region' => 1,
|
45
|
+
'stateorcountry' => 1,
|
46
|
+
'geographicfeature' => 1
|
47
|
+
}
|
48
|
+
|
49
|
+
#
|
50
|
+
# Constructor for the InputTermExtraction class
|
51
|
+
#
|
52
|
+
# Pass in the Api Key for Alchemy services.
|
53
|
+
# You can register for a free API key here:
|
54
|
+
# http://www.alchemyapi.com/api/register.html
|
55
|
+
# to test your application.
|
56
|
+
# Please read the license of Alchemy API.
|
57
|
+
#
|
58
|
+
def initialize(alchemyApiKey)
|
59
|
+
@alchemyApiKey = alchemyApiKey
|
60
|
+
end
|
61
|
+
|
62
|
+
#
|
63
|
+
# Receives a string that represents a Web page URL or some text, and returns
|
64
|
+
# a Primal topic URI.
|
65
|
+
#
|
66
|
+
# Returns nil on error
|
67
|
+
#
|
68
|
+
def getPrimalRequest(urlOrText)
|
69
|
+
if isURI(urlOrText)
|
70
|
+
getPrimalRequestURL(urlOrText)
|
71
|
+
else
|
72
|
+
getPrimalRequestTEXT(urlOrText)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
#
|
77
|
+
# Indicates whether or not a given string represents a URL
|
78
|
+
#
|
79
|
+
def isURI(string)
|
80
|
+
uri = URI.parse(string)
|
81
|
+
%w( http https ).include?(uri.scheme)
|
82
|
+
rescue URI::BadURIError
|
83
|
+
false
|
84
|
+
rescue URI::InvalidURIError
|
85
|
+
false
|
86
|
+
end
|
87
|
+
|
88
|
+
#
|
89
|
+
# Processes the given URL at Alchemy and then translates the results to a
|
90
|
+
# valid Primal URL
|
91
|
+
#
|
92
|
+
def getPrimalRequestURL(urlToProcess)
|
93
|
+
if @@debugMe
|
94
|
+
$stderr.puts "Extracting information from URL..."
|
95
|
+
end
|
96
|
+
|
97
|
+
# get category of the Web page
|
98
|
+
categoryJSON = getAlchemy("#{@@alchemyURL}/URLGetCategory",
|
99
|
+
:query => {
|
100
|
+
:outputMode => 'json',
|
101
|
+
:apikey => @alchemyApiKey,
|
102
|
+
:url => urlToProcess
|
103
|
+
})
|
104
|
+
|
105
|
+
# get entities in the Web page
|
106
|
+
entitiesJSON = getAlchemy("#{@@alchemyURL}/URLGetRankedNamedEntities",
|
107
|
+
:query => {
|
108
|
+
:outputMode => 'json',
|
109
|
+
:apikey => @alchemyApiKey,
|
110
|
+
:url => urlToProcess
|
111
|
+
})
|
112
|
+
|
113
|
+
# get keywords from the Web page
|
114
|
+
keywordsJSON = getAlchemy("#{@@alchemyURL}/URLGetRankedKeywords",
|
115
|
+
:query => {
|
116
|
+
:outputMode => 'json',
|
117
|
+
:apikey => @alchemyApiKey,
|
118
|
+
:url => urlToProcess
|
119
|
+
})
|
120
|
+
|
121
|
+
buildPrimalRequest(categoryJSON, entitiesJSON, keywordsJSON)
|
122
|
+
end
|
123
|
+
|
124
|
+
#
|
125
|
+
# Processes the given Text at Alchemy and then translates the results to a
|
126
|
+
# valid Primal URL
|
127
|
+
#
|
128
|
+
def getPrimalRequestTEXT(textToProcess)
|
129
|
+
if @@debugMe
|
130
|
+
$stderr.puts "Extracting information from text..."
|
131
|
+
end
|
132
|
+
|
133
|
+
# get category
|
134
|
+
categoryJSON = postAlchemy("#{@@alchemyText}/TextGetCategory",
|
135
|
+
:query => {
|
136
|
+
:outputMode => 'json',
|
137
|
+
:apikey => @alchemyApiKey,
|
138
|
+
:text => textToProcess
|
139
|
+
})
|
140
|
+
|
141
|
+
# get entities
|
142
|
+
entitiesJSON = postAlchemy("#{@@alchemyText}/TextGetRankedNamedEntities",
|
143
|
+
:query => {
|
144
|
+
:outputMode => 'json',
|
145
|
+
:apikey => @alchemyApiKey,
|
146
|
+
:text => textToProcess
|
147
|
+
})
|
148
|
+
|
149
|
+
# get keywords
|
150
|
+
keywordsJSON = postAlchemy("#{@@alchemyText}/TextGetRankedKeywords",
|
151
|
+
:query => {
|
152
|
+
:outputMode => 'json',
|
153
|
+
:apikey => @alchemyApiKey,
|
154
|
+
:text => textToProcess
|
155
|
+
})
|
156
|
+
|
157
|
+
buildPrimalRequest(categoryJSON, entitiesJSON, keywordsJSON)
|
158
|
+
end
|
159
|
+
|
160
|
+
#
|
161
|
+
# Uses the deconstructed Alchemy information to create a valid Primal URL
|
162
|
+
#
|
163
|
+
def buildPrimalRequest(categoryJSON, entitiesJSON, keywordsJSON)
|
164
|
+
# Check if any of the extractions failed
|
165
|
+
if !categoryJSON or !entitiesJSON or !keywordsJSON
|
166
|
+
$stderr.puts "Cannot build Primal request. Alchemy failed to extract information."
|
167
|
+
return nil
|
168
|
+
end
|
169
|
+
|
170
|
+
if @@debugMe
|
171
|
+
$stderr.puts "Building Primal request..."
|
172
|
+
end
|
173
|
+
|
174
|
+
### Get information required for building a Primal request
|
175
|
+
# Get the category from the extracted data
|
176
|
+
category = rewriteCategory(categoryJSON['category'])
|
177
|
+
|
178
|
+
if @@debugMe
|
179
|
+
$stderr.puts "Category = #{category}"
|
180
|
+
end
|
181
|
+
|
182
|
+
### Select top entities from all extracted entities
|
183
|
+
entitiesList = entitiesJSON['entities'].collect { |entity|
|
184
|
+
entity['text'].downcase
|
185
|
+
}[0, @@entitiesLimit]
|
186
|
+
|
187
|
+
if @@debugMe
|
188
|
+
prettified = entitiesJSON['entities'].collect { |entity|
|
189
|
+
entity['text']
|
190
|
+
}.join(', ')
|
191
|
+
$stderr.puts "Entities = #{prettified}"
|
192
|
+
end
|
193
|
+
|
194
|
+
### Select top keywords from all extracted keywords
|
195
|
+
# Remove keywords that intersect with entities of the types in @@categoryIgnores
|
196
|
+
allEntities = entitiesJSON['entities'].select { |entity|
|
197
|
+
@@categoryIgnores.has_key? entity['type'].downcase
|
198
|
+
}.collect { |entity|
|
199
|
+
entity['text'].downcase
|
200
|
+
}
|
201
|
+
|
202
|
+
if @@debugMe
|
203
|
+
prettified = keywordsJSON['keywords'].collect { |keyword|
|
204
|
+
keyword['text']
|
205
|
+
}.join(', ')
|
206
|
+
$stderr.puts "Keywords = #{prettified}"
|
207
|
+
end
|
208
|
+
|
209
|
+
keywordsList = keywordsJSON['keywords'].select { |keyword|
|
210
|
+
normalizedKw = keyword['text'].downcase
|
211
|
+
intersectsWithEntity = !(allEntities.select { |entity|
|
212
|
+
entity.include? normalizedKw or normalizedKw.include? entity
|
213
|
+
}.empty?)
|
214
|
+
# Ignore keywords > 4 words or those that intersect with entities
|
215
|
+
normalizedKw.split.size < 5 && !intersectsWithEntity
|
216
|
+
}.collect { |keyword|
|
217
|
+
keyword['text'].downcase
|
218
|
+
}
|
219
|
+
|
220
|
+
# Remove repeated keywords
|
221
|
+
keywordsList = getNonRepeatedKeywords(keywordsList)
|
222
|
+
|
223
|
+
### Build Primal topic URI
|
224
|
+
primalRequest = ""
|
225
|
+
unless category.nil? then primalRequest = "/" + category end
|
226
|
+
if entitiesList.size > 0 then primalRequest = primalRequest + "/" + entitiesList.join("/") end
|
227
|
+
if keywordsList.size > 0 then primalRequest = primalRequest + "/" + keywordsList.join(";") end
|
228
|
+
if @@debugMe
|
229
|
+
$stderr.puts "Primal request = #{primalRequest}"
|
230
|
+
end
|
231
|
+
URI::encode(primalRequest)
|
232
|
+
end #function
|
233
|
+
|
234
|
+
#
|
235
|
+
# Returns the top @@keywordsLimit keywords, ignoring those contained
|
236
|
+
# within other keywords
|
237
|
+
#
|
238
|
+
def getNonRepeatedKeywords(keywordsList)
|
239
|
+
# If there is less than @@keywordsLimit, or if the first @@keywordsLimit keywords
|
240
|
+
# are unique, return the first @@keywordsLimit keywords
|
241
|
+
repeatedKeywords = getRepeatedKeywords(keywordsList[0, @@keywordsLimit])
|
242
|
+
if keywordsList.length <= @@keywordsLimit or repeatedKeywords.empty?
|
243
|
+
keywordsList[0, @@keywordsLimit]
|
244
|
+
else
|
245
|
+
# Remove repeated elements from the first @@keywordsLimit keywords, and recursively
|
246
|
+
# call this function
|
247
|
+
getNonRepeatedKeywords(keywordsList - repeatedKeywords)
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
#
|
252
|
+
# Returns any repeated words in the list
|
253
|
+
#
|
254
|
+
def getRepeatedKeywords(keywordsList)
|
255
|
+
keywordsList.select { |keyword|
|
256
|
+
not(keywordsList.select { |other|
|
257
|
+
other != keyword and other.include? keyword
|
258
|
+
}.empty?)
|
259
|
+
}
|
260
|
+
end
|
261
|
+
|
262
|
+
#
|
263
|
+
# Modifies the extracted category string to become a clear topic in the Primal
|
264
|
+
# request.
|
265
|
+
#
|
266
|
+
# AlchemyAPI categorizes text into a limited set of category types.
|
267
|
+
#
|
268
|
+
# See http://www.alchemyapi.com/api/categ/categs.html for a complete list.
|
269
|
+
#
|
270
|
+
# Some of the cateogy type names have two strings concatenated by an
|
271
|
+
# underscore character. This function selects one of the two strings (or a
|
272
|
+
# totally new string) to be the topic in the Primal request.
|
273
|
+
#
|
274
|
+
def rewriteCategory(category)
|
275
|
+
case category
|
276
|
+
when "unknown" # AlchemyAPI failed to classify the text
|
277
|
+
category = nil
|
278
|
+
when "arts_entertainment"
|
279
|
+
category = "arts" # rewrite to 'arts'
|
280
|
+
when "computer_internet"
|
281
|
+
category = "technology" # rewrite to 'technology', a clearer topic for this category
|
282
|
+
when "culture_politics"
|
283
|
+
category = "politics" # rewrite to 'politics'
|
284
|
+
when "law_crime"
|
285
|
+
category = "law" # rewrite to 'law'
|
286
|
+
when "science_technology"
|
287
|
+
category = "science" # rewrite to 'science'
|
288
|
+
else
|
289
|
+
# The previous conditions should cover all the categories extracted by
|
290
|
+
# Alchemy. In case of a new category that contains an underscore, replace
|
291
|
+
# it and keep the two words as the topic.
|
292
|
+
category = category.sub('_', ' ')
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
#
|
297
|
+
# Perform a POST request to Alchemy service URL and return the response as a
|
298
|
+
# JSON object
|
299
|
+
#
|
300
|
+
# Returns nil on error
|
301
|
+
#
|
302
|
+
def postAlchemy(serviceURL, parameters)
|
303
|
+
response = self.class.post(serviceURL, parameters)
|
304
|
+
returnAlchemyResponseJSON(response)
|
305
|
+
end
|
306
|
+
|
307
|
+
#
|
308
|
+
# Perform a GET request to Alchemy service URL and return the
|
309
|
+
# response as a JSON object
|
310
|
+
#
|
311
|
+
# Returns nil on error
|
312
|
+
#
|
313
|
+
def getAlchemy(serviceURL, parameters)
|
314
|
+
response = self.class.get(serviceURL, parameters)
|
315
|
+
returnAlchemyResponseJSON(response)
|
316
|
+
end
|
317
|
+
|
318
|
+
#
|
319
|
+
# Return the body of the response in a JSON object
|
320
|
+
# or nil on error
|
321
|
+
#
|
322
|
+
def returnAlchemyResponseJSON(response)
|
323
|
+
code = response.code
|
324
|
+
body = response.body
|
325
|
+
bodyJSON = JSON.parse(body)
|
326
|
+
|
327
|
+
# A statusInfo field contains the details of the error
|
328
|
+
if bodyJSON['status'] != "OK"
|
329
|
+
puts bodyJSON['statusInfo']
|
330
|
+
nil
|
331
|
+
else
|
332
|
+
bodyJSON
|
333
|
+
end
|
334
|
+
end
|
335
|
+
end
|
336
|
+
Ï
|
337
|
+
|
338
|
+
end
|
339
|
+
|
340
|
+
|
@@ -0,0 +1,204 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'httparty'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Primal
|
6
|
+
#
|
7
|
+
# The PrimalAccess class abstracts the access to Primal such
|
8
|
+
# that you can call simple methods on it to get what you need.
|
9
|
+
#
|
10
|
+
class PrimalAccess
|
11
|
+
include HTTParty
|
12
|
+
base_uri 'https://data.primal.com'
|
13
|
+
# Uncomment this next line to see what HTTParty is doing
|
14
|
+
# debug_output $stderr
|
15
|
+
# Set this to false in order to turn off debugging of this class
|
16
|
+
@@debugMe = true
|
17
|
+
#
|
18
|
+
# Constructor for the PrimalAccess class
|
19
|
+
#
|
20
|
+
# Pass in the username and password of the user you're going
|
21
|
+
# to access in order to construct and object that will work
|
22
|
+
# with that user
|
23
|
+
#
|
24
|
+
def initialize(appId, appKey, username, password)
|
25
|
+
@headers = {
|
26
|
+
:headers => {
|
27
|
+
'Primal-App-ID' => appId,
|
28
|
+
'Primal-App-Key' => appKey
|
29
|
+
},
|
30
|
+
:basic_auth => {
|
31
|
+
:username => username,
|
32
|
+
:password => password
|
33
|
+
}
|
34
|
+
}
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Sometimes we're going to get topics that are complex (i.e. they contain a
|
39
|
+
# scheme and host) and we want to simplify those. Because we're not making
|
40
|
+
# calls with bare URLs but have told HTTParty what the base_uri is, we need to
|
41
|
+
# pull that base uri off of the topic, should it be there.
|
42
|
+
#
|
43
|
+
def extractJustTopic(topic)
|
44
|
+
topic.sub(%r{https://.*?/}, '/')
|
45
|
+
end
|
46
|
+
|
47
|
+
#
|
48
|
+
# POSTs a new topic to Primal in order to seed that topic.
|
49
|
+
#
|
50
|
+
# The 'topic' parameter will be used to construct a POST URL
|
51
|
+
# that looks like "/topic"
|
52
|
+
#
|
53
|
+
# Returns two values: the response code and the body.
|
54
|
+
# Anything but a response code of 201 is to be considered
|
55
|
+
# an error.
|
56
|
+
#
|
57
|
+
def postNewTopic(topic, opts = {})
|
58
|
+
topic = extractJustTopic(topic)
|
59
|
+
count = 0
|
60
|
+
code = 400
|
61
|
+
body = ''
|
62
|
+
options = @headers.merge(opts)
|
63
|
+
while (count < 5)
|
64
|
+
if @@debugMe
|
65
|
+
$stderr.puts "POSTing to #{topic}"
|
66
|
+
end
|
67
|
+
response = self.class.post("#{topic}", options)
|
68
|
+
code = response.code
|
69
|
+
body = response.body
|
70
|
+
#
|
71
|
+
# 400 - bad request
|
72
|
+
# 401 - application not authorized to access the user's account
|
73
|
+
# 403 - application not authorized to use Primal
|
74
|
+
#
|
75
|
+
if code >= 400 && code <= 403
|
76
|
+
if @@debugMe
|
77
|
+
$stderr.puts "POST received a #{code}"
|
78
|
+
end
|
79
|
+
break
|
80
|
+
#
|
81
|
+
# 429 - application has reached its request limit for the moment
|
82
|
+
#
|
83
|
+
elsif code == 429
|
84
|
+
# Sleep for 10 seconds
|
85
|
+
if @@debugMe
|
86
|
+
$stderr.puts "Got a 429. Waiting (#{count})."
|
87
|
+
end
|
88
|
+
sleep 10
|
89
|
+
count += 1
|
90
|
+
#
|
91
|
+
# 201 - success
|
92
|
+
#
|
93
|
+
elsif code == 201
|
94
|
+
if @@debugMe
|
95
|
+
$stderr.puts "POST successful"
|
96
|
+
end
|
97
|
+
break
|
98
|
+
else
|
99
|
+
abort "Received unexpected response code (#{code}) for POST #{uri}"
|
100
|
+
end
|
101
|
+
end
|
102
|
+
return code, body
|
103
|
+
end
|
104
|
+
|
105
|
+
#
|
106
|
+
# Uses the pre-existing topic to filter the default source
|
107
|
+
# of content through the interest network defined by the topic.
|
108
|
+
#
|
109
|
+
# The given parameter will be used to construct a GET URL
|
110
|
+
# that looks like "/topic"
|
111
|
+
#
|
112
|
+
# You can pass a dictionary of optional arguments that will
|
113
|
+
# be merged in to the query parameters, if you wish.
|
114
|
+
# e.g.
|
115
|
+
# { :"primal:contentScore:min" => 0.7 }
|
116
|
+
# { :"primal:contentCount:max" => 5 }
|
117
|
+
# { :contentSource => MyDataSource } ... or ...
|
118
|
+
# { :contentSource => PrimalSource }
|
119
|
+
#
|
120
|
+
# Returns two values: the response code, and the body.
|
121
|
+
# If successful (i.e. a response code of 200) then the body
|
122
|
+
# will be the JSON payload of the filtered content.
|
123
|
+
#
|
124
|
+
def filterContent(topic, opts = {})
|
125
|
+
topic = extractJustTopic(topic)
|
126
|
+
count = 0
|
127
|
+
code = 400
|
128
|
+
body = ''
|
129
|
+
options = @headers.merge({ :query => {
|
130
|
+
:timeOut => 'max'
|
131
|
+
}.merge(opts)
|
132
|
+
})
|
133
|
+
while (count < 10)
|
134
|
+
if @@debugMe
|
135
|
+
$stderr.puts "GETting #{topic}"
|
136
|
+
end
|
137
|
+
response = self.class.get("#{topic}", options)
|
138
|
+
code = response.code
|
139
|
+
body = response.body
|
140
|
+
#
|
141
|
+
# 400 - bad request
|
142
|
+
# 401 - application not authorized to access the user's account
|
143
|
+
# 403 - application not authorized to use Primal
|
144
|
+
# 404 - object not found
|
145
|
+
#
|
146
|
+
if code >= 400 && code <= 404
|
147
|
+
if @@debugMe
|
148
|
+
$stderr.puts "GET received a #{code}"
|
149
|
+
end
|
150
|
+
break
|
151
|
+
#
|
152
|
+
# 429 - application has reached its request limit for the moment
|
153
|
+
#
|
154
|
+
elsif code == 429
|
155
|
+
if @@debugMe
|
156
|
+
$stderr.puts "Got a 429. Waiting (#{count})."
|
157
|
+
end
|
158
|
+
# Sleep for 10 seconds
|
159
|
+
sleep 10
|
160
|
+
# We don't allow as many retries when we might be throttled
|
161
|
+
count += 2
|
162
|
+
#
|
163
|
+
# 200 - success
|
164
|
+
#
|
165
|
+
elsif code == 200
|
166
|
+
if @@debugMe
|
167
|
+
$stderr.puts "Results are complete"
|
168
|
+
end
|
169
|
+
break
|
170
|
+
#
|
171
|
+
# We don't know what happened but it can't be good
|
172
|
+
#
|
173
|
+
else
|
174
|
+
abort "Received unexpected response code (#{code}) for GET #{topic}"
|
175
|
+
end
|
176
|
+
end
|
177
|
+
return code, body
|
178
|
+
end
|
179
|
+
|
180
|
+
#
|
181
|
+
# This is a convenience method that will POST the topic to
|
182
|
+
# Primal and then filter the default source of content through the
|
183
|
+
# resulting interest network.
|
184
|
+
#
|
185
|
+
# The response from this method is a bit less clear than using
|
186
|
+
# a POST and filter explicitly, since you may not know which
|
187
|
+
# one of the two operations has failed (assuming a failure).
|
188
|
+
#
|
189
|
+
# Returns two values: the response code and the body. The only
|
190
|
+
# successful response code from this method is 200. If
|
191
|
+
# successful then the body contains the JSON payload of the
|
192
|
+
# filtered content.
|
193
|
+
#
|
194
|
+
def postThenFilter(topic, opts = {})
|
195
|
+
code, body = postNewTopic(topic)
|
196
|
+
if code == 201
|
197
|
+
code, body = filterContent(topic, opts)
|
198
|
+
end
|
199
|
+
return code, body
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
end
|
204
|
+
|
data/primal.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'primal/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "primal"
|
8
|
+
spec.version = Primal::VERSION
|
9
|
+
spec.authors = ["w2davids@gmail.com"]
|
10
|
+
spec.email = ["w2davids@gmail.com"]
|
11
|
+
spec.description = %q{Ruby gem for accessing Primal Data web services}
|
12
|
+
spec.summary = %q{Ruby client to access Primal Data web services}
|
13
|
+
spec.homepage = ""
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.rubyforge_project = "primal"
|
17
|
+
|
18
|
+
spec.files = `git ls-files`.split($/)
|
19
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
20
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
21
|
+
spec.require_paths = ["lib"]
|
22
|
+
|
23
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
24
|
+
spec.add_development_dependency "rake"
|
25
|
+
spec.add_development_dependency "json"
|
26
|
+
spec.add_development_dependency "httparty"
|
27
|
+
|
28
|
+
end
|
metadata
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: primal
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- w2davids@gmail.com
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-05-20 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.3'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.3'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: json
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: httparty
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: Ruby gem for accessing Primal Data web services
|
70
|
+
email:
|
71
|
+
- w2davids@gmail.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- .gitignore
|
77
|
+
- .project
|
78
|
+
- Gemfile
|
79
|
+
- LICENSE.txt
|
80
|
+
- README.md
|
81
|
+
- Rakefile
|
82
|
+
- lib/primal.rb
|
83
|
+
- lib/primal/AlchemyAPIWrapper.rb
|
84
|
+
- lib/primal/PrimalAccess.rb
|
85
|
+
- lib/primal/version.rb
|
86
|
+
- primal.gemspec
|
87
|
+
homepage: ''
|
88
|
+
licenses:
|
89
|
+
- MIT
|
90
|
+
metadata: {}
|
91
|
+
post_install_message:
|
92
|
+
rdoc_options: []
|
93
|
+
require_paths:
|
94
|
+
- lib
|
95
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - '>='
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: '0'
|
100
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - '>='
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
requirements: []
|
106
|
+
rubyforge_project: primal
|
107
|
+
rubygems_version: 2.0.3
|
108
|
+
signing_key:
|
109
|
+
specification_version: 4
|
110
|
+
summary: Ruby client to access Primal Data web services
|
111
|
+
test_files: []
|