primal 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.project +13 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +29 -0
- data/Rakefile +1 -0
- data/lib/primal.rb +5 -0
- data/lib/primal/AlchemyAPIWrapper.rb +340 -0
- data/lib/primal/PrimalAccess.rb +204 -0
- data/lib/primal/version.rb +3 -0
- data/primal.gemspec +28 -0
- metadata +111 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: dfe6776b27a2665fb3bf680618ca23dcbeb623d4
|
4
|
+
data.tar.gz: c3e4f6c2e0757efa8af11ce30df9fe7842c91cdd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6a1c0414e1d069298d3d30973483fac9b55d80644dcbe2917da33375012108da4e8472e8cdd211a0cfcd0dd41062f1ee291f0b6de33a381e69769f3f3b0cd9cd
|
7
|
+
data.tar.gz: 7e211be3602d8f589aa7bb7697482986c23cc97dbc78984d627b69bd060ae2baac1bc0072e56dc14343d7f1efc2e2b509b0f16d2910b8fe67fe393ebac405109
|
data/.gitignore
ADDED
data/.project
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<projectDescription>
|
3
|
+
<name>primal</name>
|
4
|
+
<comment></comment>
|
5
|
+
<projects>
|
6
|
+
</projects>
|
7
|
+
<buildSpec>
|
8
|
+
</buildSpec>
|
9
|
+
<natures>
|
10
|
+
<nature>com.aptana.projects.webnature</nature>
|
11
|
+
<nature>com.aptana.ruby.core.rubynature</nature>
|
12
|
+
</natures>
|
13
|
+
</projectDescription>
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 w2davids@gmail.com
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# Primal
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'primal'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install primal
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/lib/primal.rb
ADDED
@@ -0,0 +1,340 @@
|
|
1
|
+
# We need to use some ruby gems
|
2
|
+
require 'rubygems'
|
3
|
+
require 'uri'
|
4
|
+
require 'httparty'
|
5
|
+
require 'json'
|
6
|
+
|
7
|
+
module Primal
|
8
|
+
|
9
|
+
|
10
|
+
#
|
11
|
+
# The InputTermExtraction class abstracts accessing AlchemyAPI
|
12
|
+
# (http://www.alchemyapi.com/) to extract important information from Web
|
13
|
+
# pages/News articles/blog posts/plain text.
|
14
|
+
#
|
15
|
+
# The main function is getPrimalRequest, which accepts a string that represents
|
16
|
+
# a Web page URL or some text, and build a Primal topic URI that will direct the
|
17
|
+
# user to the Primal Web App.
|
18
|
+
#
|
19
|
+
class InputTermExtraction
|
20
|
+
include HTTParty
|
21
|
+
|
22
|
+
# Uncomment this next line to see what HTTParty is doing
|
23
|
+
# debug_output $stderr
|
24
|
+
|
25
|
+
# Set this to true/false in order to turn on/off debugging of this class
|
26
|
+
@@debugMe = true
|
27
|
+
|
28
|
+
@@alchemyRoot = "http://access.alchemyapi.com/calls"
|
29
|
+
@@alchemyURL = "#{@@alchemyRoot}/url"
|
30
|
+
@@alchemyText = "#{@@alchemyRoot}/text"
|
31
|
+
|
32
|
+
# change these variables to modify how the Primal request is built
|
33
|
+
@@entitiesLimit = 1
|
34
|
+
@@keywordsLimit = 3
|
35
|
+
|
36
|
+
# We ignore keywords that intersect with entities of the following types:
|
37
|
+
@@categoryIgnores = {
|
38
|
+
'person' => 1,
|
39
|
+
'organization' => 1,
|
40
|
+
'city' => 1,
|
41
|
+
'company' => 1,
|
42
|
+
'continent' => 1,
|
43
|
+
'country' => 1,
|
44
|
+
'region' => 1,
|
45
|
+
'stateorcountry' => 1,
|
46
|
+
'geographicfeature' => 1
|
47
|
+
}
|
48
|
+
|
49
|
+
#
|
50
|
+
# Constructor for the InputTermExtraction class
|
51
|
+
#
|
52
|
+
# Pass in the Api Key for Alchemy services.
|
53
|
+
# You can register for a free API key here:
|
54
|
+
# http://www.alchemyapi.com/api/register.html
|
55
|
+
# to test your application.
|
56
|
+
# Please read the license of Alchemy API.
|
57
|
+
#
|
58
|
+
def initialize(alchemyApiKey)
|
59
|
+
@alchemyApiKey = alchemyApiKey
|
60
|
+
end
|
61
|
+
|
62
|
+
#
|
63
|
+
# Receives a string that represents a Web page URL or some text, and returns
|
64
|
+
# a Primal topic URI.
|
65
|
+
#
|
66
|
+
# Returns nil on error
|
67
|
+
#
|
68
|
+
def getPrimalRequest(urlOrText)
|
69
|
+
if isURI(urlOrText)
|
70
|
+
getPrimalRequestURL(urlOrText)
|
71
|
+
else
|
72
|
+
getPrimalRequestTEXT(urlOrText)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
#
|
77
|
+
# Indicates whether or not a given string represents a URL
|
78
|
+
#
|
79
|
+
def isURI(string)
|
80
|
+
uri = URI.parse(string)
|
81
|
+
%w( http https ).include?(uri.scheme)
|
82
|
+
rescue URI::BadURIError
|
83
|
+
false
|
84
|
+
rescue URI::InvalidURIError
|
85
|
+
false
|
86
|
+
end
|
87
|
+
|
88
|
+
#
|
89
|
+
# Processes the given URL at Alchemy and then translates the results to a
|
90
|
+
# valid Primal URL
|
91
|
+
#
|
92
|
+
def getPrimalRequestURL(urlToProcess)
|
93
|
+
if @@debugMe
|
94
|
+
$stderr.puts "Extracting information from URL..."
|
95
|
+
end
|
96
|
+
|
97
|
+
# get category of the Web page
|
98
|
+
categoryJSON = getAlchemy("#{@@alchemyURL}/URLGetCategory",
|
99
|
+
:query => {
|
100
|
+
:outputMode => 'json',
|
101
|
+
:apikey => @alchemyApiKey,
|
102
|
+
:url => urlToProcess
|
103
|
+
})
|
104
|
+
|
105
|
+
# get entities in the Web page
|
106
|
+
entitiesJSON = getAlchemy("#{@@alchemyURL}/URLGetRankedNamedEntities",
|
107
|
+
:query => {
|
108
|
+
:outputMode => 'json',
|
109
|
+
:apikey => @alchemyApiKey,
|
110
|
+
:url => urlToProcess
|
111
|
+
})
|
112
|
+
|
113
|
+
# get keywords from the Web page
|
114
|
+
keywordsJSON = getAlchemy("#{@@alchemyURL}/URLGetRankedKeywords",
|
115
|
+
:query => {
|
116
|
+
:outputMode => 'json',
|
117
|
+
:apikey => @alchemyApiKey,
|
118
|
+
:url => urlToProcess
|
119
|
+
})
|
120
|
+
|
121
|
+
buildPrimalRequest(categoryJSON, entitiesJSON, keywordsJSON)
|
122
|
+
end
|
123
|
+
|
124
|
+
#
|
125
|
+
# Processes the given Text at Alchemy and then translates the results to a
|
126
|
+
# valid Primal URL
|
127
|
+
#
|
128
|
+
def getPrimalRequestTEXT(textToProcess)
|
129
|
+
if @@debugMe
|
130
|
+
$stderr.puts "Extracting information from text..."
|
131
|
+
end
|
132
|
+
|
133
|
+
# get category
|
134
|
+
categoryJSON = postAlchemy("#{@@alchemyText}/TextGetCategory",
|
135
|
+
:query => {
|
136
|
+
:outputMode => 'json',
|
137
|
+
:apikey => @alchemyApiKey,
|
138
|
+
:text => textToProcess
|
139
|
+
})
|
140
|
+
|
141
|
+
# get entities
|
142
|
+
entitiesJSON = postAlchemy("#{@@alchemyText}/TextGetRankedNamedEntities",
|
143
|
+
:query => {
|
144
|
+
:outputMode => 'json',
|
145
|
+
:apikey => @alchemyApiKey,
|
146
|
+
:text => textToProcess
|
147
|
+
})
|
148
|
+
|
149
|
+
# get keywords
|
150
|
+
keywordsJSON = postAlchemy("#{@@alchemyText}/TextGetRankedKeywords",
|
151
|
+
:query => {
|
152
|
+
:outputMode => 'json',
|
153
|
+
:apikey => @alchemyApiKey,
|
154
|
+
:text => textToProcess
|
155
|
+
})
|
156
|
+
|
157
|
+
buildPrimalRequest(categoryJSON, entitiesJSON, keywordsJSON)
|
158
|
+
end
|
159
|
+
|
160
|
+
#
|
161
|
+
# Uses the deconstructed Alchemy information to create a valid Primal URL
|
162
|
+
#
|
163
|
+
def buildPrimalRequest(categoryJSON, entitiesJSON, keywordsJSON)
|
164
|
+
# Check if any of the extractions failed
|
165
|
+
if !categoryJSON or !entitiesJSON or !keywordsJSON
|
166
|
+
$stderr.puts "Cannot build Primal request. Alchemy failed to extract information."
|
167
|
+
return nil
|
168
|
+
end
|
169
|
+
|
170
|
+
if @@debugMe
|
171
|
+
$stderr.puts "Building Primal request..."
|
172
|
+
end
|
173
|
+
|
174
|
+
### Get information required for building a Primal request
|
175
|
+
# Get the category from the extracted data
|
176
|
+
category = rewriteCategory(categoryJSON['category'])
|
177
|
+
|
178
|
+
if @@debugMe
|
179
|
+
$stderr.puts "Category = #{category}"
|
180
|
+
end
|
181
|
+
|
182
|
+
### Select top entities from all extracted entities
|
183
|
+
entitiesList = entitiesJSON['entities'].collect { |entity|
|
184
|
+
entity['text'].downcase
|
185
|
+
}[0, @@entitiesLimit]
|
186
|
+
|
187
|
+
if @@debugMe
|
188
|
+
prettified = entitiesJSON['entities'].collect { |entity|
|
189
|
+
entity['text']
|
190
|
+
}.join(', ')
|
191
|
+
$stderr.puts "Entities = #{prettified}"
|
192
|
+
end
|
193
|
+
|
194
|
+
### Select top keywords from all extracted keywords
|
195
|
+
# Remove keywords that intersect with entities of the types in @@categoryIgnores
|
196
|
+
allEntities = entitiesJSON['entities'].select { |entity|
|
197
|
+
@@categoryIgnores.has_key? entity['type'].downcase
|
198
|
+
}.collect { |entity|
|
199
|
+
entity['text'].downcase
|
200
|
+
}
|
201
|
+
|
202
|
+
if @@debugMe
|
203
|
+
prettified = keywordsJSON['keywords'].collect { |keyword|
|
204
|
+
keyword['text']
|
205
|
+
}.join(', ')
|
206
|
+
$stderr.puts "Keywords = #{prettified}"
|
207
|
+
end
|
208
|
+
|
209
|
+
keywordsList = keywordsJSON['keywords'].select { |keyword|
|
210
|
+
normalizedKw = keyword['text'].downcase
|
211
|
+
intersectsWithEntity = !(allEntities.select { |entity|
|
212
|
+
entity.include? normalizedKw or normalizedKw.include? entity
|
213
|
+
}.empty?)
|
214
|
+
# Ignore keywords > 4 words or those that intersect with entities
|
215
|
+
normalizedKw.split.size < 5 && !intersectsWithEntity
|
216
|
+
}.collect { |keyword|
|
217
|
+
keyword['text'].downcase
|
218
|
+
}
|
219
|
+
|
220
|
+
# Remove repeated keywords
|
221
|
+
keywordsList = getNonRepeatedKeywords(keywordsList)
|
222
|
+
|
223
|
+
### Build Primal topic URI
|
224
|
+
primalRequest = ""
|
225
|
+
unless category.nil? then primalRequest = "/" + category end
|
226
|
+
if entitiesList.size > 0 then primalRequest = primalRequest + "/" + entitiesList.join("/") end
|
227
|
+
if keywordsList.size > 0 then primalRequest = primalRequest + "/" + keywordsList.join(";") end
|
228
|
+
if @@debugMe
|
229
|
+
$stderr.puts "Primal request = #{primalRequest}"
|
230
|
+
end
|
231
|
+
URI::encode(primalRequest)
|
232
|
+
end #function
|
233
|
+
|
234
|
+
#
|
235
|
+
# Returns the top @@keywordsLimit keywords, ignoring those contained
|
236
|
+
# within other keywords
|
237
|
+
#
|
238
|
+
def getNonRepeatedKeywords(keywordsList)
|
239
|
+
# If there is less than @@keywordsLimit, or if the first @@keywordsLimit keywords
|
240
|
+
# are unique, return the first @@keywordsLimit keywords
|
241
|
+
repeatedKeywords = getRepeatedKeywords(keywordsList[0, @@keywordsLimit])
|
242
|
+
if keywordsList.length <= @@keywordsLimit or repeatedKeywords.empty?
|
243
|
+
keywordsList[0, @@keywordsLimit]
|
244
|
+
else
|
245
|
+
# Remove repeated elements from the first @@keywordsLimit keywords, and recursively
|
246
|
+
# call this function
|
247
|
+
getNonRepeatedKeywords(keywordsList - repeatedKeywords)
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
#
|
252
|
+
# Returns any repeated words in the list
|
253
|
+
#
|
254
|
+
def getRepeatedKeywords(keywordsList)
|
255
|
+
keywordsList.select { |keyword|
|
256
|
+
not(keywordsList.select { |other|
|
257
|
+
other != keyword and other.include? keyword
|
258
|
+
}.empty?)
|
259
|
+
}
|
260
|
+
end
|
261
|
+
|
262
|
+
#
|
263
|
+
# Modifies the extracted category string to become a clear topic in the Primal
|
264
|
+
# request.
|
265
|
+
#
|
266
|
+
# AlchemyAPI categorizes text into a limited set of category types.
|
267
|
+
#
|
268
|
+
# See http://www.alchemyapi.com/api/categ/categs.html for a complete list.
|
269
|
+
#
|
270
|
+
# Some of the cateogy type names have two strings concatenated by an
|
271
|
+
# underscore character. This function selects one of the two strings (or a
|
272
|
+
# totally new string) to be the topic in the Primal request.
|
273
|
+
#
|
274
|
+
def rewriteCategory(category)
|
275
|
+
case category
|
276
|
+
when "unknown" # AlchemyAPI failed to classify the text
|
277
|
+
category = nil
|
278
|
+
when "arts_entertainment"
|
279
|
+
category = "arts" # rewrite to 'arts'
|
280
|
+
when "computer_internet"
|
281
|
+
category = "technology" # rewrite to 'technology', a clearer topic for this category
|
282
|
+
when "culture_politics"
|
283
|
+
category = "politics" # rewrite to 'politics'
|
284
|
+
when "law_crime"
|
285
|
+
category = "law" # rewrite to 'law'
|
286
|
+
when "science_technology"
|
287
|
+
category = "science" # rewrite to 'science'
|
288
|
+
else
|
289
|
+
# The previous conditions should cover all the categories extracted by
|
290
|
+
# Alchemy. In case of a new category that contains an underscore, replace
|
291
|
+
# it and keep the two words as the topic.
|
292
|
+
category = category.sub('_', ' ')
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
#
|
297
|
+
# Perform a POST request to Alchemy service URL and return the response as a
|
298
|
+
# JSON object
|
299
|
+
#
|
300
|
+
# Returns nil on error
|
301
|
+
#
|
302
|
+
def postAlchemy(serviceURL, parameters)
|
303
|
+
response = self.class.post(serviceURL, parameters)
|
304
|
+
returnAlchemyResponseJSON(response)
|
305
|
+
end
|
306
|
+
|
307
|
+
#
|
308
|
+
# Perform a GET request to Alchemy service URL and return the
|
309
|
+
# response as a JSON object
|
310
|
+
#
|
311
|
+
# Returns nil on error
|
312
|
+
#
|
313
|
+
def getAlchemy(serviceURL, parameters)
|
314
|
+
response = self.class.get(serviceURL, parameters)
|
315
|
+
returnAlchemyResponseJSON(response)
|
316
|
+
end
|
317
|
+
|
318
|
+
#
|
319
|
+
# Return the body of the response in a JSON object
|
320
|
+
# or nil on error
|
321
|
+
#
|
322
|
+
def returnAlchemyResponseJSON(response)
|
323
|
+
code = response.code
|
324
|
+
body = response.body
|
325
|
+
bodyJSON = JSON.parse(body)
|
326
|
+
|
327
|
+
# A statusInfo field contains the details of the error
|
328
|
+
if bodyJSON['status'] != "OK"
|
329
|
+
puts bodyJSON['statusInfo']
|
330
|
+
nil
|
331
|
+
else
|
332
|
+
bodyJSON
|
333
|
+
end
|
334
|
+
end
|
335
|
+
end
|
336
|
+
Ï
|
337
|
+
|
338
|
+
end
|
339
|
+
|
340
|
+
|
@@ -0,0 +1,204 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'httparty'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Primal
|
6
|
+
#
|
7
|
+
# The PrimalAccess class abstracts the access to Primal such
|
8
|
+
# that you can call simple methods on it to get what you need.
|
9
|
+
#
|
10
|
+
class PrimalAccess
|
11
|
+
include HTTParty
|
12
|
+
base_uri 'https://data.primal.com'
|
13
|
+
# Uncomment this next line to see what HTTParty is doing
|
14
|
+
# debug_output $stderr
|
15
|
+
# Set this to false in order to turn off debugging of this class
|
16
|
+
@@debugMe = true
|
17
|
+
#
|
18
|
+
# Constructor for the PrimalAccess class
|
19
|
+
#
|
20
|
+
# Pass in the username and password of the user you're going
|
21
|
+
# to access in order to construct and object that will work
|
22
|
+
# with that user
|
23
|
+
#
|
24
|
+
def initialize(appId, appKey, username, password)
|
25
|
+
@headers = {
|
26
|
+
:headers => {
|
27
|
+
'Primal-App-ID' => appId,
|
28
|
+
'Primal-App-Key' => appKey
|
29
|
+
},
|
30
|
+
:basic_auth => {
|
31
|
+
:username => username,
|
32
|
+
:password => password
|
33
|
+
}
|
34
|
+
}
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Sometimes we're going to get topics that are complex (i.e. they contain a
|
39
|
+
# scheme and host) and we want to simplify those. Because we're not making
|
40
|
+
# calls with bare URLs but have told HTTParty what the base_uri is, we need to
|
41
|
+
# pull that base uri off of the topic, should it be there.
|
42
|
+
#
|
43
|
+
def extractJustTopic(topic)
|
44
|
+
topic.sub(%r{https://.*?/}, '/')
|
45
|
+
end
|
46
|
+
|
47
|
+
#
|
48
|
+
# POSTs a new topic to Primal in order to seed that topic.
|
49
|
+
#
|
50
|
+
# The 'topic' parameter will be used to construct a POST URL
|
51
|
+
# that looks like "/topic"
|
52
|
+
#
|
53
|
+
# Returns two values: the response code and the body.
|
54
|
+
# Anything but a response code of 201 is to be considered
|
55
|
+
# an error.
|
56
|
+
#
|
57
|
+
def postNewTopic(topic, opts = {})
|
58
|
+
topic = extractJustTopic(topic)
|
59
|
+
count = 0
|
60
|
+
code = 400
|
61
|
+
body = ''
|
62
|
+
options = @headers.merge(opts)
|
63
|
+
while (count < 5)
|
64
|
+
if @@debugMe
|
65
|
+
$stderr.puts "POSTing to #{topic}"
|
66
|
+
end
|
67
|
+
response = self.class.post("#{topic}", options)
|
68
|
+
code = response.code
|
69
|
+
body = response.body
|
70
|
+
#
|
71
|
+
# 400 - bad request
|
72
|
+
# 401 - application not authorized to access the user's account
|
73
|
+
# 403 - application not authorized to use Primal
|
74
|
+
#
|
75
|
+
if code >= 400 && code <= 403
|
76
|
+
if @@debugMe
|
77
|
+
$stderr.puts "POST received a #{code}"
|
78
|
+
end
|
79
|
+
break
|
80
|
+
#
|
81
|
+
# 429 - application has reached its request limit for the moment
|
82
|
+
#
|
83
|
+
elsif code == 429
|
84
|
+
# Sleep for 10 seconds
|
85
|
+
if @@debugMe
|
86
|
+
$stderr.puts "Got a 429. Waiting (#{count})."
|
87
|
+
end
|
88
|
+
sleep 10
|
89
|
+
count += 1
|
90
|
+
#
|
91
|
+
# 201 - success
|
92
|
+
#
|
93
|
+
elsif code == 201
|
94
|
+
if @@debugMe
|
95
|
+
$stderr.puts "POST successful"
|
96
|
+
end
|
97
|
+
break
|
98
|
+
else
|
99
|
+
abort "Received unexpected response code (#{code}) for POST #{uri}"
|
100
|
+
end
|
101
|
+
end
|
102
|
+
return code, body
|
103
|
+
end
|
104
|
+
|
105
|
+
#
|
106
|
+
# Uses the pre-existing topic to filter the default source
|
107
|
+
# of content through the interest network defined by the topic.
|
108
|
+
#
|
109
|
+
# The given parameter will be used to construct a GET URL
|
110
|
+
# that looks like "/topic"
|
111
|
+
#
|
112
|
+
# You can pass a dictionary of optional arguments that will
|
113
|
+
# be merged in to the query parameters, if you wish.
|
114
|
+
# e.g.
|
115
|
+
# { :"primal:contentScore:min" => 0.7 }
|
116
|
+
# { :"primal:contentCount:max" => 5 }
|
117
|
+
# { :contentSource => MyDataSource } ... or ...
|
118
|
+
# { :contentSource => PrimalSource }
|
119
|
+
#
|
120
|
+
# Returns two values: the response code, and the body.
|
121
|
+
# If successful (i.e. a response code of 200) then the body
|
122
|
+
# will be the JSON payload of the filtered content.
|
123
|
+
#
|
124
|
+
def filterContent(topic, opts = {})
|
125
|
+
topic = extractJustTopic(topic)
|
126
|
+
count = 0
|
127
|
+
code = 400
|
128
|
+
body = ''
|
129
|
+
options = @headers.merge({ :query => {
|
130
|
+
:timeOut => 'max'
|
131
|
+
}.merge(opts)
|
132
|
+
})
|
133
|
+
while (count < 10)
|
134
|
+
if @@debugMe
|
135
|
+
$stderr.puts "GETting #{topic}"
|
136
|
+
end
|
137
|
+
response = self.class.get("#{topic}", options)
|
138
|
+
code = response.code
|
139
|
+
body = response.body
|
140
|
+
#
|
141
|
+
# 400 - bad request
|
142
|
+
# 401 - application not authorized to access the user's account
|
143
|
+
# 403 - application not authorized to use Primal
|
144
|
+
# 404 - object not found
|
145
|
+
#
|
146
|
+
if code >= 400 && code <= 404
|
147
|
+
if @@debugMe
|
148
|
+
$stderr.puts "GET received a #{code}"
|
149
|
+
end
|
150
|
+
break
|
151
|
+
#
|
152
|
+
# 429 - application has reached its request limit for the moment
|
153
|
+
#
|
154
|
+
elsif code == 429
|
155
|
+
if @@debugMe
|
156
|
+
$stderr.puts "Got a 429. Waiting (#{count})."
|
157
|
+
end
|
158
|
+
# Sleep for 10 seconds
|
159
|
+
sleep 10
|
160
|
+
# We don't allow as many retries when we might be throttled
|
161
|
+
count += 2
|
162
|
+
#
|
163
|
+
# 200 - success
|
164
|
+
#
|
165
|
+
elsif code == 200
|
166
|
+
if @@debugMe
|
167
|
+
$stderr.puts "Results are complete"
|
168
|
+
end
|
169
|
+
break
|
170
|
+
#
|
171
|
+
# We don't know what happened but it can't be good
|
172
|
+
#
|
173
|
+
else
|
174
|
+
abort "Received unexpected response code (#{code}) for GET #{topic}"
|
175
|
+
end
|
176
|
+
end
|
177
|
+
return code, body
|
178
|
+
end
|
179
|
+
|
180
|
+
#
|
181
|
+
# This is a convenience method that will POST the topic to
|
182
|
+
# Primal and then filter the default source of content through the
|
183
|
+
# resulting interest network.
|
184
|
+
#
|
185
|
+
# The response from this method is a bit less clear than using
|
186
|
+
# a POST and filter explicitly, since you may not know which
|
187
|
+
# one of the two operations has failed (assuming a failure).
|
188
|
+
#
|
189
|
+
# Returns two values: the response code and the body. The only
|
190
|
+
# successful response code from this method is 200. If
|
191
|
+
# successful then the body contains the JSON payload of the
|
192
|
+
# filtered content.
|
193
|
+
#
|
194
|
+
def postThenFilter(topic, opts = {})
|
195
|
+
code, body = postNewTopic(topic)
|
196
|
+
if code == 201
|
197
|
+
code, body = filterContent(topic, opts)
|
198
|
+
end
|
199
|
+
return code, body
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
end
|
204
|
+
|
data/primal.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'primal/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "primal"
|
8
|
+
spec.version = Primal::VERSION
|
9
|
+
spec.authors = ["w2davids@gmail.com"]
|
10
|
+
spec.email = ["w2davids@gmail.com"]
|
11
|
+
spec.description = %q{Ruby gem for accessing Primal Data web services}
|
12
|
+
spec.summary = %q{Ruby client to access Primal Data web services}
|
13
|
+
spec.homepage = ""
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.rubyforge_project = "primal"
|
17
|
+
|
18
|
+
spec.files = `git ls-files`.split($/)
|
19
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
20
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
21
|
+
spec.require_paths = ["lib"]
|
22
|
+
|
23
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
24
|
+
spec.add_development_dependency "rake"
|
25
|
+
spec.add_development_dependency "json"
|
26
|
+
spec.add_development_dependency "httparty"
|
27
|
+
|
28
|
+
end
|
metadata
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: primal
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- w2davids@gmail.com
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-05-20 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.3'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.3'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: json
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: httparty
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: Ruby gem for accessing Primal Data web services
|
70
|
+
email:
|
71
|
+
- w2davids@gmail.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- .gitignore
|
77
|
+
- .project
|
78
|
+
- Gemfile
|
79
|
+
- LICENSE.txt
|
80
|
+
- README.md
|
81
|
+
- Rakefile
|
82
|
+
- lib/primal.rb
|
83
|
+
- lib/primal/AlchemyAPIWrapper.rb
|
84
|
+
- lib/primal/PrimalAccess.rb
|
85
|
+
- lib/primal/version.rb
|
86
|
+
- primal.gemspec
|
87
|
+
homepage: ''
|
88
|
+
licenses:
|
89
|
+
- MIT
|
90
|
+
metadata: {}
|
91
|
+
post_install_message:
|
92
|
+
rdoc_options: []
|
93
|
+
require_paths:
|
94
|
+
- lib
|
95
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - '>='
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: '0'
|
100
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - '>='
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
requirements: []
|
106
|
+
rubyforge_project: primal
|
107
|
+
rubygems_version: 2.0.3
|
108
|
+
signing_key:
|
109
|
+
specification_version: 4
|
110
|
+
summary: Ruby client to access Primal Data web services
|
111
|
+
test_files: []
|