keybase-proofs 2.3.24 → 2.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/lib/scrapers/base.js +126 -8
- package/lib/scrapers/generic_social.js +2 -2
- package/lib/scrapers/reddit.js +168 -13
- package/package.json +2 -2
- package/src/scrapers/base.iced +75 -0
- package/src/scrapers/generic_social.iced +1 -1
- package/src/scrapers/reddit.iced +74 -6
package/CHANGELOG.md
CHANGED
package/lib/scrapers/base.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
// Generated by IcedCoffeeScript 108.0.
|
|
1
|
+
// Generated by IcedCoffeeScript 108.0.11
|
|
2
2
|
(function() {
|
|
3
|
-
var BaseScraper, b64find, callbackify, constants, decode_sig, iced, parse_int_or_undefined, pkg, sncmp, space_normalize, urlmod, user_agent, v_codes, __iced_k, __iced_k_noop;
|
|
3
|
+
var BaseBearerToken, BaseScraper, Lock, b64find, callbackify, constants, decode_sig, iced, parse_int_or_undefined, pkg, sncmp, space_normalize, urlmod, user_agent, v_codes, __iced_k, __iced_k_noop;
|
|
4
4
|
|
|
5
5
|
iced = require('iced-runtime');
|
|
6
6
|
__iced_k = __iced_k_noop = function() {};
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
|
|
22
22
|
callbackify = require('util').callbackify;
|
|
23
23
|
|
|
24
|
+
Lock = require('../util').Lock;
|
|
25
|
+
|
|
24
26
|
exports.user_agent = user_agent = constants.user_agent + pkg.version;
|
|
25
27
|
|
|
26
28
|
BaseScraper = (function() {
|
|
@@ -94,7 +96,7 @@
|
|
|
94
96
|
(function(__iced_k) {
|
|
95
97
|
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
96
98
|
parent: ___iced_passed_deferral,
|
|
97
|
-
filename: "/
|
|
99
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
|
|
98
100
|
funcname: "BaseScraper.validate"
|
|
99
101
|
});
|
|
100
102
|
_this.check_status(args, __iced_deferrals.defer({
|
|
@@ -104,7 +106,7 @@
|
|
|
104
106
|
return rc = arguments[1];
|
|
105
107
|
};
|
|
106
108
|
})(),
|
|
107
|
-
lineno:
|
|
109
|
+
lineno: 54
|
|
108
110
|
}));
|
|
109
111
|
__iced_deferrals._fulfill();
|
|
110
112
|
})(__iced_k);
|
|
@@ -160,7 +162,7 @@
|
|
|
160
162
|
return (function(__iced_k) {
|
|
161
163
|
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
162
164
|
parent: ___iced_passed_deferral,
|
|
163
|
-
filename: "/
|
|
165
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
|
|
164
166
|
funcname: "BaseScraper._get_url_body"
|
|
165
167
|
});
|
|
166
168
|
_this.libs.fetch(opts.url, opts, __iced_deferrals.defer({
|
|
@@ -170,7 +172,7 @@
|
|
|
170
172
|
return response = arguments[1];
|
|
171
173
|
};
|
|
172
174
|
})(),
|
|
173
|
-
lineno:
|
|
175
|
+
lineno: 88
|
|
174
176
|
}));
|
|
175
177
|
__iced_deferrals._fulfill();
|
|
176
178
|
});
|
|
@@ -201,7 +203,7 @@
|
|
|
201
203
|
(function(__iced_k) {
|
|
202
204
|
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
203
205
|
parent: ___iced_passed_deferral,
|
|
204
|
-
filename: "/
|
|
206
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
|
|
205
207
|
funcname: "BaseScraper._get_url_body"
|
|
206
208
|
});
|
|
207
209
|
f.call(response, __iced_deferrals.defer({
|
|
@@ -211,7 +213,7 @@
|
|
|
211
213
|
return body = arguments[1];
|
|
212
214
|
};
|
|
213
215
|
})(),
|
|
214
|
-
lineno:
|
|
216
|
+
lineno: 110
|
|
215
217
|
}));
|
|
216
218
|
__iced_deferrals._fulfill();
|
|
217
219
|
})(function() {
|
|
@@ -244,6 +246,122 @@
|
|
|
244
246
|
|
|
245
247
|
exports.BaseScraper = BaseScraper;
|
|
246
248
|
|
|
249
|
+
exports.BaseBearerToken = BaseBearerToken = (function() {
|
|
250
|
+
function BaseBearerToken(_arg) {
|
|
251
|
+
this.name = _arg.name, this.base = _arg.base, this.access_token_url = _arg.access_token_url, this.scope = _arg.scope, this.user_agent = _arg.user_agent;
|
|
252
|
+
if (this.access_token_url == null) {
|
|
253
|
+
throw new Error("@access_token_url is required");
|
|
254
|
+
}
|
|
255
|
+
if ((this.scope != null) && !Array.isArray(this.scope)) {
|
|
256
|
+
throw new Error("@scope has to be an array if present");
|
|
257
|
+
}
|
|
258
|
+
this._tok = null;
|
|
259
|
+
this._created = 0;
|
|
260
|
+
this._lock = new Lock();
|
|
261
|
+
this.auth = this.base.auth;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
BaseBearerToken.prototype.get = function(cb) {
|
|
265
|
+
var body, cred, e, err, now, opts, rc, req, res, tok, ___iced_passed_deferral, __iced_deferrals, __iced_k;
|
|
266
|
+
__iced_k = __iced_k_noop;
|
|
267
|
+
___iced_passed_deferral = iced.findDeferral(arguments);
|
|
268
|
+
(function(_this) {
|
|
269
|
+
return (function(__iced_k) {
|
|
270
|
+
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
271
|
+
parent: ___iced_passed_deferral,
|
|
272
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
|
|
273
|
+
funcname: "BaseBearerToken.get"
|
|
274
|
+
});
|
|
275
|
+
_this._lock.acquire(__iced_deferrals.defer({
|
|
276
|
+
lineno: 148
|
|
277
|
+
}));
|
|
278
|
+
__iced_deferrals._fulfill();
|
|
279
|
+
});
|
|
280
|
+
})(this)((function(_this) {
|
|
281
|
+
return function() {
|
|
282
|
+
err = null;
|
|
283
|
+
now = Math.floor(Date.now() / 1000);
|
|
284
|
+
(function(__iced_k) {
|
|
285
|
+
if (((res = _this._tok) == null) || (now - _this._created > _this.auth.lifespan)) {
|
|
286
|
+
_this.base.log("+ Request for bearer token");
|
|
287
|
+
cred = (Buffer.from([_this.auth.key, _this.auth.secret].join(":"))).toString('base64');
|
|
288
|
+
req = 'grant_type=client_credentials';
|
|
289
|
+
if (_this.scope) {
|
|
290
|
+
req += "&scope=" + (encodeURIComponent(_this.scope.join(' ')));
|
|
291
|
+
}
|
|
292
|
+
opts = {
|
|
293
|
+
url: _this.access_token_url,
|
|
294
|
+
headers: {
|
|
295
|
+
Authorization: "Basic " + cred,
|
|
296
|
+
'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
|
|
297
|
+
'Content-Length': req.length
|
|
298
|
+
},
|
|
299
|
+
method: "post",
|
|
300
|
+
body: req
|
|
301
|
+
};
|
|
302
|
+
if (_this.user_agent) {
|
|
303
|
+
opts.user_agent = _this.user_agent;
|
|
304
|
+
}
|
|
305
|
+
(function(__iced_k) {
|
|
306
|
+
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
307
|
+
parent: ___iced_passed_deferral,
|
|
308
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
|
|
309
|
+
funcname: "BaseBearerToken.get"
|
|
310
|
+
});
|
|
311
|
+
_this.base._get_url_body(opts, __iced_deferrals.defer({
|
|
312
|
+
assign_fn: (function() {
|
|
313
|
+
return function() {
|
|
314
|
+
err = arguments[0];
|
|
315
|
+
rc = arguments[1];
|
|
316
|
+
return body = arguments[2];
|
|
317
|
+
};
|
|
318
|
+
})(),
|
|
319
|
+
lineno: 173
|
|
320
|
+
}));
|
|
321
|
+
__iced_deferrals._fulfill();
|
|
322
|
+
})(function() {
|
|
323
|
+
if (err != null) {
|
|
324
|
+
_this.base.logl('error', "In getting bearer_token: " + err.message);
|
|
325
|
+
} else if (rc !== v_codes.OK) {
|
|
326
|
+
_this.base.logl('error', "error in getting bearer token: " + rc);
|
|
327
|
+
err = new Error("error: " + rc);
|
|
328
|
+
} else {
|
|
329
|
+
try {
|
|
330
|
+
body = JSON.parse(body);
|
|
331
|
+
} catch (_error) {
|
|
332
|
+
e = _error;
|
|
333
|
+
_this.base.logl('warn', "Could not parse JSON reply: " + e);
|
|
334
|
+
err = e;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (err != null) {
|
|
338
|
+
|
|
339
|
+
} else if ((tok = body.access_token) == null) {
|
|
340
|
+
_this.base.logl('warn', "No access token found in reply");
|
|
341
|
+
err = new Error("" + _this.name + " error: no access token");
|
|
342
|
+
} else {
|
|
343
|
+
_this._tok = tok;
|
|
344
|
+
_this._created = Math.floor(Date.now() / 1000);
|
|
345
|
+
res = _this._tok;
|
|
346
|
+
}
|
|
347
|
+
_this.base.log("- Request for bearer token for " + _this.name + " -> err: " + err);
|
|
348
|
+
return __iced_k(!err ? _this.base.log("Bearer token for " + _this.name + " is: " + (res != null ? res.substr(0, 5) : void 0) + "...") : void 0);
|
|
349
|
+
});
|
|
350
|
+
} else {
|
|
351
|
+
return __iced_k();
|
|
352
|
+
}
|
|
353
|
+
})(function() {
|
|
354
|
+
_this._lock.release();
|
|
355
|
+
return cb(err, res);
|
|
356
|
+
});
|
|
357
|
+
};
|
|
358
|
+
})(this));
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
return BaseBearerToken;
|
|
362
|
+
|
|
363
|
+
})();
|
|
364
|
+
|
|
247
365
|
parse_int_or_undefined = function(value) {
|
|
248
366
|
var out;
|
|
249
367
|
if ((out = Number(value)) === parseInt(value) && !isNaN(out)) {
|
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
return [err, rc];
|
|
75
75
|
}
|
|
76
76
|
found = obj.find(function(x) {
|
|
77
|
-
return sncmp(x.kb_username, kb_username) && sncmp(x.sig_hash, sig_id);
|
|
77
|
+
return x && sncmp(x.kb_username, kb_username) && sncmp(x.sig_hash, sig_id);
|
|
78
78
|
});
|
|
79
79
|
if (found != null) {
|
|
80
80
|
rc = v_codes.OK;
|
|
@@ -100,7 +100,7 @@
|
|
|
100
100
|
return (function(__iced_k) {
|
|
101
101
|
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
102
102
|
parent: ___iced_passed_deferral,
|
|
103
|
-
filename: "/
|
|
103
|
+
filename: "/home/sy/go/src/github.com/keybase/proofs/src/scrapers/generic_social.iced",
|
|
104
104
|
funcname: "GenericSocialScraper.check_status"
|
|
105
105
|
});
|
|
106
106
|
_this._get_url_body({
|
package/lib/scrapers/reddit.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
// Generated by IcedCoffeeScript 108.0.11
|
|
2
2
|
(function() {
|
|
3
|
-
var BaseScraper, PREFIX, RedditScraper, SUBREDDIT, constants, iced, proof_text_check_to_med_id, sncmp, v_codes, __iced_k, __iced_k_noop, _ref,
|
|
3
|
+
var BaseBearerToken, BaseScraper, OAUTH_REDDIT_URL, PREFIX, REDDIT_USER_AGENT, RedditBearerToken, RedditScraper, SUBREDDIT, bearer_token, constants, iced, inspect, proof_text_check_to_med_id, sncmp, urlmod, user_agent, v_codes, __iced_k, __iced_k_noop, _bearer_token, _ref,
|
|
4
4
|
__hasProp = {}.hasOwnProperty,
|
|
5
5
|
__extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
|
|
6
6
|
|
|
7
7
|
iced = require('iced-runtime');
|
|
8
8
|
__iced_k = __iced_k_noop = function() {};
|
|
9
9
|
|
|
10
|
-
_ref = require('./base'), sncmp = _ref.sncmp, BaseScraper = _ref.BaseScraper;
|
|
10
|
+
_ref = require('./base'), sncmp = _ref.sncmp, user_agent = _ref.user_agent, BaseScraper = _ref.BaseScraper, BaseBearerToken = _ref.BaseBearerToken;
|
|
11
11
|
|
|
12
12
|
constants = require('../constants').constants;
|
|
13
13
|
|
|
@@ -15,14 +15,54 @@
|
|
|
15
15
|
|
|
16
16
|
proof_text_check_to_med_id = require('../base').proof_text_check_to_med_id;
|
|
17
17
|
|
|
18
|
+
urlmod = require('url');
|
|
19
|
+
|
|
20
|
+
inspect = require('util').inspect;
|
|
21
|
+
|
|
18
22
|
PREFIX = "https://www.reddit.com";
|
|
19
23
|
|
|
20
24
|
SUBREDDIT = PREFIX + "/r/keybaseproofs";
|
|
21
25
|
|
|
26
|
+
OAUTH_REDDIT_URL = "https://oauth.reddit.com";
|
|
27
|
+
|
|
28
|
+
REDDIT_USER_AGENT = user_agent + " (by /u/maxtaco)";
|
|
29
|
+
|
|
30
|
+
RedditBearerToken = (function(_super) {
|
|
31
|
+
__extends(RedditBearerToken, _super);
|
|
32
|
+
|
|
33
|
+
function RedditBearerToken(_arg) {
|
|
34
|
+
this.base = _arg.base;
|
|
35
|
+
RedditBearerToken.__super__.constructor.call(this, {
|
|
36
|
+
name: "Reddit",
|
|
37
|
+
base: this.base,
|
|
38
|
+
access_token_url: "" + OAUTH_REDDIT_URL + "/api/v1/access_token",
|
|
39
|
+
scope: ['history', 'read'],
|
|
40
|
+
user_agent: REDDIT_USER_AGENT
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return RedditBearerToken;
|
|
45
|
+
|
|
46
|
+
})(BaseBearerToken);
|
|
47
|
+
|
|
48
|
+
_bearer_token = null;
|
|
49
|
+
|
|
50
|
+
bearer_token = function(_arg) {
|
|
51
|
+
var base;
|
|
52
|
+
base = _arg.base;
|
|
53
|
+
if (!_bearer_token) {
|
|
54
|
+
_bearer_token = new RedditBearerToken({
|
|
55
|
+
base: base
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return _bearer_token;
|
|
59
|
+
};
|
|
60
|
+
|
|
22
61
|
exports.RedditScraper = RedditScraper = (function(_super) {
|
|
23
62
|
__extends(RedditScraper, _super);
|
|
24
63
|
|
|
25
64
|
function RedditScraper(opts) {
|
|
65
|
+
this.auth = opts.auth;
|
|
26
66
|
RedditScraper.__super__.constructor.call(this, opts);
|
|
27
67
|
}
|
|
28
68
|
|
|
@@ -60,11 +100,11 @@
|
|
|
60
100
|
return (function(__iced_k) {
|
|
61
101
|
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
62
102
|
parent: ___iced_passed_deferral,
|
|
63
|
-
filename: "/Users/
|
|
103
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
|
|
64
104
|
funcname: "RedditScraper.hunt2"
|
|
65
105
|
});
|
|
66
|
-
_this.
|
|
67
|
-
url: "" +
|
|
106
|
+
_this._get_body_api({
|
|
107
|
+
url: "" + OAUTH_REDDIT_URL + "/user/" + (encodeURIComponent(username)) + "/submitted.json?count=25&cachebust=" + (Math.random()),
|
|
68
108
|
json: true
|
|
69
109
|
}, __iced_deferrals.defer({
|
|
70
110
|
assign_fn: (function() {
|
|
@@ -74,7 +114,7 @@
|
|
|
74
114
|
return json = arguments[2];
|
|
75
115
|
};
|
|
76
116
|
})(),
|
|
77
|
-
lineno:
|
|
117
|
+
lineno: 69
|
|
78
118
|
}));
|
|
79
119
|
__iced_deferrals._fulfill();
|
|
80
120
|
});
|
|
@@ -169,21 +209,32 @@
|
|
|
169
209
|
};
|
|
170
210
|
|
|
171
211
|
RedditScraper.prototype.check_status = function(_arg, cb) {
|
|
172
|
-
var api_url, err, json, post, proof_text_check, rc, remote_id, username, ___iced_passed_deferral, __iced_deferrals, __iced_k;
|
|
212
|
+
var api_url, err, json, post, proof_text_check, rc, remote_id, u, username, ___iced_passed_deferral, __iced_deferrals, __iced_k, _ref1;
|
|
173
213
|
__iced_k = __iced_k_noop;
|
|
174
214
|
___iced_passed_deferral = iced.findDeferral(arguments);
|
|
175
215
|
username = _arg.username, api_url = _arg.api_url, proof_text_check = _arg.proof_text_check, remote_id = _arg.remote_id;
|
|
216
|
+
u = urlmod.parse(api_url);
|
|
217
|
+
if (!u) {
|
|
218
|
+
return cb(new Error("Failed to parse api_url"));
|
|
219
|
+
}
|
|
220
|
+
if ((_ref1 = u.host) !== 'www.reddit.com' && _ref1 !== 'oauth.reddit.com') {
|
|
221
|
+
return cb(new Error("Unexpected api_url host: " + host));
|
|
222
|
+
}
|
|
223
|
+
u = urlmod.format({
|
|
224
|
+
host: 'oauth.reddit.com',
|
|
225
|
+
protocol: 'https:',
|
|
226
|
+
pathname: u.pathname
|
|
227
|
+
});
|
|
176
228
|
(function(_this) {
|
|
177
229
|
return (function(__iced_k) {
|
|
178
230
|
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
179
231
|
parent: ___iced_passed_deferral,
|
|
180
|
-
filename: "/Users/
|
|
232
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
|
|
181
233
|
funcname: "RedditScraper.check_status"
|
|
182
234
|
});
|
|
183
|
-
_this.
|
|
184
|
-
url:
|
|
185
|
-
json: true
|
|
186
|
-
user_agent: "linux:com.github/keybase/proofs:v2.1.41 (by /u/maxtaco)"
|
|
235
|
+
_this._get_body_api({
|
|
236
|
+
url: u,
|
|
237
|
+
json: true
|
|
187
238
|
}, __iced_deferrals.defer({
|
|
188
239
|
assign_fn: (function() {
|
|
189
240
|
return function() {
|
|
@@ -192,7 +243,7 @@
|
|
|
192
243
|
return json = arguments[2];
|
|
193
244
|
};
|
|
194
245
|
})(),
|
|
195
|
-
lineno:
|
|
246
|
+
lineno: 158
|
|
196
247
|
}));
|
|
197
248
|
__iced_deferrals._fulfill();
|
|
198
249
|
});
|
|
@@ -208,6 +259,110 @@
|
|
|
208
259
|
})(this));
|
|
209
260
|
};
|
|
210
261
|
|
|
262
|
+
RedditScraper.prototype._get_bearer_token = function(cb) {
|
|
263
|
+
var bt, err, rc, tok, ___iced_passed_deferral, __iced_deferrals, __iced_k;
|
|
264
|
+
__iced_k = __iced_k_noop;
|
|
265
|
+
___iced_passed_deferral = iced.findDeferral(arguments);
|
|
266
|
+
bt = bearer_token({
|
|
267
|
+
base: this
|
|
268
|
+
});
|
|
269
|
+
(function(_this) {
|
|
270
|
+
return (function(__iced_k) {
|
|
271
|
+
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
272
|
+
parent: ___iced_passed_deferral,
|
|
273
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
|
|
274
|
+
funcname: "RedditScraper._get_bearer_token"
|
|
275
|
+
});
|
|
276
|
+
bt.get(__iced_deferrals.defer({
|
|
277
|
+
assign_fn: (function() {
|
|
278
|
+
return function() {
|
|
279
|
+
err = arguments[0];
|
|
280
|
+
return tok = arguments[1];
|
|
281
|
+
};
|
|
282
|
+
})(),
|
|
283
|
+
lineno: 169
|
|
284
|
+
}));
|
|
285
|
+
__iced_deferrals._fulfill();
|
|
286
|
+
});
|
|
287
|
+
})(this)((function(_this) {
|
|
288
|
+
return function() {
|
|
289
|
+
rc = typeof err !== "undefined" && err !== null ? v_codes.AUTH_FAILED : v_codes.OK;
|
|
290
|
+
return cb(err, rc, tok);
|
|
291
|
+
};
|
|
292
|
+
})(this));
|
|
293
|
+
};
|
|
294
|
+
|
|
295
|
+
RedditScraper.prototype._get_body_api = function(_arg, cb) {
|
|
296
|
+
var args, body, err, json, rc, tok, url, ___iced_passed_deferral, __iced_deferrals, __iced_k;
|
|
297
|
+
__iced_k = __iced_k_noop;
|
|
298
|
+
___iced_passed_deferral = iced.findDeferral(arguments);
|
|
299
|
+
url = _arg.url, json = _arg.json;
|
|
300
|
+
rc = body = err = null;
|
|
301
|
+
(function(_this) {
|
|
302
|
+
return (function(__iced_k) {
|
|
303
|
+
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
304
|
+
parent: ___iced_passed_deferral,
|
|
305
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
|
|
306
|
+
funcname: "RedditScraper._get_body_api"
|
|
307
|
+
});
|
|
308
|
+
_this._get_bearer_token(__iced_deferrals.defer({
|
|
309
|
+
assign_fn: (function() {
|
|
310
|
+
return function() {
|
|
311
|
+
err = arguments[0];
|
|
312
|
+
rc = arguments[1];
|
|
313
|
+
return tok = arguments[2];
|
|
314
|
+
};
|
|
315
|
+
})(),
|
|
316
|
+
lineno: 177
|
|
317
|
+
}));
|
|
318
|
+
__iced_deferrals._fulfill();
|
|
319
|
+
});
|
|
320
|
+
})(this)((function(_this) {
|
|
321
|
+
return function() {
|
|
322
|
+
(function(__iced_k) {
|
|
323
|
+
if (err == null) {
|
|
324
|
+
_this.log("| HTTP API request for URL '" + url + "'");
|
|
325
|
+
if (json == null) {
|
|
326
|
+
json = true;
|
|
327
|
+
}
|
|
328
|
+
args = {
|
|
329
|
+
url: url,
|
|
330
|
+
json: json,
|
|
331
|
+
headers: {
|
|
332
|
+
Authorization: "Bearer " + tok
|
|
333
|
+
},
|
|
334
|
+
method: "get",
|
|
335
|
+
json: true,
|
|
336
|
+
user_agent: REDDIT_USER_AGENT
|
|
337
|
+
};
|
|
338
|
+
(function(__iced_k) {
|
|
339
|
+
__iced_deferrals = new iced.Deferrals(__iced_k, {
|
|
340
|
+
parent: ___iced_passed_deferral,
|
|
341
|
+
filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
|
|
342
|
+
funcname: "RedditScraper._get_body_api"
|
|
343
|
+
});
|
|
344
|
+
_this._get_url_body(args, __iced_deferrals.defer({
|
|
345
|
+
assign_fn: (function() {
|
|
346
|
+
return function() {
|
|
347
|
+
err = arguments[0];
|
|
348
|
+
rc = arguments[1];
|
|
349
|
+
return body = arguments[2];
|
|
350
|
+
};
|
|
351
|
+
})(),
|
|
352
|
+
lineno: 189
|
|
353
|
+
}));
|
|
354
|
+
__iced_deferrals._fulfill();
|
|
355
|
+
})(__iced_k);
|
|
356
|
+
} else {
|
|
357
|
+
return __iced_k();
|
|
358
|
+
}
|
|
359
|
+
})(function() {
|
|
360
|
+
return cb(err, rc, body);
|
|
361
|
+
});
|
|
362
|
+
};
|
|
363
|
+
})(this));
|
|
364
|
+
};
|
|
365
|
+
|
|
211
366
|
return RedditScraper;
|
|
212
367
|
|
|
213
368
|
})(BaseScraper);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "keybase-proofs",
|
|
3
|
-
"version": "2.3.
|
|
3
|
+
"version": "2.3.26",
|
|
4
4
|
"description": "Publicly-verifiable proofs of identity",
|
|
5
5
|
"main": "lib/main.js",
|
|
6
6
|
"scripts": {
|
|
@@ -36,4 +36,4 @@
|
|
|
36
36
|
"iced-coffee-script": "^108.0.11",
|
|
37
37
|
"iced-test": "0.0.22"
|
|
38
38
|
}
|
|
39
|
-
}
|
|
39
|
+
}
|
package/src/scrapers/base.iced
CHANGED
|
@@ -6,6 +6,7 @@ pkg = require '../../package.json'
|
|
|
6
6
|
{b64find} = require '../b64extract'
|
|
7
7
|
urlmod = require 'url'
|
|
8
8
|
{callbackify} = require 'util'
|
|
9
|
+
{Lock} = require '../util'
|
|
9
10
|
|
|
10
11
|
#==============================================================
|
|
11
12
|
|
|
@@ -118,6 +119,8 @@ class BaseScraper
|
|
|
118
119
|
catch err
|
|
119
120
|
cb err, rc, null
|
|
120
121
|
else
|
|
122
|
+
# TODO: It's possible to get response body here, which might be useful
|
|
123
|
+
# for debugging and logging.
|
|
121
124
|
cb err, rc, null
|
|
122
125
|
|
|
123
126
|
#--------------------------------------------------------------
|
|
@@ -128,6 +131,78 @@ exports.BaseScraper = BaseScraper
|
|
|
128
131
|
|
|
129
132
|
#==============================================================
|
|
130
133
|
|
|
134
|
+
exports.BaseBearerToken = class BaseBearerToken
|
|
135
|
+
constructor : ({@name, @base, @access_token_url, @scope, @user_agent}) ->
|
|
136
|
+
unless @access_token_url?
|
|
137
|
+
throw new Error "@access_token_url is required"
|
|
138
|
+
if @scope? and not Array.isArray(@scope)
|
|
139
|
+
throw new Error "@scope has to be an array if present"
|
|
140
|
+
|
|
141
|
+
@_tok = null
|
|
142
|
+
@_created = 0
|
|
143
|
+
@_lock = new Lock()
|
|
144
|
+
@auth = @base.auth
|
|
145
|
+
|
|
146
|
+
#----------------
|
|
147
|
+
|
|
148
|
+
get : (cb) ->
|
|
149
|
+
await @_lock.acquire defer()
|
|
150
|
+
err = null
|
|
151
|
+
now = Math.floor(Date.now() / 1000)
|
|
152
|
+
|
|
153
|
+
if not (res = @_tok)? or (now - @_created > @auth.lifespan)
|
|
154
|
+
|
|
155
|
+
@base.log "+ Request for bearer token"
|
|
156
|
+
|
|
157
|
+
# Very crypto! Not sure why this is done, but it's done
|
|
158
|
+
cred = (Buffer.from [ @auth.key, @auth.secret ].join(":")).toString('base64')
|
|
159
|
+
|
|
160
|
+
req = 'grant_type=client_credentials'
|
|
161
|
+
if @scope
|
|
162
|
+
req += "&scope=#{encodeURIComponent(@scope.join(' '))}"
|
|
163
|
+
|
|
164
|
+
opts =
|
|
165
|
+
url : @access_token_url
|
|
166
|
+
headers :
|
|
167
|
+
Authorization : "Basic #{cred}"
|
|
168
|
+
'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
|
|
169
|
+
'Content-Length': req.length
|
|
170
|
+
method : "post"
|
|
171
|
+
body : req
|
|
172
|
+
if @user_agent
|
|
173
|
+
opts.user_agent = @user_agent
|
|
174
|
+
await @base._get_url_body opts, defer err, rc, body
|
|
175
|
+
|
|
176
|
+
if err?
|
|
177
|
+
@base.logl 'error', "In getting bearer_token: #{err.message}"
|
|
178
|
+
else if (rc isnt v_codes.OK)
|
|
179
|
+
@base.logl 'error', "error in getting bearer token: #{rc}"
|
|
180
|
+
err = new Error "error: #{rc}"
|
|
181
|
+
else
|
|
182
|
+
try
|
|
183
|
+
body = JSON.parse body
|
|
184
|
+
catch e
|
|
185
|
+
@base.logl 'warn', "Could not parse JSON reply: #{e}"
|
|
186
|
+
err = e
|
|
187
|
+
|
|
188
|
+
if err? then # noop
|
|
189
|
+
else if not (tok = body.access_token)?
|
|
190
|
+
@base.logl 'warn', "No access token found in reply"
|
|
191
|
+
err = new Error "#{@name} error: no access token"
|
|
192
|
+
else
|
|
193
|
+
@_tok = tok
|
|
194
|
+
@_created = Math.floor(Date.now() / 1000)
|
|
195
|
+
res = @_tok
|
|
196
|
+
|
|
197
|
+
@base.log "- Request for bearer token for #{@name} -> err: #{err}"
|
|
198
|
+
unless err
|
|
199
|
+
@base.log "Bearer token for #{@name} is: #{res?.substr(0,5)}..."
|
|
200
|
+
|
|
201
|
+
@_lock.release()
|
|
202
|
+
cb err, res
|
|
203
|
+
|
|
204
|
+
#==============================================================
|
|
205
|
+
|
|
131
206
|
parse_int_or_undefined = (value) ->
|
|
132
207
|
if (out = Number(value)) is parseInt(value) and !isNaN(out)
|
|
133
208
|
return out
|
|
@@ -52,7 +52,7 @@ exports.GenericSocialScraper = class GenericSocialScraper extends BaseScraper
|
|
|
52
52
|
unless obj? and Array.isArray obj
|
|
53
53
|
err = new Error "did not find proof list on #{check_path.join('.')} in json data"
|
|
54
54
|
return [err, rc]
|
|
55
|
-
found = obj.find (x) -> sncmp(x.kb_username, kb_username) and sncmp(x.sig_hash, sig_id)
|
|
55
|
+
found = obj.find (x) -> x and sncmp(x.kb_username, kb_username) and sncmp(x.sig_hash, sig_id)
|
|
56
56
|
if found?
|
|
57
57
|
rc = v_codes.OK
|
|
58
58
|
else
|
package/src/scrapers/reddit.iced
CHANGED
|
@@ -1,18 +1,44 @@
|
|
|
1
|
-
{sncmp,BaseScraper} = require './base'
|
|
1
|
+
{sncmp,user_agent,BaseScraper,BaseBearerToken} = require './base'
|
|
2
2
|
{constants} = require '../constants'
|
|
3
3
|
{v_codes} = constants
|
|
4
4
|
{proof_text_check_to_med_id} = require '../base'
|
|
5
|
+
urlmod = require 'url'
|
|
6
|
+
{inspect} = require 'util'
|
|
5
7
|
|
|
6
8
|
#================================================================================
|
|
7
9
|
|
|
8
10
|
PREFIX = "https://www.reddit.com"
|
|
9
11
|
SUBREDDIT = PREFIX + "/r/keybaseproofs"
|
|
12
|
+
OAUTH_REDDIT_URL = "https://oauth.reddit.com"
|
|
13
|
+
|
|
14
|
+
REDDIT_USER_AGENT = user_agent + " (by /u/maxtaco)"
|
|
15
|
+
|
|
16
|
+
#================================================================================
|
|
17
|
+
|
|
18
|
+
class RedditBearerToken extends BaseBearerToken
|
|
19
|
+
constructor : ({@base}) ->
|
|
20
|
+
super {
|
|
21
|
+
name: "Reddit"
|
|
22
|
+
@base
|
|
23
|
+
access_token_url : "#{OAUTH_REDDIT_URL}/api/v1/access_token"
|
|
24
|
+
scope : [ 'history', 'read' ]
|
|
25
|
+
user_agent : REDDIT_USER_AGENT
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
#================================================================================
|
|
29
|
+
|
|
30
|
+
_bearer_token = null
|
|
31
|
+
bearer_token = ({base}) ->
|
|
32
|
+
unless _bearer_token
|
|
33
|
+
_bearer_token = new RedditBearerToken { base }
|
|
34
|
+
return _bearer_token
|
|
10
35
|
|
|
11
36
|
#================================================================================
|
|
12
37
|
|
|
13
38
|
exports.RedditScraper = class RedditScraper extends BaseScraper
|
|
14
39
|
|
|
15
40
|
constructor: (opts) ->
|
|
41
|
+
@auth = opts.auth
|
|
16
42
|
super opts
|
|
17
43
|
|
|
18
44
|
# ---------------------------------------------------------------------------
|
|
@@ -38,8 +64,8 @@ exports.RedditScraper = class RedditScraper extends BaseScraper
|
|
|
38
64
|
if (err = @_check_args { username, name })?
|
|
39
65
|
return cb err, { rc: v_codes.BAD_ARGS }
|
|
40
66
|
|
|
41
|
-
await @
|
|
42
|
-
url: "#{
|
|
67
|
+
await @_get_body_api
|
|
68
|
+
url: "#{OAUTH_REDDIT_URL}/user/#{encodeURIComponent(username)}/submitted.json?count=25&cachebust=#{Math.random()}"
|
|
43
69
|
json: true
|
|
44
70
|
, defer err, rc, json
|
|
45
71
|
|
|
@@ -68,6 +94,9 @@ exports.RedditScraper = class RedditScraper extends BaseScraper
|
|
|
68
94
|
if not proof_post?
|
|
69
95
|
return cb null, rc: v_codes.NOT_FOUND
|
|
70
96
|
|
|
97
|
+
# NOTE: Still return "www.reddit.com" api_url instead of "oauth.reddit.com",
|
|
98
|
+
# for compatibility with existing rows in the keybase db. `check_status`
|
|
99
|
+
# method is rewriting www.reddit.com into oauth.reddit.com.
|
|
71
100
|
cb null,
|
|
72
101
|
rc : v_codes.OK
|
|
73
102
|
api_url : PREFIX + proof_post.permalink + ".json"
|
|
@@ -109,12 +138,24 @@ exports.RedditScraper = class RedditScraper extends BaseScraper
|
|
|
109
138
|
# ---------------------------------------------------------------------------
|
|
110
139
|
|
|
111
140
|
check_status: ({username, api_url, proof_text_check, remote_id}, cb) ->
|
|
141
|
+
u = urlmod.parse(api_url)
|
|
142
|
+
unless u
|
|
143
|
+
return cb new Error "Failed to parse api_url"
|
|
144
|
+
|
|
145
|
+
# Host should either be www.reddit.com or oauth.reddit.com.
|
|
146
|
+
if u.host not in ['www.reddit.com', 'oauth.reddit.com']
|
|
147
|
+
return cb new Error "Unexpected api_url host: #{host}"
|
|
148
|
+
|
|
149
|
+
u = urlmod.format {
|
|
150
|
+
host : 'oauth.reddit.com'
|
|
151
|
+
protocol : 'https:'
|
|
152
|
+
pathname : u.pathname
|
|
153
|
+
}
|
|
112
154
|
|
|
113
155
|
# calls back with a v_code or null if it was ok
|
|
114
|
-
await @
|
|
115
|
-
url :
|
|
156
|
+
await @_get_body_api {
|
|
157
|
+
url : u
|
|
116
158
|
json : true
|
|
117
|
-
user_agent : "linux:com.github/keybase/proofs:v2.1.41 (by /u/maxtaco)"
|
|
118
159
|
}, defer err, rc, json
|
|
119
160
|
|
|
120
161
|
rc = if rc isnt v_codes.OK then rc
|
|
@@ -122,5 +163,32 @@ exports.RedditScraper = class RedditScraper extends BaseScraper
|
|
|
122
163
|
else @check_post { post, username, proof_text_check }
|
|
123
164
|
cb err, rc
|
|
124
165
|
|
|
166
|
+
# ---------------------------------------------------------------------------
|
|
167
|
+
|
|
168
|
+
_get_bearer_token : (cb) ->
|
|
169
|
+
bt = bearer_token { base : @ }
|
|
170
|
+
await bt.get defer err, tok
|
|
171
|
+
rc = if err? then v_codes.AUTH_FAILED else v_codes.OK
|
|
172
|
+
cb err, rc, tok
|
|
173
|
+
|
|
174
|
+
# ---------------------------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
_get_body_api : ({url, json}, cb) ->
|
|
177
|
+
rc = body = err = null
|
|
178
|
+
await @_get_bearer_token defer err, rc, tok
|
|
179
|
+
unless err?
|
|
180
|
+
@log "| HTTP API request for URL '#{url}'"
|
|
181
|
+
json ?= true
|
|
182
|
+
args =
|
|
183
|
+
url : url
|
|
184
|
+
json : json
|
|
185
|
+
headers :
|
|
186
|
+
Authorization : "Bearer #{tok}"
|
|
187
|
+
method : "get"
|
|
188
|
+
json : true
|
|
189
|
+
user_agent : REDDIT_USER_AGENT
|
|
190
|
+
await @_get_url_body args, defer err, rc, body
|
|
191
|
+
cb err, rc, body
|
|
192
|
+
|
|
125
193
|
#================================================================================
|
|
126
194
|
|