keybase-proofs 2.3.25 → 2.3.28

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 2.3.28 (2024-07-19)
+
+- Use version 2 API for X (formerly known as Twitter) hunting.
+
+## 2.3.27 (2024-07-18)
+
+- Fix error handling when an invalid unicode domain is passed to DnsScraper.
+
+## 2.3.26 (2024-06-17)
+
+- Use oauth API for reddit scraping.
+
 ## 2.3.24 (2020-12-23)
 
 - Use oembed API for twitter scraping.

package/lib/constants.js

@@ -1,4 +1,4 @@
-// Generated by IcedCoffeeScript 108.0.12
+// Generated by IcedCoffeeScript 108.0.11
 (function() {
   var constants, d, k, v, _ref;
 
@@ -172,6 +172,7 @@
       HTTP_400: 240,
       HTTP_OTHER: 260,
       EMPTY_JSON: 270,
+      INVALID_JSON: 271,
       DELETED: 301,
       SERVICE_DEAD: 302,
       BAD_SIGNATURE: 303

package/lib/scrapers/base.js

@@ -1,6 +1,6 @@
-// Generated by IcedCoffeeScript 108.0.12
+// Generated by IcedCoffeeScript 108.0.11
 (function() {
-  var BaseScraper, b64find, callbackify, constants, decode_sig, iced, parse_int_or_undefined, pkg, sncmp, space_normalize, urlmod, user_agent, v_codes, __iced_k, __iced_k_noop;
+  var BaseBearerToken, BaseScraper, Lock, b64find, callbackify, constants, decode_sig, iced, parse_int_or_undefined, pkg, sncmp, space_normalize, urlmod, user_agent, v_codes, __iced_k, __iced_k_noop;
 
   iced = require('iced-runtime');
   __iced_k = __iced_k_noop = function() {};
@@ -21,6 +21,8 @@
 
   callbackify = require('util').callbackify;
 
+  Lock = require('../util').Lock;
+
   exports.user_agent = user_agent = constants.user_agent + pkg.version;
 
   BaseScraper = (function() {
@@ -94,7 +96,7 @@
             (function(__iced_k) {
               __iced_deferrals = new iced.Deferrals(__iced_k, {
                 parent: ___iced_passed_deferral,
-                filename: "/home/zapu/Projects/KB/proofs/src/scrapers/base.iced",
+                filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
                 funcname: "BaseScraper.validate"
               });
               _this.check_status(args, __iced_deferrals.defer({
@@ -104,7 +106,7 @@
                     return rc = arguments[1];
                   };
                 })(),
-                lineno: 53
+                lineno: 54
               }));
               __iced_deferrals._fulfill();
             })(__iced_k);
@@ -141,6 +143,15 @@
       return b64find(raw, ptc_buf);
     };
 
+    BaseScraper.prototype._get_rate_limit_header = function(headers, name) {
+      var v;
+      if ((v = parse_int_or_undefined(headers.get("X-RateLimit-" + name))) || (v = parse_int_or_undefined(headers.get("X-Rate-Limit-" + name)))) {
+        return v;
+      } else {
+        return void 0;
+      }
+    };
+
     BaseScraper.prototype._get_url_body = function(opts, cb) {
       var body, err, f, rc, response, rl_limit, rl_remaining, rl_reset, ___iced_passed_deferral, __iced_deferrals, __iced_k, _base;
       __iced_k = __iced_k_noop;
@@ -160,7 +171,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/base.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
             funcname: "BaseScraper._get_url_body"
           });
           _this.libs.fetch(opts.url, opts, __iced_deferrals.defer({
@@ -170,7 +181,7 @@
                 return response = arguments[1];
               };
             })(),
-            lineno: 87
+            lineno: 96
           }));
           __iced_deferrals._fulfill();
         });
@@ -178,9 +189,9 @@
         return function() {
           var _base1, _ref;
           if (opts.log_ratelimit && (typeof response !== "undefined" && response !== null)) {
-            rl_limit = parse_int_or_undefined(response.headers.get('X-RateLimit-Limit'));
-            rl_remaining = parse_int_or_undefined(response.headers.get('X-RateLimit-Remaining'));
-            rl_reset = parse_int_or_undefined(response.headers.get('X-RateLimit-Reset'));
+            rl_limit = _this._get_rate_limit_header(response.headers, 'Limit');
+            rl_remaining = _this._get_rate_limit_header(response.headers, 'Remaining');
+            rl_reset = _this._get_rate_limit_header(response.headers, 'Reset');
             if (rl_limit != null) {
               _this.log("| ratelimit info limit=" + rl_limit + " remaining=" + rl_remaining + " reset=" + rl_reset);
             }
@@ -188,7 +199,8 @@
               _base1.ratelimit_inform({
                 limit: rl_limit,
                 remaining: rl_remaining,
-                reset: rl_reset
+                reset: rl_reset,
+                endpoint_name: opts.endpoint_name
               });
             }
           }
@@ -201,7 +213,7 @@
                   (function(__iced_k) {
                     __iced_deferrals = new iced.Deferrals(__iced_k, {
                       parent: ___iced_passed_deferral,
-                      filename: "/home/zapu/Projects/KB/proofs/src/scrapers/base.iced",
+                      filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
                       funcname: "BaseScraper._get_url_body"
                     });
                     f.call(response, __iced_deferrals.defer({
@@ -211,7 +223,7 @@
                           return body = arguments[1];
                         };
                       })(),
-                      lineno: 109
+                      lineno: 122
                     }));
                     __iced_deferrals._fulfill();
                   })(function() {
@@ -244,6 +256,122 @@
 
   exports.BaseScraper = BaseScraper;
 
+  exports.BaseBearerToken = BaseBearerToken = (function() {
+    function BaseBearerToken(_arg) {
+      this.name = _arg.name, this.base = _arg.base, this.access_token_url = _arg.access_token_url, this.scope = _arg.scope, this.user_agent = _arg.user_agent;
+      if (this.access_token_url == null) {
+        throw new Error("@access_token_url is required");
+      }
+      if ((this.scope != null) && !Array.isArray(this.scope)) {
+        throw new Error("@scope has to be an array if present");
+      }
+      this._tok = null;
+      this._created = 0;
+      this._lock = new Lock();
+      this.auth = this.base.auth;
+    }
+
+    BaseBearerToken.prototype.get = function(cb) {
+      var body, cred, e, err, now, opts, rc, req, res, tok, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      __iced_k = __iced_k_noop;
+      ___iced_passed_deferral = iced.findDeferral(arguments);
+      (function(_this) {
+        return (function(__iced_k) {
+          __iced_deferrals = new iced.Deferrals(__iced_k, {
+            parent: ___iced_passed_deferral,
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
+            funcname: "BaseBearerToken.get"
+          });
+          _this._lock.acquire(__iced_deferrals.defer({
+            lineno: 160
+          }));
+          __iced_deferrals._fulfill();
+        });
+      })(this)((function(_this) {
+        return function() {
+          err = null;
+          now = Math.floor(Date.now() / 1000);
+          (function(__iced_k) {
+            if (((res = _this._tok) == null) || (now - _this._created > _this.auth.lifespan)) {
+              _this.base.log("+ Request for bearer token");
+              cred = (Buffer.from([_this.auth.key, _this.auth.secret].join(":"))).toString('base64');
+              req = 'grant_type=client_credentials';
+              if (_this.scope) {
+                req += "&scope=" + (encodeURIComponent(_this.scope.join(' ')));
+              }
+              opts = {
+                url: _this.access_token_url,
+                headers: {
+                  Authorization: "Basic " + cred,
+                  'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
+                  'Content-Length': req.length
+                },
+                method: "post",
+                body: req
+              };
+              if (_this.user_agent) {
+                opts.user_agent = _this.user_agent;
+              }
+              (function(__iced_k) {
+                __iced_deferrals = new iced.Deferrals(__iced_k, {
+                  parent: ___iced_passed_deferral,
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/base.iced",
+                  funcname: "BaseBearerToken.get"
+                });
+                _this.base._get_url_body(opts, __iced_deferrals.defer({
+                  assign_fn: (function() {
+                    return function() {
+                      err = arguments[0];
+                      rc = arguments[1];
+                      return body = arguments[2];
+                    };
+                  })(),
+                  lineno: 185
+                }));
+                __iced_deferrals._fulfill();
+              })(function() {
+                if (err != null) {
+                  _this.base.logl('error', "In getting bearer_token: " + err.message);
+                } else if (rc !== v_codes.OK) {
+                  _this.base.logl('error', "error in getting bearer token: " + rc);
+                  err = new Error("error: " + rc);
+                } else {
+                  try {
+                    body = JSON.parse(body);
+                  } catch (_error) {
+                    e = _error;
+                    _this.base.logl('warn', "Could not parse JSON reply: " + e);
+                    err = e;
+                  }
+                }
+                if (err != null) {
+
+                } else if ((tok = body.access_token) == null) {
+                  _this.base.logl('warn', "No access token found in reply");
+                  err = new Error("" + _this.name + " error: no access token");
+                } else {
+                  _this._tok = tok;
+                  _this._created = Math.floor(Date.now() / 1000);
+                  res = _this._tok;
+                }
+                _this.base.log("- Request for bearer token for " + _this.name + " -> err: " + err);
+                return __iced_k(!err ? _this.base.log("Bearer token for " + _this.name + " is: " + (res != null ? res.substr(0, 5) : void 0) + "...") : void 0);
+              });
+            } else {
+              return __iced_k();
+            }
+          })(function() {
+            _this._lock.release();
+            return cb(err, res);
+          });
+        };
+      })(this));
+    };
+
+    return BaseBearerToken;
+
+  })();
+
   parse_int_or_undefined = function(value) {
     var out;
     if ((out = Number(value)) === parseInt(value) && !isNaN(out)) {

package/lib/scrapers/dns.js

@@ -151,7 +151,7 @@
                   (function(__iced_k) {
                     __iced_deferrals = new iced.Deferrals(__iced_k, {
                       parent: ___iced_passed_deferral,
-                      filename: "/Users/max/src/keybase/proofs/src/scrapers/dns.iced",
+                      filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/dns.iced",
                       funcname: "DnsScraper.check_status"
                     });
                     _this._check_status({
@@ -191,28 +191,40 @@ _break()
       })(this));
     };
 
+    DnsScraper.prototype._resolve_txt = function(domain, cb) {
+      var dnslib, err;
+      dnslib = this.libs.dns || dns;
+      try {
+        return dnslib.resolveTxt(domain, function(err, records) {
+          return cb(err, records);
+        });
+      } catch (_error) {
+        err = _error;
+        return cb(err);
+      }
+    };
+
     DnsScraper.prototype._check_status = function(_arg, cb) {
-      var dnslib, domain, err, proof_text_check, rc, records, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      var domain, err, proof_text_check, rc, records, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
       domain = _arg.domain, proof_text_check = _arg.proof_text_check;
       this.log("+ DNS check for " + domain);
-      dnslib = this.libs.dns || dns;
       (function(_this) {
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/max/src/keybase/proofs/src/scrapers/dns.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/dns.iced",
             funcname: "DnsScraper._check_status"
           });
-          dnslib.resolveTxt(domain, __iced_deferrals.defer({
+          _this._resolve_txt(domain, __iced_deferrals.defer({
             assign_fn: (function() {
               return function() {
                 err = arguments[0];
                 return records = arguments[1];
               };
             })(),
-            lineno: 105
+            lineno: 115
           }));
           __iced_deferrals._fulfill();
         });

package/lib/scrapers/reddit.js

@@ -1,13 +1,13 @@
 // Generated by IcedCoffeeScript 108.0.11
 (function() {
-  var BaseScraper, PREFIX, RedditScraper, SUBREDDIT, constants, iced, proof_text_check_to_med_id, sncmp, v_codes, __iced_k, __iced_k_noop, _ref,
+  var BaseBearerToken, BaseScraper, OAUTH_REDDIT_URL, PREFIX, REDDIT_USER_AGENT, RedditBearerToken, RedditScraper, SUBREDDIT, bearer_token, constants, iced, inspect, proof_text_check_to_med_id, sncmp, urlmod, user_agent, v_codes, __iced_k, __iced_k_noop, _bearer_token, _ref,
     __hasProp = {}.hasOwnProperty,
     __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
 
   iced = require('iced-runtime');
   __iced_k = __iced_k_noop = function() {};
 
-  _ref = require('./base'), sncmp = _ref.sncmp, BaseScraper = _ref.BaseScraper;
+  _ref = require('./base'), sncmp = _ref.sncmp, user_agent = _ref.user_agent, BaseScraper = _ref.BaseScraper, BaseBearerToken = _ref.BaseBearerToken;
 
   constants = require('../constants').constants;
 
@@ -15,14 +15,54 @@
 
   proof_text_check_to_med_id = require('../base').proof_text_check_to_med_id;
 
+  urlmod = require('url');
+
+  inspect = require('util').inspect;
+
   PREFIX = "https://www.reddit.com";
 
   SUBREDDIT = PREFIX + "/r/keybaseproofs";
 
+  OAUTH_REDDIT_URL = "https://oauth.reddit.com";
+
+  REDDIT_USER_AGENT = user_agent + " (by /u/maxtaco)";
+
+  RedditBearerToken = (function(_super) {
+    __extends(RedditBearerToken, _super);
+
+    function RedditBearerToken(_arg) {
+      this.base = _arg.base;
+      RedditBearerToken.__super__.constructor.call(this, {
+        name: "Reddit",
+        base: this.base,
+        access_token_url: "" + OAUTH_REDDIT_URL + "/api/v1/access_token",
+        scope: ['history', 'read'],
+        user_agent: REDDIT_USER_AGENT
+      });
+    }
+
+    return RedditBearerToken;
+
+  })(BaseBearerToken);
+
+  _bearer_token = null;
+
+  bearer_token = function(_arg) {
+    var base;
+    base = _arg.base;
+    if (!_bearer_token) {
+      _bearer_token = new RedditBearerToken({
+        base: base
+      });
+    }
+    return _bearer_token;
+  };
+
   exports.RedditScraper = RedditScraper = (function(_super) {
     __extends(RedditScraper, _super);
 
     function RedditScraper(opts) {
+      this.auth = opts.auth;
       RedditScraper.__super__.constructor.call(this, opts);
     }
 
@@ -60,11 +100,11 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/max/src/keybase/proofs/src/scrapers/reddit.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
             funcname: "RedditScraper.hunt2"
           });
-          _this._get_url_body({
-            url: "" + PREFIX + "/user/" + (encodeURIComponent(username)) + "/submitted.json?count=25&cachebust=" + (Math.random()),
+          _this._get_body_api({
+            url: "" + OAUTH_REDDIT_URL + "/user/" + (encodeURIComponent(username)) + "/submitted.json?count=25&cachebust=" + (Math.random()),
             json: true
           }, __iced_deferrals.defer({
             assign_fn: (function() {
@@ -74,7 +114,7 @@
                 return json = arguments[2];
               };
             })(),
-            lineno: 43
+            lineno: 69
           }));
           __iced_deferrals._fulfill();
         });
@@ -169,21 +209,32 @@
     };
 
     RedditScraper.prototype.check_status = function(_arg, cb) {
-      var api_url, err, json, post, proof_text_check, rc, remote_id, username, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      var api_url, err, json, post, proof_text_check, rc, remote_id, u, username, ___iced_passed_deferral, __iced_deferrals, __iced_k, _ref1;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
       username = _arg.username, api_url = _arg.api_url, proof_text_check = _arg.proof_text_check, remote_id = _arg.remote_id;
+      u = urlmod.parse(api_url);
+      if (!u) {
+        return cb(new Error("Failed to parse api_url"));
+      }
+      if ((_ref1 = u.host) !== 'www.reddit.com' && _ref1 !== 'oauth.reddit.com') {
+        return cb(new Error("Unexpected api_url host: " + host));
+      }
+      u = urlmod.format({
+        host: 'oauth.reddit.com',
+        protocol: 'https:',
+        pathname: u.pathname
+      });
       (function(_this) {
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/max/src/keybase/proofs/src/scrapers/reddit.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
             funcname: "RedditScraper.check_status"
           });
-          _this._get_url_body({
-            url: api_url + ("?cachebust=" + (Math.random())),
-            json: true,
-            user_agent: "linux:com.github/keybase/proofs:v2.1.41 (by /u/maxtaco)"
+          _this._get_body_api({
+            url: u,
+            json: true
           }, __iced_deferrals.defer({
             assign_fn: (function() {
               return function() {
@@ -192,7 +243,7 @@
                 return json = arguments[2];
               };
             })(),
-            lineno: 117
+            lineno: 158
           }));
           __iced_deferrals._fulfill();
         });
@@ -208,6 +259,110 @@
       })(this));
     };
 
+    RedditScraper.prototype._get_bearer_token = function(cb) {
+      var bt, err, rc, tok, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      __iced_k = __iced_k_noop;
+      ___iced_passed_deferral = iced.findDeferral(arguments);
+      bt = bearer_token({
+        base: this
+      });
+      (function(_this) {
+        return (function(__iced_k) {
+          __iced_deferrals = new iced.Deferrals(__iced_k, {
+            parent: ___iced_passed_deferral,
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
+            funcname: "RedditScraper._get_bearer_token"
+          });
+          bt.get(__iced_deferrals.defer({
+            assign_fn: (function() {
+              return function() {
+                err = arguments[0];
+                return tok = arguments[1];
+              };
+            })(),
+            lineno: 169
+          }));
+          __iced_deferrals._fulfill();
+        });
+      })(this)((function(_this) {
+        return function() {
+          rc = typeof err !== "undefined" && err !== null ? v_codes.AUTH_FAILED : v_codes.OK;
+          return cb(err, rc, tok);
+        };
+      })(this));
+    };
+
+    RedditScraper.prototype._get_body_api = function(_arg, cb) {
+      var args, body, err, json, rc, tok, url, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      __iced_k = __iced_k_noop;
+      ___iced_passed_deferral = iced.findDeferral(arguments);
+      url = _arg.url, json = _arg.json;
+      rc = body = err = null;
+      (function(_this) {
+        return (function(__iced_k) {
+          __iced_deferrals = new iced.Deferrals(__iced_k, {
+            parent: ___iced_passed_deferral,
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
+            funcname: "RedditScraper._get_body_api"
+          });
+          _this._get_bearer_token(__iced_deferrals.defer({
+            assign_fn: (function() {
+              return function() {
+                err = arguments[0];
+                rc = arguments[1];
+                return tok = arguments[2];
+              };
+            })(),
+            lineno: 177
+          }));
+          __iced_deferrals._fulfill();
+        });
+      })(this)((function(_this) {
+        return function() {
+          (function(__iced_k) {
+            if (err == null) {
+              _this.log("| HTTP API request for URL '" + url + "'");
+              if (json == null) {
+                json = true;
+              }
+              args = {
+                url: url,
+                json: json,
+                headers: {
+                  Authorization: "Bearer " + tok
+                },
+                method: "get",
+                json: true,
+                user_agent: REDDIT_USER_AGENT
+              };
+              (function(__iced_k) {
+                __iced_deferrals = new iced.Deferrals(__iced_k, {
+                  parent: ___iced_passed_deferral,
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/reddit.iced",
+                  funcname: "RedditScraper._get_body_api"
+                });
+                _this._get_url_body(args, __iced_deferrals.defer({
+                  assign_fn: (function() {
+                    return function() {
+                      err = arguments[0];
+                      rc = arguments[1];
+                      return body = arguments[2];
+                    };
+                  })(),
+                  lineno: 189
+                }));
+                __iced_deferrals._fulfill();
+              })(__iced_k);
+            } else {
+              return __iced_k();
+            }
+          })(function() {
+            return cb(err, rc, body);
+          });
+        };
+      })(this));
+    };
+
     return RedditScraper;
 
   })(BaseScraper);