keybase-proofs 2.3.26 → 2.3.29

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 2.3.29 (2024-07-22)
+
+- Fix Twitter scraper to handle empty result list case.
+
+## 2.3.28 (2024-07-19)
+
+- Use version 2 API for X (formerly known as Twitter) hunting.
+
+## 2.3.27 (2024-07-18)
+
+- Fix error handling when an invalid unicode domain is passed to DnsScraper.
+
 ## 2.3.26 (2024-06-17)
 
 - Use oauth API for reddit scraping.

package/lib/constants.js

@@ -1,4 +1,4 @@
-// Generated by IcedCoffeeScript 108.0.12
+// Generated by IcedCoffeeScript 108.0.11
 (function() {
   var constants, d, k, v, _ref;
 
@@ -172,6 +172,7 @@
       HTTP_400: 240,
       HTTP_OTHER: 260,
       EMPTY_JSON: 270,
+      INVALID_JSON: 271,
       DELETED: 301,
       SERVICE_DEAD: 302,
       BAD_SIGNATURE: 303

package/lib/scrapers/base.js

@@ -143,6 +143,15 @@
       return b64find(raw, ptc_buf);
     };
 
+    BaseScraper.prototype._get_rate_limit_header = function(headers, name) {
+      var v;
+      if ((v = parse_int_or_undefined(headers.get("X-RateLimit-" + name))) || (v = parse_int_or_undefined(headers.get("X-Rate-Limit-" + name)))) {
+        return v;
+      } else {
+        return void 0;
+      }
+    };
+
     BaseScraper.prototype._get_url_body = function(opts, cb) {
       var body, err, f, rc, response, rl_limit, rl_remaining, rl_reset, ___iced_passed_deferral, __iced_deferrals, __iced_k, _base;
       __iced_k = __iced_k_noop;
@@ -172,7 +181,7 @@
                 return response = arguments[1];
               };
             })(),
-            lineno: 88
+            lineno: 96
           }));
           __iced_deferrals._fulfill();
         });
@@ -180,9 +189,9 @@
         return function() {
           var _base1, _ref;
           if (opts.log_ratelimit && (typeof response !== "undefined" && response !== null)) {
-            rl_limit = parse_int_or_undefined(response.headers.get('X-RateLimit-Limit'));
-            rl_remaining = parse_int_or_undefined(response.headers.get('X-RateLimit-Remaining'));
-            rl_reset = parse_int_or_undefined(response.headers.get('X-RateLimit-Reset'));
+            rl_limit = _this._get_rate_limit_header(response.headers, 'Limit');
+            rl_remaining = _this._get_rate_limit_header(response.headers, 'Remaining');
+            rl_reset = _this._get_rate_limit_header(response.headers, 'Reset');
             if (rl_limit != null) {
               _this.log("| ratelimit info limit=" + rl_limit + " remaining=" + rl_remaining + " reset=" + rl_reset);
             }
@@ -190,7 +199,8 @@
               _base1.ratelimit_inform({
                 limit: rl_limit,
                 remaining: rl_remaining,
-                reset: rl_reset
+                reset: rl_reset,
+                endpoint_name: opts.endpoint_name
               });
             }
           }
@@ -213,7 +223,7 @@
                           return body = arguments[1];
                         };
                       })(),
-                      lineno: 110
+                      lineno: 122
                     }));
                     __iced_deferrals._fulfill();
                   })(function() {
@@ -273,7 +283,7 @@
             funcname: "BaseBearerToken.get"
           });
           _this._lock.acquire(__iced_deferrals.defer({
-            lineno: 148
+            lineno: 160
           }));
           __iced_deferrals._fulfill();
         });
@@ -316,7 +326,7 @@
                       return body = arguments[2];
                     };
                   })(),
-                  lineno: 173
+                  lineno: 185
                 }));
                 __iced_deferrals._fulfill();
               })(function() {

package/lib/scrapers/dns.js

@@ -151,7 +151,7 @@
                   (function(__iced_k) {
                     __iced_deferrals = new iced.Deferrals(__iced_k, {
                       parent: ___iced_passed_deferral,
-                      filename: "/Users/max/src/keybase/proofs/src/scrapers/dns.iced",
+                      filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/dns.iced",
                       funcname: "DnsScraper.check_status"
                     });
                     _this._check_status({
@@ -191,28 +191,40 @@ _break()
       })(this));
     };
 
+    DnsScraper.prototype._resolve_txt = function(domain, cb) {
+      var dnslib, err;
+      dnslib = this.libs.dns || dns;
+      try {
+        return dnslib.resolveTxt(domain, function(err, records) {
+          return cb(err, records);
+        });
+      } catch (_error) {
+        err = _error;
+        return cb(err);
+      }
+    };
+
     DnsScraper.prototype._check_status = function(_arg, cb) {
-      var dnslib, domain, err, proof_text_check, rc, records, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      var domain, err, proof_text_check, rc, records, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
       domain = _arg.domain, proof_text_check = _arg.proof_text_check;
       this.log("+ DNS check for " + domain);
-      dnslib = this.libs.dns || dns;
       (function(_this) {
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/max/src/keybase/proofs/src/scrapers/dns.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/dns.iced",
             funcname: "DnsScraper._check_status"
           });
-          dnslib.resolveTxt(domain, __iced_deferrals.defer({
+          _this._resolve_txt(domain, __iced_deferrals.defer({
             assign_fn: (function() {
               return function() {
                 err = arguments[0];
                 return records = arguments[1];
               };
             })(),
-            lineno: 105
+            lineno: 115
           }));
           __iced_deferrals._fulfill();
         });

package/lib/scrapers/twitter.js

@@ -1,20 +1,18 @@
-// Generated by IcedCoffeeScript 108.0.12
+// Generated by IcedCoffeeScript 108.0.11
 (function() {
-  var BaseScraper, BearerToken, Lock, TwitterScraper, bearer_token, constants, decode_sig, iced, make_ids, schema, sncmp, urlmod, v_codes, ws_normalize, __iced_k, __iced_k_noop, _bearer_token, _ref,
+  var BaseBearerToken, BaseScraper, TwitterBearerToken, TwitterScraper, bearer_token, constants, decode_sig, iced, make_ids, schema, sncmp, urlmod, v_codes, ws_normalize, __iced_k, __iced_k_noop, _bearer_token, _ref,
     __hasProp = {}.hasOwnProperty,
     __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
 
   iced = require('iced-runtime');
   __iced_k = __iced_k_noop = function() {};
 
-  _ref = require('./base'), sncmp = _ref.sncmp, BaseScraper = _ref.BaseScraper;
+  _ref = require('./base'), sncmp = _ref.sncmp, BaseScraper = _ref.BaseScraper, BaseBearerToken = _ref.BaseBearerToken;
 
   make_ids = require('../base').make_ids;
 
   constants = require('../constants').constants;
 
-  Lock = require('../util').Lock;
-
   v_codes = constants.v_codes;
 
   decode_sig = require('kbpgp').ukm.decode_sig;
@@ -35,108 +33,22 @@
     return v.join(' ');
   };
 
-  BearerToken = (function() {
-    function BearerToken(_arg) {
+  TwitterBearerToken = (function(_super) {
+    __extends(TwitterBearerToken, _super);
+
+    function TwitterBearerToken(_arg) {
       this.base = _arg.base;
-      this._tok = null;
-      this._created = 0;
-      this._lock = new Lock();
-      this.auth = this.base.auth;
+      TwitterBearerToken.__super__.constructor.call(this, {
+        name: "Twitter",
+        base: this.base,
+        access_token_url: "https://api.twitter.com/oauth2/token",
+        scope: ['history', 'read']
+      });
     }
 
-    BearerToken.prototype.get = function(cb) {
-      var body, cred, e, err, now, opts, rc, req, res, tok, ___iced_passed_deferral, __iced_deferrals, __iced_k;
-      __iced_k = __iced_k_noop;
-      ___iced_passed_deferral = iced.findDeferral(arguments);
-      (function(_this) {
-        return (function(__iced_k) {
-          __iced_deferrals = new iced.Deferrals(__iced_k, {
-            parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
-            funcname: "BearerToken.get"
-          });
-          _this._lock.acquire(__iced_deferrals.defer({
-            lineno: 32
-          }));
-          __iced_deferrals._fulfill();
-        });
-      })(this)((function(_this) {
-        return function() {
-          err = null;
-          now = Math.floor(Date.now() / 1000);
-          (function(__iced_k) {
-            if (((res = _this._tok) == null) || (now - _this._created > _this.auth.lifespan)) {
-              _this.base.log("+ Request for bearer token");
-              cred = (Buffer.from([_this.auth.key, _this.auth.secret].join(":"))).toString('base64');
-              req = 'grant_type=client_credentials';
-              opts = {
-                url: "https://api.twitter.com/oauth2/token",
-                headers: {
-                  Authorization: "Basic " + cred,
-                  'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
-                  'Content-Length': req.length
-                },
-                method: "post",
-                body: req
-              };
-              (function(__iced_k) {
-                __iced_deferrals = new iced.Deferrals(__iced_k, {
-                  parent: ___iced_passed_deferral,
-                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
-                  funcname: "BearerToken.get"
-                });
-                _this.base._get_url_body(opts, __iced_deferrals.defer({
-                  assign_fn: (function() {
-                    return function() {
-                      err = arguments[0];
-                      rc = arguments[1];
-                      return body = arguments[2];
-                    };
-                  })(),
-                  lineno: 52
-                }));
-                __iced_deferrals._fulfill();
-              })(function() {
-                if (err != null) {
-                  _this.base.logl('error', "In getting bearer_token: " + err.message);
-                } else if (rc !== v_codes.OK) {
-                  _this.base.logl('error', "HTTP error in getting bearer token: " + rc);
-                  err = new Error("HTTP error: " + rc);
-                } else {
-                  try {
-                    body = JSON.parse(body);
-                  } catch (_error) {
-                    e = _error;
-                    _this.base.logl('warn', "Could not parse JSON reply: " + e);
-                    err = e;
-                  }
-                }
-                if (err != null) {
-
-                } else if ((tok = body.access_token) == null) {
-                  _this.base.logl('warn', "No access token found in reply");
-                  err = new Error("Twitter error: no access token");
-                } else {
-                  _this._tok = tok;
-                  _this._created = Math.floor(Date.now() / 1000);
-                  res = _this._tok;
-                }
-                return __iced_k(_this.base.log("- Request for bearer token -> " + err));
-              });
-            } else {
-              return __iced_k();
-            }
-          })(function() {
-            _this._lock.release();
-            return cb(err, res);
-          });
-        };
-      })(this));
-    };
-
-    return BearerToken;
+    return TwitterBearerToken;
 
-  })();
+  })(BaseBearerToken);
 
   _bearer_token = null;
 
@@ -144,7 +56,7 @@
     var base;
     base = _arg.base;
     if (!_bearer_token) {
-      _bearer_token = new BearerToken({
+      _bearer_token = new TwitterBearerToken({
         base: base
       });
     }
@@ -152,8 +64,12 @@
   };
 
   exports.TwitterScraper = TwitterScraper = (function(_super) {
+    var username_regexp;
+
     __extends(TwitterScraper, _super);
 
+    username_regexp = /^[a-z0-9_-]{2,15}$/;
+
     function TwitterScraper(opts) {
       this.auth = opts.auth;
       TwitterScraper.__super__.constructor.call(this, opts);
@@ -164,13 +80,15 @@
         return new Error("Bad args to Twitter proof: no username given");
       } else if (!(args.name != null) || (args.name !== 'twitter')) {
         return new Error("Bad args to Twitter proof: type is " + args.name);
+      } else if (!args.username.match(username_regexp)) {
+        return new Error("Invalid username passed to Twitter proof");
       } else {
         return null;
       }
     };
 
     TwitterScraper.prototype.hunt2 = function(_arg, cb) {
-      var api_url, err, human_url, i, id_str, json, name, out, proof_text_check, rc, remote_id, text, u, username, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      var api_url, endpoint_name, err, human_url, i, id, json, name, out, proof_text_check, rc, remote_id, text, u, username, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
       username = _arg.username, name = _arg.name, proof_text_check = _arg.proof_text_check;
@@ -182,24 +100,25 @@
       })) != null) {
         return cb(err, out);
       }
+      endpoint_name = "/2/tweets/search/recent";
       u = urlmod.format({
         host: "api.twitter.com",
         protocol: "https:",
-        pathname: "/1.1/statuses/user_timeline.json",
+        pathname: endpoint_name,
         query: {
-          count: 100,
-          screen_name: username
+          query: "\"Verifying myself\" \"Keybase.io\" from:" + username
         }
       });
       (function(_this) {
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.hunt2"
           });
           _this._get_body_api({
-            url: u
+            url: u,
+            endpoint_name: endpoint_name
           }, __iced_deferrals.defer({
             assign_fn: (function() {
               return function() {
@@ -208,28 +127,36 @@
                 return json = arguments[2];
               };
             })(),
-            lineno: 124
+            lineno: 73
           }));
           __iced_deferrals._fulfill();
         });
       })(this)((function(_this) {
         return function() {
-          var _i, _len, _ref1;
+          var _i, _len, _ref1, _ref2, _ref3;
           _this.log("| search index " + u + " -> " + rc);
           if (rc !== v_codes.OK) {
 
           } else if ((typeof json === "undefined" || json === null) || (json.length === 0)) {
             rc = v_codes.EMPTY_JSON;
+          } else if (json.data == null) {
+            if (((_ref1 = json.meta) != null ? _ref1.result_count : void 0) === 0) {
+              rc = v_codes.NOT_FOUND;
+            } else {
+              rc = v_codes.INVALID_JSON;
+            }
           } else {
-            for (i = _i = 0, _len = json.length; _i < _len; i = ++_i) {
-              _ref1 = json[i], text = _ref1.text, id_str = _ref1.id_str;
+            rc = v_codes.NOT_FOUND;
+            _ref2 = json.data;
+            for (i = _i = 0, _len = _ref2.length; _i < _len; i = ++_i) {
+              _ref3 = _ref2[i], text = _ref3.text, id = _ref3.id;
               if ((_this.find_sig_in_tweet({
                 inside: text,
                 proof_text_check: proof_text_check
               })) === v_codes.OK) {
                 _this.log("| found valid tweet in stream @ " + i);
                 rc = v_codes.OK;
-                remote_id = id_str;
+                remote_id = id;
                 api_url = human_url = _this._id_to_url(username, remote_id);
                 out = {
                   remote_id: remote_id,
@@ -295,7 +222,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.users_lookup"
                 });
                 _this._get_body_api({
@@ -308,7 +235,7 @@
                       return json = arguments[2];
                     };
                   })(),
-                  lineno: 172
+                  lineno: 129
                 }));
                 __iced_deferrals._fulfill();
               })(function() {
@@ -338,11 +265,11 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.users_lookup"
                                   });
                                   setTimeout(__iced_deferrals.defer({
-                                    lineno: 186
+                                    lineno: 143
                                   }), cursor_wait);
                                   __iced_deferrals._fulfill();
                                 })(__iced_k);
@@ -424,7 +351,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.get_follower_ids"
                 });
                 _this._get_body_api({
@@ -437,7 +364,7 @@
                       return json = arguments[2];
                     };
                   })(),
-                  lineno: 224
+                  lineno: 181
                 }));
                 __iced_deferrals._fulfill();
               })(function() {
@@ -470,11 +397,11 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.get_follower_ids"
                                   });
                                   setTimeout(__iced_deferrals.defer({
-                                    lineno: 240
+                                    lineno: 197
                                   }), cursor_wait);
                                   __iced_deferrals._fulfill();
                                 })(__iced_k);
@@ -576,7 +503,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.check_status"
           });
           _this._get_url_body({
@@ -590,7 +517,7 @@
                 return body_obj = arguments[2];
               };
             })(),
-            lineno: 331
+            lineno: 288
           }));
           __iced_deferrals._fulfill();
         });
@@ -663,7 +590,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_bearer_token"
           });
           bt.get(__iced_deferrals.defer({
@@ -673,7 +600,7 @@
                 return tok = arguments[1];
               };
             })(),
-            lineno: 392
+            lineno: 349
           }));
           __iced_deferrals._fulfill();
         });
@@ -686,16 +613,16 @@
     };
 
     TwitterScraper.prototype._get_body_api = function(_arg, cb) {
-      var args, body, err, rc, tok, url, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      var args, body, endpoint_name, err, rc, tok, url, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
-      url = _arg.url;
+      url = _arg.url, endpoint_name = _arg.endpoint_name;
       rc = body = err = null;
       (function(_this) {
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_body_api"
           });
           _this._get_bearer_token(__iced_deferrals.defer({
@@ -706,7 +633,7 @@
                 return tok = arguments[2];
               };
             })(),
-            lineno: 401
+            lineno: 358
           }));
           __iced_deferrals._fulfill();
         });
@@ -721,12 +648,14 @@
                   Authorization: "Bearer " + tok
                 },
                 method: "get",
-                json: true
+                json: true,
+                log_ratelimit: endpoint_name != null,
+                endpoint_name: endpoint_name
               };
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper._get_body_api"
                 });
                 _this._get_url_body(args, __iced_deferrals.defer({
@@ -737,7 +666,7 @@
                       return body = arguments[2];
                     };
                   })(),
-                  lineno: 410
+                  lineno: 369
                 }));
                 __iced_deferrals._fulfill();
               })(__iced_k);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keybase-proofs",
-  "version": "2.3.26",
+  "version": "2.3.29",
   "description": "Publicly-verifiable proofs of identity",
   "main": "lib/main.js",
   "scripts": {

package/src/constants.iced

@@ -163,6 +163,7 @@ exports.constants = constants =
     HTTP_400:          240
     HTTP_OTHER:        260
     EMPTY_JSON:        270
+    INVALID_JSON:      271
 
     # Hard final errors
     DELETED:           301

package/src/scrapers/base.iced

@@ -79,6 +79,14 @@ class BaseScraper
 
   #-------------------------------------------------------------
 
+  _get_rate_limit_header : (headers, name) ->
+    # Twitter and Github can't agree on rate limit header names.
+    if ((v = parse_int_or_undefined headers.get("X-RateLimit-#{name}")) or
+        (v = parse_int_or_undefined headers.get("X-Rate-Limit-#{name}")))
+      return v
+    else
+      return undefined
+
   _get_url_body: (opts, cb) ->
     ###
       cb(err, status, body) only replies with body if status is 200
@@ -88,13 +96,17 @@ class BaseScraper
     opts.headers["User-Agent"] ?= (opts.user_agent or user_agent)
     await @libs.fetch opts.url, opts, defer(err, response)
     if opts.log_ratelimit and response?
-      rl_limit     = parse_int_or_undefined response.headers.get 'X-RateLimit-Limit' # 5000 for github # https://developer.github.com/v3/#rate-limiting
-      rl_remaining = parse_int_or_undefined response.headers.get 'X-RateLimit-Remaining'
-      rl_reset     = parse_int_or_undefined response.headers.get 'X-RateLimit-Reset' # utc timestamp in seconds when limit will replenish
+      rl_limit     = @_get_rate_limit_header response.headers, 'Limit' # 5000 for github # https://developer.github.com/v3/#rate-limiting
+      rl_remaining = @_get_rate_limit_header response.headers, 'Remaining'
+      rl_reset     = @_get_rate_limit_header response.headers, 'Reset' # utc timestamp in seconds when limit will replenish
       if rl_limit?
         @log "| ratelimit info limit=#{rl_limit} remaining=#{rl_remaining} reset=#{rl_reset}"
-      @libs.ratelimit_inform? {limit: rl_limit, remaining: rl_remaining, reset: rl_reset}
-
+      @libs.ratelimit_inform? {
+        limit: rl_limit
+        remaining: rl_remaining
+        reset: rl_reset
+        endpoint_name : opts.endpoint_name
+      }
     rc = if err?
       if err.message.includes('network timeout') then v_codes.TIMEOUT
       else                                            v_codes.HOST_UNREACHABLE

package/src/scrapers/dns.iced

@@ -97,13 +97,23 @@ exports.DnsScraper = class DnsScraper extends BaseScraper
 
   # ---------------------------------------------------------------------------
 
+  _resolve_txt : (domain, cb) ->
+    # We can use a DNS library passed in (in the case of native-dns running on
+    # the server)
+    dnslib = @libs.dns or dns
+    try
+      dnslib.resolveTxt domain, (err, records) ->
+        cb err, records
+    catch err
+      cb err
+
+  # ---------------------------------------------------------------------------
+
   # calls back with a v_code or null if it was ok
   _check_status: ({domain, proof_text_check}, cb) ->
     @log "+ DNS check for #{domain}"
 
-    # We can use a DNS library passed in (in the case of native-dns running on the server)
-    dnslib = @libs.dns or dns
-    await dnslib.resolveTxt domain, defer err, records
+    await @_resolve_txt domain, defer err, records
     rc = if err?
       @log "| DNS error: #{err}"
       v_codes.DNS_ERROR

package/src/scrapers/twitter.iced

@@ -1,7 +1,6 @@
-{sncmp,BaseScraper} = require './base'
+{sncmp,BaseScraper,BaseBearerToken} = require './base'
 {make_ids} = require '../base'
 {constants} = require '../constants'
-{Lock} = require '../util'
 {v_codes} = constants
 {decode_sig} = require('kbpgp').ukm
 urlmod = require 'url'
@@ -17,78 +16,27 @@ ws_normalize = (x) ->
 
 #================================================================================
 
-class BearerToken
-
-  #----------------
-
+class TwitterBearerToken extends BaseBearerToken
   constructor : ({@base}) ->
-    @_tok = null
-    @_created = 0
-    @_lock = new Lock()
-    @auth = @base.auth
-
-  #----------------
-
-  get : (cb) ->
-    await @_lock.acquire defer()
-    err = null
-    now = Math.floor(Date.now() / 1000)
-
-    if not (res = @_tok)? or (now - @_created > @auth.lifespan)
-
-      @base.log "+ Request for bearer token"
-
-      # Very crypto!  Not sure why this is done, but it's done
-      cred = (Buffer.from [ @auth.key, @auth.secret].join(":")).toString('base64')
-
-      req = 'grant_type=client_credentials'
-      opts =
-        url : "https://api.twitter.com/oauth2/token"
-        headers :
-          Authorization : "Basic #{cred}"
-          'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
-          'Content-Length': req.length
-        method : "post"
-        body : req
-      await @base._get_url_body opts, defer err, rc, body
-
-      if err?
-        @base.logl 'error', "In getting bearer_token: #{err.message}"
-      else if (rc isnt v_codes.OK)
-        @base.logl 'error', "HTTP error in getting bearer token: #{rc}"
-        err = new Error "HTTP error: #{rc}"
-      else
-        try
-          body = JSON.parse body
-        catch e
-          @base.logl 'warn', "Could not parse JSON reply: #{e}"
-          err = e
-
-      if err? then # noop
-      else if not (tok = body.access_token)?
-        @base.logl 'warn', "No access token found in reply"
-        err = new Error "Twitter error: no access token"
-      else
-        @_tok = tok
-        @_created = Math.floor(Date.now() / 1000)
-        res = @_tok
-
-      @base.log "- Request for bearer token -> #{err}"
-
-    @_lock.release()
-    cb err, res
+    super {
+      name: "Twitter"
+      @base
+      access_token_url : "https://api.twitter.com/oauth2/token"
+      scope : [ 'history', 'read' ]
+    }
 
 #================================================================================
 
 _bearer_token = null
 bearer_token = ({base}) ->
   unless _bearer_token
-    _bearer_token = new BearerToken { base }
+    _bearer_token = new TwitterBearerToken { base }
   return _bearer_token
 
 #================================================================================
 
 exports.TwitterScraper = class TwitterScraper extends BaseScraper
+  username_regexp = /^[a-z0-9_-]{2,15}$/
 
   constructor: (opts) ->
     @auth = opts.auth
@@ -101,6 +49,8 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
       new Error "Bad args to Twitter proof: no username given"
     else if not (args.name?) or (args.name isnt 'twitter')
       new Error "Bad args to Twitter proof: type is #{args.name}"
+    else if not args.username.match(username_regexp)
+      new Error "Invalid username passed to Twitter proof"
     else
       null
 
@@ -113,25 +63,32 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
 
     return cb(err,out) if (err = @_check_args { username, name })?
 
+    endpoint_name = "/2/tweets/search/recent"
     u = urlmod.format {
       host : "api.twitter.com"
       protocol : "https:"
-      pathname : "/1.1/statuses/user_timeline.json"
+      pathname : endpoint_name
       query :
-        count : 100
-        screen_name : username
+        query : "\"Verifying myself\" \"Keybase.io\" from:#{username}"
     }
-
-    await @_get_body_api { url : u }, defer err, rc, json
+    await @_get_body_api { url : u, endpoint_name }, defer err, rc, json
     @log "| search index #{u} -> #{rc}"
     if rc isnt v_codes.OK then #noop
     else if not json? or (json.length is 0) then rc = v_codes.EMPTY_JSON
+    else if not json.data?
+      if json.meta?.result_count is 0
+        # No results.
+        rc = v_codes.NOT_FOUND
+      else
+        # Unknown JSON structure.
+        rc = v_codes.INVALID_JSON
     else
-      for {text, id_str},i in json
+      rc = v_codes.NOT_FOUND
+      for {text, id},i in json.data
         if (@find_sig_in_tweet { inside : text, proof_text_check }) is v_codes.OK
           @log "| found valid tweet in stream @ #{i}"
           rc = v_codes.OK
-          remote_id = id_str
+          remote_id = id
           api_url = human_url = @_id_to_url username, remote_id
           out = { remote_id, api_url, human_url }
           break
@@ -397,7 +354,7 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
   # ---------------------------------------------------------------------------
 
   # Only the hunter needs this
-  _get_body_api : ({url}, cb) ->
+  _get_body_api : ({url, endpoint_name}, cb) ->
     rc = body = err = null
     await @_get_bearer_token defer err, rc, tok
     unless err?
@@ -408,6 +365,8 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
           Authorization : "Bearer #{tok}"
         method : "get"
         json : true
+        log_ratelimit : endpoint_name?
+        endpoint_name : endpoint_name
       await @_get_url_body args, defer err, rc, body
     cb err, rc, body
 

package/SIGNED.md

@@ -1,131 +0,0 @@
-##### Signed by https://keybase.io/max
-```
------BEGIN PGP SIGNATURE-----
-Comment: GPGTools - https://gpgtools.org
-
-iQEcBAABCgAGBQJWBZ3PAAoJEJgKPw0B/gTfPrkIALMDLUb+CS2oPrFZlILoC31o
-FUKtCiN+hzC9+FgMYp18kWsWDh3DxzqPIRs28mwt9QMQTHdywv6K+nfQw2dE4wns
-1Wi7ZVPbvR5P0/ANIQ0QwetYxbkNow20dU67CRj0aEvvbopQDBgBgyH9b6LjSb+n
-8OTUFLhtiXN0PeZJ5/OyMtq8jyrjT7MEpxAb+8uFt95fPax17TNjCBEF4mlNqvtl
-YIOd/1v6YTlaqbYSzEZoMZsSI12UGK/tLaPG6qYf+5gGi96YjEMuQyDsL7ANQpkG
-jplkuKpykiY1Ndc1B3TOnTK8n+lrRaWesLmHnhldhR047ZNYgETunz+NYks7MHI=
-=96ur
------END PGP SIGNATURE-----
-
-```
-
-<!-- END SIGNATURES -->
-
-### Begin signed statement 
-
-#### Expect
-
-```
-size   exec  file                             contents                                                        
-             ./                                                                                               
-109            .gitignore                     ec278daeb8f83cac2579d262b92ee6d7d872c4d1544e881ba515d8bcc05361ab
-10709          CHANGELOG.md                   8ce81ae853a72be10648f712497199b316dfe57c8519853687cee1b508965af1
-66             CONTIBUTORS.md                 e5a536ada9dc7f25b352f28cc8936db4ded1a0860f6f23ef61d0b14aeae351da
-1483           LICENSE                        333be7050513d91d9e77ca9acb4a91261721f0050209636076ed58676bfc643d
-1383           Makefile                       b90c2b9165497660a610d5ec6177b854aafaa318aa1252af23484c755f1d959a
-43             README.md                      b2db580fc880756ae739bb148570f6afb1c250e5c0d2c5fa56ce41491753ca6c
-               lib/                                                                                           
-2498             alloc.js                     af1347e6f17197fee04fba783839279b07b07f79bf407ddf44e09d360289aa48
-1189             announcement.js              ff01f39dca21c1387faa8b79633dec40414f5645886015a46bd795772987c79e
-1383             auth.js                      86403419686d1cac766c8d3d6c8f6e10176694e33de84faafca652fd01e54e0e
-766              b64extract.js                1466f81bb930ddc439f877ef0cba4d20048f5477aafda294290f0fd0da527073
-33694            base.js                      b57791baad7fd3b648749320875ada1adc6e8d0dc3265c6f2fe4faeb960c05bb
-2043             constants.js                 e88df2ce555fe3fa8f70bbac3ebdca7c169571cbd5055948ecd8215e231d75fb
-1382             cryptocurrency.js            40536713081183e8b698a345df9a288a0967e1c95bd0699fe8ea18b618a6de6a
-1087             device.js                    f77d694ccf19c81b8c1977bb5c0280979a8b15983ced95675d6f5d44cedf67a7
-1294             eldest.js                    878cc36219a2d25c1e1897a672f03b5d97ece05581b6ece7707e78cbb0a8e02e
-909              main.js                      1cc4c7fd9cfee3ce317d8993507a953c8a22803eaea69eb16d4c7be9c0d7731b
-1549             pgp_update.js                c084bee65b6e7cee2d9fd7ddeea107751a319c03831205005ed997fa4840f939
-2578             revoke.js                    57106f23d108ce01a6d781a795417c86a09633af73ff1c9630d3893a8bc47e5f
-                 scrapers/                                                                                    
-6392               base.js                    db98e2349217287c98feba718b0bdd9570c5e1108370942af9ca2a2b59be5ed9
-15450              bitbucket.js               7530f245a6836d29177cae7e2a1e0efa9860e1adc9965731cf1e79c5f82f3f4c
-4137               coinbase.js                5d28f2c525f69e64881d0ae5dd38c535a324f894d98b67d1a5b8217a4c564ce3
-8239               dns.js                     74ced258122595b80997594c4f8fb82c210b5f316848b035b2b66582bb6052e8
-9369               generic_web_site.js        aa3fff8c4d36f86182666642117ec71c4c1574d44eeeba675f698d08fb9a1edd
-9901               github.js                  221586b7fd13ec28a51479319902fdb143a1b37d25dcdb7926826b100917ffbd
-5995               hackernews.js              0e73885c9dba4dbba56fc042afe6002aeae1699094f77555c831d2a15348033c
-8535               reddit.js                  508d1b349ce6a992afe5c882d7c541c1ff85990645343dbfd748c09888d8ce3f
-27418              twitter.js                 fda7930b6de79013601c22d0585833e6c2bf09594ec7890a4852ef3e21d24df5
-1774             sibkey.js                    b434bb5e226214d5113eae44201ead14e7430bc92e56e2446e80d2cf9076ab1d
-10983            subkey.js                    76aca817ffb548ffe3dd668d9c2b189c3c3040300f748c6acf105c5f5a5dae82
-1669             track.js                     b85423eb1309ac58f5cb2b68697f612c091ef2d44fa8cc9e4b85ab6039ae8b9a
-1506             update_passphrase_hash.js    f2cfce382fd677e31a88f3646eb2d90258583a1ef68fe74dcd178a2e178229f6
-1544             util.js                      1760f3955e71bed8a332c02a1c9a96733f9af0e0f297c89ec354f83ed5bf3908
-19530            web_service.js               769a2c81aeb9b76ec627126776c346c20209ed588536dca528a617b5fbcdc553
-744            package.json                   b670f8f7efdd7d69fa92f089729bdfe4486ded548e03b677bab888e3e21f2ab0
-               src/                                                                                           
-2376             alloc.iced                   bce30dc075d16e0551aeb8ce89f2a0515e8b061f7cbd1c9842a93756c028254b
-537              announcement.iced            98ddc17f5c6b66874ed2a273032d5c5e118d1ef3a7f5a08f4c7fb9cf06e811cf
-671              auth.iced                    3f0f211734a2f8408a5f5e1f5c353cad3373f952b01209f0f2d644b681612b0e
-704              b64extract.iced              47b06460fd61d8eb788878566c62086b1fd1f792a1712b3e8736766401bc8fee
-15746            base.iced                    cbd3229cab90232ce33ca7ced8c2d492c2eaa5091a48019486f401f3df689f4a
-2264             constants.iced               ed833b20ade7e8e95d1e4c65577cfb8e7de20b554afaf7aaa7963e86d5f2b370
-609              cryptocurrency.iced          9c839bdd1c12591ca73c6fbc86d44c51f1803718d37286e9eade945bee0761d1
-489              device.iced                  b0c028eb2598c48d9d24be615e066e0a03e8b7b1930a4ca58f29d23517daf0bc
-566              eldest.iced                  8dd63bec35fca4ab933de049888777e957827a862ecf2dec7e91e91e4265917c
-741              main.iced                    0c4af12c6c069e3ece0f446c922ba8f306df4f386e3bee3c4ba852bc92a4df0b
-589              pgp_update.iced              cdc5fed1d47c0bfe144062cf7985d466ced99fb8ec7bd17901d504127359fc64
-788              revoke.iced                  2b94671a923dcdd274d0f769f186f0757a27db64828eaad5fa5eb1a24f9e479f
-                 scrapers/                                                                                    
-4029               base.iced                  e110786284dda6cd78bc0be507ff9ac0add3a128c959c073562416b513d8b2d8
-3967               bitbucket.iced             f22e9c29d781afbecccc0c720513fa63710e3c199cc9780ef34b87a4db0e1fd9
-2370               coinbase.iced              5b1132561f0c33d4c35ba2d7d9c32200e77d83c68a3bce3d770b27085d649bcd
-4081               dns.iced                   773ea8e65f46b2db994e41ed4849c4637200ee2e978f280524d63b06a42ff454
-3249               generic_web_site.iced      7a70be6c922e5adb73459973b81143ec9020b109cdbcc5acf349e47c5ca1f68d
-3305               github.iced                f10410ccca3bb3fc5045024dada97a4974984ee433476bea30c1ab47f870da90
-2909               hackernews.iced            cb14793b3ea889bcbc2121430bc3728655e125806001211864d3c90c1104505a
-4082               reddit.iced                8a4f6902eee7e1609df18bf9a00b5934d2e9a16eaf20bd6231b3d11b2554effe
-11418              twitter.iced               9bbb6cfdc464a5ed3261cc8f1a14a12e18325d6b0edc753c7a9e133b58d9328e
-761              sibkey.iced                  308c4cfef33dbc79809600a46c4f1704d4240be18478d2e5e8f71e523fd6962d
-3503             subkey.iced                  6f41a5aa38813486cbcb5a3c918c2e7aa464665b9dac852f92bb62ffa1e0a3b1
-838              track.iced                   d72d0eb5cda7f0f9800f17225fedf552f45111b802f66c463c4408794bd11aca
-658              update_passphrase_hash.iced  ea87dd15d3a36ee581e48a5436f4c64f38a102906efcdc20bc925b821210bfd9
-1052             util.iced                    88092afaf23bf8dfe42a72252ac156be45e94588911584ecf535d228b108f2b9
-10177            web_service.iced             bb436e165233be85342d0abe01237fcfb5773b0cc050abef49f1f19094abd8f0
-1673           test_scraper.iced              099823ea87768b45dd205bd6dc36773999730dadb26fe3844e0a05f8668bb9d1
-```
-
-#### Ignore
-
-```
-/SIGNED.md
-```
-
-#### Presets
-
-```
-git      # ignore .git and anything as described by .gitignore files
-dropbox  # ignore .dropbox-cache and other Dropbox-related files    
-kb       # ignore anything as described by .kbignore files          
-```
-
-<!-- summarize version = 0.0.9 -->
-
-### End signed statement
-
-<hr>
-
-#### Notes
-
-With keybase you can sign any directory's contents, whether it's a git repo,
-source code distribution, or a personal documents folder. It aims to replace the drudgery of:
-
-  1. comparing a zipped file to a detached statement
-  2. downloading a public key
-  3. confirming it is in fact the author's by reviewing public statements they've made, using it
-
-All in one simple command:
-
-```bash
-keybase dir verify
-```
-
-There are lots of options, including assertions for automating your checks.
-
-For more info, check out https://keybase.io/docs/command_line/code_signing