keybase-proofs 2.3.25 → 2.3.28

package/lib/scrapers/twitter.js

@@ -1,20 +1,18 @@
-// Generated by IcedCoffeeScript 108.0.12
+// Generated by IcedCoffeeScript 108.0.11
 (function() {
-  var BaseScraper, BearerToken, Lock, TwitterScraper, bearer_token, constants, decode_sig, iced, make_ids, schema, sncmp, urlmod, v_codes, ws_normalize, __iced_k, __iced_k_noop, _bearer_token, _ref,
+  var BaseBearerToken, BaseScraper, TwitterBearerToken, TwitterScraper, bearer_token, constants, decode_sig, iced, make_ids, schema, sncmp, urlmod, v_codes, ws_normalize, __iced_k, __iced_k_noop, _bearer_token, _ref,
     __hasProp = {}.hasOwnProperty,
     __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
 
   iced = require('iced-runtime');
   __iced_k = __iced_k_noop = function() {};
 
-  _ref = require('./base'), sncmp = _ref.sncmp, BaseScraper = _ref.BaseScraper;
+  _ref = require('./base'), sncmp = _ref.sncmp, BaseScraper = _ref.BaseScraper, BaseBearerToken = _ref.BaseBearerToken;
 
   make_ids = require('../base').make_ids;
 
   constants = require('../constants').constants;
 
-  Lock = require('../util').Lock;
-
   v_codes = constants.v_codes;
 
   decode_sig = require('kbpgp').ukm.decode_sig;
@@ -35,108 +33,22 @@
     return v.join(' ');
   };
 
-  BearerToken = (function() {
-    function BearerToken(_arg) {
+  TwitterBearerToken = (function(_super) {
+    __extends(TwitterBearerToken, _super);
+
+    function TwitterBearerToken(_arg) {
       this.base = _arg.base;
-      this._tok = null;
-      this._created = 0;
-      this._lock = new Lock();
-      this.auth = this.base.auth;
+      TwitterBearerToken.__super__.constructor.call(this, {
+        name: "Twitter",
+        base: this.base,
+        access_token_url: "https://api.twitter.com/oauth2/token",
+        scope: ['history', 'read']
+      });
     }
 
-    BearerToken.prototype.get = function(cb) {
-      var body, cred, e, err, now, opts, rc, req, res, tok, ___iced_passed_deferral, __iced_deferrals, __iced_k;
-      __iced_k = __iced_k_noop;
-      ___iced_passed_deferral = iced.findDeferral(arguments);
-      (function(_this) {
-        return (function(__iced_k) {
-          __iced_deferrals = new iced.Deferrals(__iced_k, {
-            parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
-            funcname: "BearerToken.get"
-          });
-          _this._lock.acquire(__iced_deferrals.defer({
-            lineno: 32
-          }));
-          __iced_deferrals._fulfill();
-        });
-      })(this)((function(_this) {
-        return function() {
-          err = null;
-          now = Math.floor(Date.now() / 1000);
-          (function(__iced_k) {
-            if (((res = _this._tok) == null) || (now - _this._created > _this.auth.lifespan)) {
-              _this.base.log("+ Request for bearer token");
-              cred = (Buffer.from([_this.auth.key, _this.auth.secret].join(":"))).toString('base64');
-              req = 'grant_type=client_credentials';
-              opts = {
-                url: "https://api.twitter.com/oauth2/token",
-                headers: {
-                  Authorization: "Basic " + cred,
-                  'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
-                  'Content-Length': req.length
-                },
-                method: "post",
-                body: req
-              };
-              (function(__iced_k) {
-                __iced_deferrals = new iced.Deferrals(__iced_k, {
-                  parent: ___iced_passed_deferral,
-                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
-                  funcname: "BearerToken.get"
-                });
-                _this.base._get_url_body(opts, __iced_deferrals.defer({
-                  assign_fn: (function() {
-                    return function() {
-                      err = arguments[0];
-                      rc = arguments[1];
-                      return body = arguments[2];
-                    };
-                  })(),
-                  lineno: 52
-                }));
-                __iced_deferrals._fulfill();
-              })(function() {
-                if (err != null) {
-                  _this.base.logl('error', "In getting bearer_token: " + err.message);
-                } else if (rc !== v_codes.OK) {
-                  _this.base.logl('error', "HTTP error in getting bearer token: " + rc);
-                  err = new Error("HTTP error: " + rc);
-                } else {
-                  try {
-                    body = JSON.parse(body);
-                  } catch (_error) {
-                    e = _error;
-                    _this.base.logl('warn', "Could not parse JSON reply: " + e);
-                    err = e;
-                  }
-                }
-                if (err != null) {
-
-                } else if ((tok = body.access_token) == null) {
-                  _this.base.logl('warn', "No access token found in reply");
-                  err = new Error("Twitter error: no access token");
-                } else {
-                  _this._tok = tok;
-                  _this._created = Math.floor(Date.now() / 1000);
-                  res = _this._tok;
-                }
-                return __iced_k(_this.base.log("- Request for bearer token -> " + err));
-              });
-            } else {
-              return __iced_k();
-            }
-          })(function() {
-            _this._lock.release();
-            return cb(err, res);
-          });
-        };
-      })(this));
-    };
-
-    return BearerToken;
+    return TwitterBearerToken;
 
-  })();
+  })(BaseBearerToken);
 
   _bearer_token = null;
 
@@ -144,7 +56,7 @@
     var base;
     base = _arg.base;
     if (!_bearer_token) {
-      _bearer_token = new BearerToken({
+      _bearer_token = new TwitterBearerToken({
         base: base
       });
     }
@@ -152,8 +64,12 @@
   };
 
   exports.TwitterScraper = TwitterScraper = (function(_super) {
+    var username_regexp;
+
     __extends(TwitterScraper, _super);
 
+    username_regexp = /^[a-z0-9_-]{2,15}$/;
+
     function TwitterScraper(opts) {
       this.auth = opts.auth;
       TwitterScraper.__super__.constructor.call(this, opts);
@@ -164,13 +80,15 @@
         return new Error("Bad args to Twitter proof: no username given");
       } else if (!(args.name != null) || (args.name !== 'twitter')) {
         return new Error("Bad args to Twitter proof: type is " + args.name);
+      } else if (!args.username.match(username_regexp)) {
+        return new Error("Invalid username passed to Twitter proof");
       } else {
         return null;
       }
     };
 
     TwitterScraper.prototype.hunt2 = function(_arg, cb) {
-      var api_url, err, human_url, i, id_str, json, name, out, proof_text_check, rc, remote_id, text, u, username, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      var api_url, endpoint_name, err, human_url, i, id, json, name, out, proof_text_check, rc, remote_id, text, u, username, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
       username = _arg.username, name = _arg.name, proof_text_check = _arg.proof_text_check;
@@ -182,24 +100,25 @@
       })) != null) {
         return cb(err, out);
       }
+      endpoint_name = "/2/tweets/search/recent";
       u = urlmod.format({
         host: "api.twitter.com",
         protocol: "https:",
-        pathname: "/1.1/statuses/user_timeline.json",
+        pathname: endpoint_name,
         query: {
-          count: 100,
-          screen_name: username
+          query: "\"Verifying myself\" \"Keybase.io\" from:" + username
         }
       });
       (function(_this) {
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.hunt2"
           });
           _this._get_body_api({
-            url: u
+            url: u,
+            endpoint_name: endpoint_name
           }, __iced_deferrals.defer({
             assign_fn: (function() {
               return function() {
@@ -208,28 +127,32 @@
                 return json = arguments[2];
               };
             })(),
-            lineno: 124
+            lineno: 73
           }));
           __iced_deferrals._fulfill();
         });
       })(this)((function(_this) {
         return function() {
-          var _i, _len, _ref1;
+          var _i, _len, _ref1, _ref2;
           _this.log("| search index " + u + " -> " + rc);
           if (rc !== v_codes.OK) {
 
           } else if ((typeof json === "undefined" || json === null) || (json.length === 0)) {
             rc = v_codes.EMPTY_JSON;
+          } else if (json.data == null) {
+            rc = v_codes.INVALID_JSON;
           } else {
-            for (i = _i = 0, _len = json.length; _i < _len; i = ++_i) {
-              _ref1 = json[i], text = _ref1.text, id_str = _ref1.id_str;
+            rc = v_codes.NOT_FOUND;
+            _ref1 = json.data;
+            for (i = _i = 0, _len = _ref1.length; _i < _len; i = ++_i) {
+              _ref2 = _ref1[i], text = _ref2.text, id = _ref2.id;
               if ((_this.find_sig_in_tweet({
                 inside: text,
                 proof_text_check: proof_text_check
               })) === v_codes.OK) {
                 _this.log("| found valid tweet in stream @ " + i);
                 rc = v_codes.OK;
-                remote_id = id_str;
+                remote_id = id;
                 api_url = human_url = _this._id_to_url(username, remote_id);
                 out = {
                   remote_id: remote_id,
@@ -295,7 +218,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.users_lookup"
                 });
                 _this._get_body_api({
@@ -308,7 +231,7 @@
                       return json = arguments[2];
                     };
                   })(),
-                  lineno: 172
+                  lineno: 123
                 }));
                 __iced_deferrals._fulfill();
               })(function() {
@@ -338,11 +261,11 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.users_lookup"
                                   });
                                   setTimeout(__iced_deferrals.defer({
-                                    lineno: 186
+                                    lineno: 137
                                   }), cursor_wait);
                                   __iced_deferrals._fulfill();
                                 })(__iced_k);
@@ -424,7 +347,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.get_follower_ids"
                 });
                 _this._get_body_api({
@@ -437,7 +360,7 @@
                       return json = arguments[2];
                     };
                   })(),
-                  lineno: 224
+                  lineno: 175
                 }));
                 __iced_deferrals._fulfill();
               })(function() {
@@ -470,11 +393,11 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.get_follower_ids"
                                   });
                                   setTimeout(__iced_deferrals.defer({
-                                    lineno: 240
+                                    lineno: 191
                                   }), cursor_wait);
                                   __iced_deferrals._fulfill();
                                 })(__iced_k);
@@ -576,7 +499,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.check_status"
           });
           _this._get_url_body({
@@ -590,7 +513,7 @@
                 return body_obj = arguments[2];
               };
             })(),
-            lineno: 331
+            lineno: 282
           }));
           __iced_deferrals._fulfill();
         });
@@ -663,7 +586,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_bearer_token"
           });
           bt.get(__iced_deferrals.defer({
@@ -673,7 +596,7 @@
                 return tok = arguments[1];
               };
             })(),
-            lineno: 392
+            lineno: 343
           }));
           __iced_deferrals._fulfill();
         });
@@ -686,16 +609,16 @@
     };
 
     TwitterScraper.prototype._get_body_api = function(_arg, cb) {
-      var args, body, err, rc, tok, url, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      var args, body, endpoint_name, err, rc, tok, url, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
-      url = _arg.url;
+      url = _arg.url, endpoint_name = _arg.endpoint_name;
       rc = body = err = null;
       (function(_this) {
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_body_api"
           });
           _this._get_bearer_token(__iced_deferrals.defer({
@@ -706,7 +629,7 @@
                 return tok = arguments[2];
               };
             })(),
-            lineno: 401
+            lineno: 352
           }));
           __iced_deferrals._fulfill();
         });
@@ -721,12 +644,14 @@
                   Authorization: "Bearer " + tok
                 },
                 method: "get",
-                json: true
+                json: true,
+                log_ratelimit: endpoint_name != null,
+                endpoint_name: endpoint_name
               };
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper._get_body_api"
                 });
                 _this._get_url_body(args, __iced_deferrals.defer({
@@ -737,7 +662,7 @@
                       return body = arguments[2];
                     };
                   })(),
-                  lineno: 410
+                  lineno: 363
                 }));
                 __iced_deferrals._fulfill();
               })(__iced_k);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keybase-proofs",
-  "version": "2.3.25",
+  "version": "2.3.28",
   "description": "Publicly-verifiable proofs of identity",
   "main": "lib/main.js",
   "scripts": {

package/src/constants.iced

@@ -163,6 +163,7 @@ exports.constants = constants =
     HTTP_400:          240
     HTTP_OTHER:        260
     EMPTY_JSON:        270
+    INVALID_JSON:      271
 
     # Hard final errors
     DELETED:           301

package/src/scrapers/base.iced

@@ -6,6 +6,7 @@ pkg = require '../../package.json'
 {b64find} = require '../b64extract'
 urlmod = require 'url'
 {callbackify} = require 'util'
+{Lock} = require '../util'
 
 #==============================================================
 
@@ -78,6 +79,14 @@ class BaseScraper
 
   #-------------------------------------------------------------
 
+  _get_rate_limit_header : (headers, name) ->
+    # Twitter and Github can't agree on rate limit header names.
+    if ((v = parse_int_or_undefined headers.get("X-RateLimit-#{name}")) or
+        (v = parse_int_or_undefined headers.get("X-Rate-Limit-#{name}")))
+      return v
+    else
+      return undefined
+
   _get_url_body: (opts, cb) ->
     ###
       cb(err, status, body) only replies with body if status is 200
@@ -87,13 +96,17 @@ class BaseScraper
     opts.headers["User-Agent"] ?= (opts.user_agent or user_agent)
     await @libs.fetch opts.url, opts, defer(err, response)
     if opts.log_ratelimit and response?
-      rl_limit     = parse_int_or_undefined response.headers.get 'X-RateLimit-Limit' # 5000 for github # https://developer.github.com/v3/#rate-limiting
-      rl_remaining = parse_int_or_undefined response.headers.get 'X-RateLimit-Remaining'
-      rl_reset     = parse_int_or_undefined response.headers.get 'X-RateLimit-Reset' # utc timestamp in seconds when limit will replenish
+      rl_limit     = @_get_rate_limit_header response.headers, 'Limit' # 5000 for github # https://developer.github.com/v3/#rate-limiting
+      rl_remaining = @_get_rate_limit_header response.headers, 'Remaining'
+      rl_reset     = @_get_rate_limit_header response.headers, 'Reset' # utc timestamp in seconds when limit will replenish
       if rl_limit?
         @log "| ratelimit info limit=#{rl_limit} remaining=#{rl_remaining} reset=#{rl_reset}"
-      @libs.ratelimit_inform? {limit: rl_limit, remaining: rl_remaining, reset: rl_reset}
-
+      @libs.ratelimit_inform? {
+        limit: rl_limit
+        remaining: rl_remaining
+        reset: rl_reset
+        endpoint_name : opts.endpoint_name
+      }
     rc = if err?
       if err.message.includes('network timeout') then v_codes.TIMEOUT
       else                                            v_codes.HOST_UNREACHABLE
@@ -118,6 +131,8 @@ class BaseScraper
       catch err
         cb err, rc, null
     else
+      # TODO: It's possible to get response body here, which might be useful
+      # for debugging and logging.
       cb err, rc, null
 
   #--------------------------------------------------------------
@@ -128,6 +143,78 @@ exports.BaseScraper = BaseScraper
 
 #==============================================================
 
+exports.BaseBearerToken = class BaseBearerToken
+  constructor : ({@name, @base, @access_token_url, @scope, @user_agent}) ->
+    unless @access_token_url?
+      throw new Error "@access_token_url is required"
+    if @scope? and not Array.isArray(@scope)
+      throw new Error "@scope has to be an array if present"
+
+    @_tok = null
+    @_created = 0
+    @_lock = new Lock()
+    @auth = @base.auth
+
+  #----------------
+
+  get : (cb) ->
+    await @_lock.acquire defer()
+    err = null
+    now = Math.floor(Date.now() / 1000)
+
+    if not (res = @_tok)? or (now - @_created > @auth.lifespan)
+
+      @base.log "+ Request for bearer token"
+
+      # Very crypto!  Not sure why this is done, but it's done
+      cred = (Buffer.from [ @auth.key, @auth.secret ].join(":")).toString('base64')
+
+      req = 'grant_type=client_credentials'
+      if @scope
+        req += "&scope=#{encodeURIComponent(@scope.join(' '))}"
+
+      opts =
+        url : @access_token_url
+        headers :
+          Authorization : "Basic #{cred}"
+          'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
+          'Content-Length': req.length
+        method : "post"
+        body : req
+      if @user_agent
+        opts.user_agent = @user_agent
+      await @base._get_url_body opts, defer err, rc, body
+
+      if err?
+        @base.logl 'error', "In getting bearer_token: #{err.message}"
+      else if (rc isnt v_codes.OK)
+        @base.logl 'error', "error in getting bearer token: #{rc}"
+        err = new Error "error: #{rc}"
+      else
+        try
+          body = JSON.parse body
+        catch e
+          @base.logl 'warn', "Could not parse JSON reply: #{e}"
+          err = e
+
+      if err? then # noop
+      else if not (tok = body.access_token)?
+        @base.logl 'warn', "No access token found in reply"
+        err = new Error "#{@name} error: no access token"
+      else
+        @_tok = tok
+        @_created = Math.floor(Date.now() / 1000)
+        res = @_tok
+
+      @base.log "- Request for bearer token for #{@name} -> err: #{err}"
+      unless err
+        @base.log "Bearer token for #{@name} is: #{res?.substr(0,5)}..."
+
+    @_lock.release()
+    cb err, res
+
+#==============================================================
+
 parse_int_or_undefined = (value) ->
   if (out = Number(value)) is parseInt(value) and !isNaN(out)
     return out

package/src/scrapers/dns.iced

@@ -97,13 +97,23 @@ exports.DnsScraper = class DnsScraper extends BaseScraper
 
   # ---------------------------------------------------------------------------
 
+  _resolve_txt : (domain, cb) ->
+    # We can use a DNS library passed in (in the case of native-dns running on
+    # the server)
+    dnslib = @libs.dns or dns
+    try
+      dnslib.resolveTxt domain, (err, records) ->
+        cb err, records
+    catch err
+      cb err
+
+  # ---------------------------------------------------------------------------
+
   # calls back with a v_code or null if it was ok
   _check_status: ({domain, proof_text_check}, cb) ->
     @log "+ DNS check for #{domain}"
 
-    # We can use a DNS library passed in (in the case of native-dns running on the server)
-    dnslib = @libs.dns or dns
-    await dnslib.resolveTxt domain, defer err, records
+    await @_resolve_txt domain, defer err, records
     rc = if err?
       @log "| DNS error: #{err}"
       v_codes.DNS_ERROR