keybase-proofs 2.3.30 → 2.3.32

package/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 2.3.32 (2026-05-28)
+
+- Fix Twitter scraper, allow returned URL to be either in x.com or twitter.com
+  domain.
+
+## 2.3.31 (2025-12-18)
+
+- Set response size limit when making a request in GenericWebSiteScraper.
+
 ## 2.3.30 (2025-02-28)
 
 - Fix BaseScraper not properly handling promise rejections when fetching text

package/lib/constants.js

@@ -132,6 +132,7 @@
     },
     expire_in: 60 * 60 * 24 * 365 * 5,
     http_timeout: 15 * 1000,
+    generic_website_max_size: 128000,
     short_id_bytes: 27,
     shortest_pgp_signature: 100,
     critical_clock_skew_secs: 65 * 60,

package/lib/scrapers/generic_web_site.js

@@ -58,19 +58,21 @@
     };
 
     GenericWebSiteScraper.prototype._check_url = function(_arg, cb) {
-      var err, proof_text_check, raw, rc, url, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      var err, proof_text_check, raw, rc, size, url, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
       url = _arg.url, proof_text_check = _arg.proof_text_check;
+      size = constants.generic_website_max_size;
       (function(_this) {
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/max/src/keybase/proofs/src/scrapers/generic_web_site.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/generic_web_site.iced",
             funcname: "GenericWebSiteScraper._check_url"
           });
           _this._get_url_body({
-            url: url
+            url: url,
+            size: size
           }, __iced_deferrals.defer({
             assign_fn: (function() {
               return function() {
@@ -79,7 +81,7 @@
                 return raw = arguments[2];
               };
             })(),
-            lineno: 48
+            lineno: 50
           }));
           __iced_deferrals._fulfill();
         });
@@ -132,7 +134,7 @@
                   (function(__iced_k) {
                     __iced_deferrals = new iced.Deferrals(__iced_k, {
                       parent: ___iced_passed_deferral,
-                      filename: "/Users/max/src/keybase/proofs/src/scrapers/generic_web_site.iced",
+                      filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/generic_web_site.iced",
                       funcname: "GenericWebSiteScraper.hunt2"
                     });
                     _this._check_url({
@@ -145,7 +147,7 @@
                           return rc = arguments[1];
                         };
                       })(),
-                      lineno: 66
+                      lineno: 68
                     }));
                     __iced_deferrals._fulfill();
                   })(function() {
@@ -215,7 +217,7 @@ _break()
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/max/src/keybase/proofs/src/scrapers/generic_web_site.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/generic_web_site.iced",
             funcname: "GenericWebSiteScraper.check_status"
           });
           _this._check_url({
@@ -228,7 +230,7 @@ _break()
                 return rc = arguments[1];
               };
             })(),
-            lineno: 95
+            lineno: 97
           }));
           __iced_deferrals._fulfill();
         });

package/lib/scrapers/twitter.js

@@ -113,7 +113,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.hunt2"
           });
           _this._get_body_api({
@@ -222,7 +222,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.users_lookup"
                 });
                 _this._get_body_api({
@@ -265,7 +265,7 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.users_lookup"
                                   });
                                   setTimeout(__iced_deferrals.defer({
@@ -351,7 +351,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.get_follower_ids"
                 });
                 _this._get_body_api({
@@ -397,7 +397,7 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.get_follower_ids"
                                   });
                                   setTimeout(__iced_deferrals.defer({
@@ -483,6 +483,21 @@
       return rc;
     };
 
+    TwitterScraper.prototype._comp_twitter_url = function(url_a_str, url_b_str) {
+      var twitter_hosts, url_a, url_b;
+      if (!url_a_str || !url_b_str) {
+        return false;
+      }
+      twitter_hosts = ['x.com', 'twitter.com'];
+      try {
+        url_a = new urlmod.URL(url_a_str.toLowerCase());
+        url_b = new urlmod.URL(url_b_str.toLowerCase());
+        return twitter_hosts.includes(url_a.host) && twitter_hosts.includes(url_b.host) && url_a.protocol === url_b.protocol && url_a.pathname === url_b.pathname && url_a.search === url_b.search && url_a.hash === url_b.hash;
+      } catch (_error) {
+        return false;
+      }
+    };
+
     TwitterScraper.prototype.check_status = function(_arg, cb) {
       var $, api_url, api_url_matches, author_url_matches, author_username, body_obj, err, new_api_url, proof_text_check, rc, remote_id, schm, tweet_id, tweet_p, u, username, username_from_url, _, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
@@ -503,7 +518,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.check_status"
           });
           _this._get_url_body({
@@ -517,7 +532,7 @@
                 return body_obj = arguments[2];
               };
             })(),
-            lineno: 288
+            lineno: 309
           }));
           __iced_deferrals._fulfill();
         });
@@ -532,16 +547,16 @@
             if (err = schm.check(body_obj)) {
               return cb(err, v_codes.CONTENT_FAILURE);
             }
-            if (!(sncmp(body_obj.url, api_url))) {
+            if (!(_this._comp_twitter_url(body_obj.url, api_url))) {
               err = new Error("returned url field doesn't match api_url (found: " + body_obj.url + ", expected: " + api_url + ")");
               return cb(err, v_codes.CONTENT_FAILURE);
             }
-            api_url_matches = api_url.match(new RegExp("^https://twitter\\.com/([^/]+)/status/(\\d+)(.*)$"));
+            api_url_matches = api_url.match(new RegExp("^https://(x|twitter)\\.com/([^/]+)/status/(\\d+)(.*)$"));
             if (!api_url_matches) {
               err = new Error("api_url field doesn't match regexp, got: " + api_url_matches);
               return cb(err, v_codes.CONTENT_FAILURE);
             }
-            _ = api_url_matches[0], username_from_url = api_url_matches[1], tweet_id = api_url_matches[2];
+            _ = api_url_matches[0], _ = api_url_matches[1], username_from_url = api_url_matches[2], tweet_id = api_url_matches[3];
             if (!(sncmp(username, username_from_url))) {
               err = new Error("username from api_url didn't match, expected: " + username + ", got: " + username_from_url);
               return cb(err, v_codes.BAD_USERNAME);
@@ -549,12 +564,12 @@
             if (tweet_id !== remote_id.toString()) {
               return cb(err, v_codes.BAD_REMOTE_ID);
             }
-            author_url_matches = body_obj.author_url.match(new RegExp("^https://twitter\\.com/(.+)$"));
+            author_url_matches = body_obj.author_url.match(new RegExp("^https://(x|twitter)\\.com/(.+)$"));
             if (!author_url_matches) {
               err = new Error("author_url doesn't match regexp, got: " + body_obj.author_url);
               return cb(err, v_codes.CONTENT_FAILURE);
             }
-            _ = author_url_matches[0], author_username = author_url_matches[1];
+            _ = author_url_matches[0], _ = author_url_matches[1], author_username = author_url_matches[2];
             if (!(sncmp(username, author_username))) {
               err = new Error("username from author_url didn't match, expected: " + username + ", got: " + author_username);
               return cb(err, v_codes.BAD_USERNAME);
@@ -590,7 +605,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_bearer_token"
           });
           bt.get(__iced_deferrals.defer({
@@ -600,7 +615,7 @@
                 return tok = arguments[1];
               };
             })(),
-            lineno: 349
+            lineno: 372
           }));
           __iced_deferrals._fulfill();
         });
@@ -622,7 +637,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_body_api"
           });
           _this._get_bearer_token(__iced_deferrals.defer({
@@ -633,7 +648,7 @@
                 return tok = arguments[2];
               };
             })(),
-            lineno: 358
+            lineno: 381
           }));
           __iced_deferrals._fulfill();
         });
@@ -655,7 +670,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper._get_body_api"
                 });
                 _this._get_url_body(args, __iced_deferrals.defer({
@@ -666,7 +681,7 @@
                       return body = arguments[2];
                     };
                   })(),
-                  lineno: 369
+                  lineno: 392
                 }));
                 __iced_deferrals._fulfill();
               })(__iced_k);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keybase-proofs",
-  "version": "2.3.30",
+  "version": "2.3.32",
   "description": "Publicly-verifiable proofs of identity",
   "main": "lib/main.js",
   "scripts": {

package/src/constants.iced

@@ -115,8 +115,10 @@ exports.constants = constants =
     generic_web_site : 1000
     dns              : 1001
     pgp              : 1002
+
   expire_in : 60*60*24*365*5 # 5 years....
   http_timeout : 15*1000 # give up after 15 seconds....
+  generic_website_max_size : 128000 # do not download more than 128KB from generic websites
   short_id_bytes : 27
   shortest_pgp_signature : 100 # can't have a PGP signature shorter than this...
   critical_clock_skew_secs : 65*60 # if you clock is ~1 hour off, we won't accept the sig

package/src/scrapers/generic_web_site.iced

@@ -45,8 +45,10 @@ exports.GenericWebSiteScraper = class GenericWebSiteScraper extends BaseScraper
   # ---------------------------------------------------------------------------
 
   _check_url : ( { url, proof_text_check }, cb) ->
+    # limit how much data we want to receive from arbitrary websites
+    size = constants.generic_website_max_size
     # calls back with a v_code or null if it was ok
-    await @_get_url_body {url }, defer err, rc, raw
+    await @_get_url_body { url, size }, defer err, rc, raw
     rc = if rc isnt v_codes.OK                             then rc
     else if @_find_sig_in_raw(proof_text_check, raw)       then v_codes.OK
     else                                                   v_codes.NOT_FOUND

package/src/scrapers/twitter.iced

@@ -262,6 +262,27 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
 
   # ---------------------------------------------------------------------------
 
+  _comp_twitter_url : (url_a_str, url_b_str) ->
+    # Do a smarter check for twitter URL but allowing hosts to be either
+    # x.com or twitter.com. Rest of the URL has to match (protocol, path,
+    # hash etc.)
+    if not url_a_str or not url_b_str
+      return false
+    twitter_hosts = ['x.com', 'twitter.com']
+    try
+      url_a = new urlmod.URL(url_a_str.toLowerCase())
+      url_b = new urlmod.URL(url_b_str.toLowerCase())
+      return (
+        twitter_hosts.includes(url_a.host) and
+        twitter_hosts.includes(url_b.host) and
+        url_a.protocol is url_b.protocol and
+        url_a.pathname is url_b.pathname and
+        url_a.search is url_b.search and
+        url_a.hash is url_b.hash
+        )
+    catch
+      return false
+
   check_status: ({username, api_url, proof_text_check, remote_id}, cb) ->
 
     if not api_url?.length
@@ -297,20 +318,22 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
       if err = schm.check body_obj
         return cb err, v_codes.CONTENT_FAILURE
 
-      # "url" field returned by the API should match api_url.
-      if not(sncmp(body_obj.url, api_url))
+      # "url" field returned by the API should match api_url, but twitter may
+      # return x.com domain in the "url" field. Our `api_url` is always
+      # twitter.com right now (for new and existing proofs).
+      if not(@_comp_twitter_url(body_obj.url, api_url))
         err = new Error "returned url field doesn't match api_url (found: #{body_obj.url}, expected: #{api_url})"
         return cb err, v_codes.CONTENT_FAILURE
 
       # Extract username from URL, do not use "author_name" because it's the
       # full name.
-      api_url_matches = api_url.match(new RegExp("^https://twitter\\.com/([^/]+)/status/(\\d+)(.*)$"))
+      api_url_matches = api_url.match(new RegExp("^https://(x|twitter)\\.com/([^/]+)/status/(\\d+)(.*)$"))
       if not api_url_matches
         err = new Error "api_url field doesn't match regexp, got: #{api_url_matches}"
         return cb err, v_codes.CONTENT_FAILURE
 
       # Check username and tweet ID.
-      [_, username_from_url, tweet_id] = api_url_matches
+      [_, _, username_from_url, tweet_id] = api_url_matches
       if not(sncmp(username, username_from_url))
         err = new Error("username from api_url didn't match, expected: #{username}, got: #{username_from_url}")
         return cb err, v_codes.BAD_USERNAME
@@ -319,12 +342,12 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
 
       # Check "author_url" field, it contains a link to user's twitter profile.
       # It should match our username.
-      author_url_matches = body_obj.author_url.match(new RegExp("^https://twitter\\.com/(.+)$"))
+      author_url_matches = body_obj.author_url.match(new RegExp("^https://(x|twitter)\\.com/(.+)$"))
       if not author_url_matches
         err = new Error("author_url doesn't match regexp, got: #{body_obj.author_url}")
         return cb err, v_codes.CONTENT_FAILURE
 
-      [_, author_username] = author_url_matches
+      [_, _, author_username] = author_url_matches
       if not(sncmp(username, author_username))
         err = new Error("username from author_url didn't match, expected: #{username}, got: #{author_username}")
         return cb err, v_codes.BAD_USERNAME

package/test/make_esc.iced

@@ -0,0 +1,48 @@
+C = require('iced-runtime').const
+
+exports.skip = true
+
+ipush = (e, msg) ->
+  if msg?
+    # Note: `e.istack = []` will be a noop if e is a primitive type
+    # like number or string. We have to check if `e.istack` exists
+    # before pushing so we don't crash.
+    e.istack = [] unless e.istack?
+    e.istack?.push msg
+
+copy_trace = (src, dst) ->
+  dst[C.trace] = src[C.trace]
+  dst
+
+# Error short-circuit connector with more verbose istack.
+
+# When resulting `esc` function is called, an istack entry will be constructed
+# based on iced trace object. This way istack will be populated by not only esc
+# identifiers (usually call site function names) but also filenames and line
+# numbers of call sites of individual esc() calls. If an error happens and
+# short-circuited back from deep in some test, it should be easier to identify
+# where exactly it originates from using the new istack.
+
+exports.make_esc = make_esc = (gcb, where) ->
+  # Create "global where" based on make_esc() call site.
+  where = make_esc.caller?.name unless where?
+  return (lcb) ->
+    # Create "local where" based on esc() call site. So every
+    # esc(defer(...)) will have its own istack entry, if iced
+    # trace object is available.
+    sb = [ where ]
+    if (tr = lcb[C.trace])
+      sb.push "Trace:"
+      sb.push tr[C.funcname]
+      sb.push "(#{tr[C.filename]}:#{tr[C.lineno] + 1})"
+    local_where = sb.filter((x) -> x).join(' ')
+    copy_trace lcb, (err, args...) ->
+      if not err? then lcb args...
+      else if not gcb.__esc
+        gcb.__esc = true
+        ipush err, local_where
+        gcb err
+
+exports.patch_make_esc = () ->
+  iced_error = require 'iced-error'
+  iced_error.make_esc = make_esc