keybase-proofs 2.3.31 → 2.3.32

package/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.3.32 (2026-05-28)
+
+- Fix Twitter scraper, allow returned URL to be either in x.com or twitter.com
+  domain.
+
 ## 2.3.31 (2025-12-18)
 
 - Set response size limit when making a request in GenericWebSiteScraper.

package/lib/scrapers/twitter.js

@@ -113,7 +113,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.hunt2"
           });
           _this._get_body_api({
@@ -222,7 +222,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.users_lookup"
                 });
                 _this._get_body_api({
@@ -265,7 +265,7 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.users_lookup"
                                   });
                                   setTimeout(__iced_deferrals.defer({
@@ -351,7 +351,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.get_follower_ids"
                 });
                 _this._get_body_api({
@@ -397,7 +397,7 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                                    filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.get_follower_ids"
                                   });
                                   setTimeout(__iced_deferrals.defer({
@@ -483,6 +483,21 @@
       return rc;
     };
 
+    TwitterScraper.prototype._comp_twitter_url = function(url_a_str, url_b_str) {
+      var twitter_hosts, url_a, url_b;
+      if (!url_a_str || !url_b_str) {
+        return false;
+      }
+      twitter_hosts = ['x.com', 'twitter.com'];
+      try {
+        url_a = new urlmod.URL(url_a_str.toLowerCase());
+        url_b = new urlmod.URL(url_b_str.toLowerCase());
+        return twitter_hosts.includes(url_a.host) && twitter_hosts.includes(url_b.host) && url_a.protocol === url_b.protocol && url_a.pathname === url_b.pathname && url_a.search === url_b.search && url_a.hash === url_b.hash;
+      } catch (_error) {
+        return false;
+      }
+    };
+
     TwitterScraper.prototype.check_status = function(_arg, cb) {
       var $, api_url, api_url_matches, author_url_matches, author_username, body_obj, err, new_api_url, proof_text_check, rc, remote_id, schm, tweet_id, tweet_p, u, username, username_from_url, _, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
@@ -503,7 +518,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.check_status"
           });
           _this._get_url_body({
@@ -517,7 +532,7 @@
                 return body_obj = arguments[2];
               };
             })(),
-            lineno: 288
+            lineno: 309
           }));
           __iced_deferrals._fulfill();
         });
@@ -532,16 +547,16 @@
             if (err = schm.check(body_obj)) {
               return cb(err, v_codes.CONTENT_FAILURE);
             }
-            if (!(sncmp(body_obj.url, api_url))) {
+            if (!(_this._comp_twitter_url(body_obj.url, api_url))) {
               err = new Error("returned url field doesn't match api_url (found: " + body_obj.url + ", expected: " + api_url + ")");
               return cb(err, v_codes.CONTENT_FAILURE);
             }
-            api_url_matches = api_url.match(new RegExp("^https://twitter\\.com/([^/]+)/status/(\\d+)(.*)$"));
+            api_url_matches = api_url.match(new RegExp("^https://(x|twitter)\\.com/([^/]+)/status/(\\d+)(.*)$"));
             if (!api_url_matches) {
               err = new Error("api_url field doesn't match regexp, got: " + api_url_matches);
               return cb(err, v_codes.CONTENT_FAILURE);
             }
-            _ = api_url_matches[0], username_from_url = api_url_matches[1], tweet_id = api_url_matches[2];
+            _ = api_url_matches[0], _ = api_url_matches[1], username_from_url = api_url_matches[2], tweet_id = api_url_matches[3];
             if (!(sncmp(username, username_from_url))) {
               err = new Error("username from api_url didn't match, expected: " + username + ", got: " + username_from_url);
               return cb(err, v_codes.BAD_USERNAME);
@@ -549,12 +564,12 @@
             if (tweet_id !== remote_id.toString()) {
               return cb(err, v_codes.BAD_REMOTE_ID);
             }
-            author_url_matches = body_obj.author_url.match(new RegExp("^https://twitter\\.com/(.+)$"));
+            author_url_matches = body_obj.author_url.match(new RegExp("^https://(x|twitter)\\.com/(.+)$"));
             if (!author_url_matches) {
               err = new Error("author_url doesn't match regexp, got: " + body_obj.author_url);
               return cb(err, v_codes.CONTENT_FAILURE);
             }
-            _ = author_url_matches[0], author_username = author_url_matches[1];
+            _ = author_url_matches[0], _ = author_url_matches[1], author_username = author_url_matches[2];
             if (!(sncmp(username, author_username))) {
               err = new Error("username from author_url didn't match, expected: " + username + ", got: " + author_username);
               return cb(err, v_codes.BAD_USERNAME);
@@ -590,7 +605,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_bearer_token"
           });
           bt.get(__iced_deferrals.defer({
@@ -600,7 +615,7 @@
                 return tok = arguments[1];
               };
             })(),
-            lineno: 349
+            lineno: 372
           }));
           __iced_deferrals._fulfill();
         });
@@ -622,7 +637,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_body_api"
           });
           _this._get_bearer_token(__iced_deferrals.defer({
@@ -633,7 +648,7 @@
                 return tok = arguments[2];
               };
             })(),
-            lineno: 358
+            lineno: 381
           }));
           __iced_deferrals._fulfill();
         });
@@ -655,7 +670,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter.iced",
+                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper._get_body_api"
                 });
                 _this._get_url_body(args, __iced_deferrals.defer({
@@ -666,7 +681,7 @@
                       return body = arguments[2];
                     };
                   })(),
-                  lineno: 369
+                  lineno: 392
                 }));
                 __iced_deferrals._fulfill();
               })(__iced_k);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keybase-proofs",
-  "version": "2.3.31",
+  "version": "2.3.32",
   "description": "Publicly-verifiable proofs of identity",
   "main": "lib/main.js",
   "scripts": {

package/src/scrapers/twitter.iced

@@ -262,6 +262,27 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
 
   # ---------------------------------------------------------------------------
 
+  _comp_twitter_url : (url_a_str, url_b_str) ->
+    # Do a smarter check for twitter URL but allowing hosts to be either
+    # x.com or twitter.com. Rest of the URL has to match (protocol, path,
+    # hash etc.)
+    if not url_a_str or not url_b_str
+      return false
+    twitter_hosts = ['x.com', 'twitter.com']
+    try
+      url_a = new urlmod.URL(url_a_str.toLowerCase())
+      url_b = new urlmod.URL(url_b_str.toLowerCase())
+      return (
+        twitter_hosts.includes(url_a.host) and
+        twitter_hosts.includes(url_b.host) and
+        url_a.protocol is url_b.protocol and
+        url_a.pathname is url_b.pathname and
+        url_a.search is url_b.search and
+        url_a.hash is url_b.hash
+        )
+    catch
+      return false
+
   check_status: ({username, api_url, proof_text_check, remote_id}, cb) ->
 
     if not api_url?.length
@@ -297,20 +318,22 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
       if err = schm.check body_obj
         return cb err, v_codes.CONTENT_FAILURE
 
-      # "url" field returned by the API should match api_url.
-      if not(sncmp(body_obj.url, api_url))
+      # "url" field returned by the API should match api_url, but twitter may
+      # return x.com domain in the "url" field. Our `api_url` is always
+      # twitter.com right now (for new and existing proofs).
+      if not(@_comp_twitter_url(body_obj.url, api_url))
         err = new Error "returned url field doesn't match api_url (found: #{body_obj.url}, expected: #{api_url})"
         return cb err, v_codes.CONTENT_FAILURE
 
       # Extract username from URL, do not use "author_name" because it's the
       # full name.
-      api_url_matches = api_url.match(new RegExp("^https://twitter\\.com/([^/]+)/status/(\\d+)(.*)$"))
+      api_url_matches = api_url.match(new RegExp("^https://(x|twitter)\\.com/([^/]+)/status/(\\d+)(.*)$"))
       if not api_url_matches
         err = new Error "api_url field doesn't match regexp, got: #{api_url_matches}"
         return cb err, v_codes.CONTENT_FAILURE
 
       # Check username and tweet ID.
-      [_, username_from_url, tweet_id] = api_url_matches
+      [_, _, username_from_url, tweet_id] = api_url_matches
       if not(sncmp(username, username_from_url))
         err = new Error("username from api_url didn't match, expected: #{username}, got: #{username_from_url}")
         return cb err, v_codes.BAD_USERNAME
@@ -319,12 +342,12 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
 
       # Check "author_url" field, it contains a link to user's twitter profile.
       # It should match our username.
-      author_url_matches = body_obj.author_url.match(new RegExp("^https://twitter\\.com/(.+)$"))
+      author_url_matches = body_obj.author_url.match(new RegExp("^https://(x|twitter)\\.com/(.+)$"))
       if not author_url_matches
         err = new Error("author_url doesn't match regexp, got: #{body_obj.author_url}")
         return cb err, v_codes.CONTENT_FAILURE
 
-      [_, author_username] = author_url_matches
+      [_, _, author_username] = author_url_matches
       if not(sncmp(username, author_username))
         err = new Error("username from author_url didn't match, expected: #{username}, got: #{author_username}")
         return cb err, v_codes.BAD_USERNAME