keybase-proofs 2.3.23 → 2.3.25

package/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.3.24 (2020-12-23)
+
+- Use oembed API for twitter scraping.
+
 ## 2.3.23 (2020-03-10)
 
 - update WOT schema again

package/lib/constants.js

@@ -1,4 +1,4 @@
-// Generated by IcedCoffeeScript 108.0.11
+// Generated by IcedCoffeeScript 108.0.12
 (function() {
   var constants, d, k, v, _ref;
 

package/lib/schema3.js

@@ -103,6 +103,7 @@
       var keys;
       keys = _arg.keys;
       this._keys = keys;
+      this._allow_extra_keys = false;
       Dict.__super__.constructor.apply(this, arguments);
     }
 
@@ -116,6 +117,9 @@
         v = obj[k];
         new_path = path.extend(k);
         if ((checker = this._keys[k]) == null) {
+          if (this._allow_extra_keys) {
+            continue;
+          }
           return mkerr(new_path, "key is not supported");
         }
         if ((err = this._check_value({
@@ -154,6 +158,11 @@
       return this._keys[k] = v;
     };
 
+    Dict.prototype.allow_extra_keys = function() {
+      this._allow_extra_keys = true;
+      return this;
+    };
+
     return Dict;
 
   })(Node);

package/lib/scrapers/base.js

@@ -1,6 +1,6 @@
-// Generated by IcedCoffeeScript 108.0.11
+// Generated by IcedCoffeeScript 108.0.12
 (function() {
-  var BaseScraper, b64find, constants, decode_sig, iced, parse_int_or_undefined, pkg, sncmp, space_normalize, urlmod, user_agent, v_codes, __iced_k, __iced_k_noop;
+  var BaseScraper, b64find, callbackify, constants, decode_sig, iced, parse_int_or_undefined, pkg, sncmp, space_normalize, urlmod, user_agent, v_codes, __iced_k, __iced_k_noop;
 
   iced = require('iced-runtime');
   __iced_k = __iced_k_noop = function() {};
@@ -19,6 +19,8 @@
 
   urlmod = require('url');
 
+  callbackify = require('util').callbackify;
+
   exports.user_agent = user_agent = constants.user_agent + pkg.version;
 
   BaseScraper = (function() {
@@ -92,7 +94,7 @@
             (function(__iced_k) {
               __iced_deferrals = new iced.Deferrals(__iced_k, {
                 parent: ___iced_passed_deferral,
-                filename: "/Users/joshblum/go/src/github.com/keybase/proofs/src/scrapers/base.iced",
+                filename: "/home/zapu/Projects/KB/proofs/src/scrapers/base.iced",
                 funcname: "BaseScraper.validate"
               });
               _this.check_status(args, __iced_deferrals.defer({
@@ -102,7 +104,7 @@
                     return rc = arguments[1];
                   };
                 })(),
-                lineno: 52
+                lineno: 53
               }));
               __iced_deferrals._fulfill();
             })(__iced_k);
@@ -140,7 +142,7 @@
     };
 
     BaseScraper.prototype._get_url_body = function(opts, cb) {
-      var err, rc, response, rl_limit, rl_remaining, rl_reset, ___iced_passed_deferral, __iced_deferrals, __iced_k, _base;
+      var body, err, f, rc, response, rl_limit, rl_remaining, rl_reset, ___iced_passed_deferral, __iced_deferrals, __iced_k, _base;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
 
@@ -158,7 +160,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/joshblum/go/src/github.com/keybase/proofs/src/scrapers/base.iced",
+            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/base.iced",
             funcname: "BaseScraper._get_url_body"
           });
           _this.libs.fetch(opts.url, opts, __iced_deferrals.defer({
@@ -168,7 +170,7 @@
                 return response = arguments[1];
               };
             })(),
-            lineno: 86
+            lineno: 87
           }));
           __iced_deferrals._fulfill();
         });
@@ -192,22 +194,45 @@
           }
           rc = typeof err !== "undefined" && err !== null ? err.message.includes('network timeout') ? v_codes.TIMEOUT : v_codes.HOST_UNREACHABLE : ((_ref = response.status) === 401 || _ref === 403) ? v_codes.PERMISSION_DENIED : response.status === 200 ? v_codes.OK : response.status >= 500 ? v_codes.HTTP_500 : response.status >= 400 ? v_codes.HTTP_400 : response.status >= 300 ? v_codes.HTTP_300 : v_codes.HTTP_OTHER;
           if (rc === v_codes.OK) {
-            try {
-              if (opts.json) {
-                return response.json().then(function(body) {
-                  return cb(err, rc, body);
-                });
-              } else {
-                return response.text().then(function(body) {
-                  return cb(err, rc, body);
-                });
+            (function(__iced_k) {
+              try {
+                if (opts.json) {
+                  f = callbackify(response.json);
+                  (function(__iced_k) {
+                    __iced_deferrals = new iced.Deferrals(__iced_k, {
+                      parent: ___iced_passed_deferral,
+                      filename: "/home/zapu/Projects/KB/proofs/src/scrapers/base.iced",
+                      funcname: "BaseScraper._get_url_body"
+                    });
+                    f.call(response, __iced_deferrals.defer({
+                      assign_fn: (function() {
+                        return function() {
+                          err = arguments[0];
+                          return body = arguments[1];
+                        };
+                      })(),
+                      lineno: 109
+                    }));
+                    __iced_deferrals._fulfill();
+                  })(function() {
+                    if (err) {
+                      _this.log("| _get_url_body response.json() failed with: " + (err.toString()));
+                      rc = v_codes.CONTENT_FAILURE;
+                    }
+                    return __iced_k(cb(err, rc, body));
+                  });
+                } else {
+                  return __iced_k(response.text().then(function(body) {
+                    return cb(err, rc, body);
+                  }));
+                }
+              } catch (_error) {
+                err = _error;
+                return cb(err, rc, null);
               }
-            } catch (_error) {
-              err = _error;
-              return cb(err, rc, null);
-            }
+            })(__iced_k);
           } else {
-            return cb(err, rc, null);
+            return __iced_k(cb(err, rc, null));
           }
         };
       })(this));

package/lib/scrapers/generic_social.js

@@ -74,7 +74,7 @@
         return [err, rc];
       }
       found = obj.find(function(x) {
-        return sncmp(x.kb_username, kb_username) && sncmp(x.sig_hash, sig_id);
+        return x && sncmp(x.kb_username, kb_username) && sncmp(x.sig_hash, sig_id);
       });
       if (found != null) {
         rc = v_codes.OK;
@@ -100,7 +100,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/max/src/keybase/proofs/src/scrapers/generic_social.iced",
+            filename: "/home/sy/go/src/github.com/keybase/proofs/src/scrapers/generic_social.iced",
             funcname: "GenericSocialScraper.check_status"
           });
           _this._get_url_body({

package/lib/scrapers/twitter.js

@@ -1,6 +1,6 @@
-// Generated by IcedCoffeeScript 108.0.11
+// Generated by IcedCoffeeScript 108.0.12
 (function() {
-  var BaseScraper, BearerToken, Lock, TwitterScraper, bearer_token, constants, decode_sig, iced, make_ids, sncmp, urlmod, v_codes, ws_normalize, __iced_k, __iced_k_noop, _bearer_token, _ref,
+  var BaseScraper, BearerToken, Lock, TwitterScraper, bearer_token, constants, decode_sig, iced, make_ids, schema, sncmp, urlmod, v_codes, ws_normalize, __iced_k, __iced_k_noop, _bearer_token, _ref,
     __hasProp = {}.hasOwnProperty,
     __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
 
@@ -21,6 +21,8 @@
 
   urlmod = require('url');
 
+  schema = require('../schema3');
+
   ws_normalize = function(x) {
     var v;
     v = x.split(/[\t\r\n ]+/);
@@ -50,11 +52,11 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
             funcname: "BearerToken.get"
           });
           _this._lock.acquire(__iced_deferrals.defer({
-            lineno: 31
+            lineno: 32
           }));
           __iced_deferrals._fulfill();
         });
@@ -80,7 +82,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
                   funcname: "BearerToken.get"
                 });
                 _this.base._get_url_body(opts, __iced_deferrals.defer({
@@ -91,7 +93,7 @@
                       return body = arguments[2];
                     };
                   })(),
-                  lineno: 51
+                  lineno: 52
                 }));
                 __iced_deferrals._fulfill();
               })(function() {
@@ -193,7 +195,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.hunt2"
           });
           _this._get_body_api({
@@ -206,7 +208,7 @@
                 return json = arguments[2];
               };
             })(),
-            lineno: 123
+            lineno: 124
           }));
           __iced_deferrals._fulfill();
         });
@@ -293,7 +295,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.users_lookup"
                 });
                 _this._get_body_api({
@@ -306,7 +308,7 @@
                       return json = arguments[2];
                     };
                   })(),
-                  lineno: 171
+                  lineno: 172
                 }));
                 __iced_deferrals._fulfill();
               })(function() {
@@ -336,11 +338,11 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+                                    filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.users_lookup"
                                   });
                                   setTimeout(__iced_deferrals.defer({
-                                    lineno: 185
+                                    lineno: 186
                                   }), cursor_wait);
                                   __iced_deferrals._fulfill();
                                 })(__iced_k);
@@ -422,7 +424,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper.get_follower_ids"
                 });
                 _this._get_body_api({
@@ -435,7 +437,7 @@
                       return json = arguments[2];
                     };
                   })(),
-                  lineno: 223
+                  lineno: 224
                 }));
                 __iced_deferrals._fulfill();
               })(function() {
@@ -468,11 +470,11 @@
                                 (function(__iced_k) {
                                   __iced_deferrals = new iced.Deferrals(__iced_k, {
                                     parent: ___iced_passed_deferral,
-                                    filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+                                    filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
                                     funcname: "TwitterScraper.get_follower_ids"
                                   });
                                   setTimeout(__iced_deferrals.defer({
-                                    lineno: 239
+                                    lineno: 240
                                   }), cursor_wait);
                                   __iced_deferrals._fulfill();
                                 })(__iced_k);
@@ -555,7 +557,7 @@
     };
 
     TwitterScraper.prototype.check_status = function(_arg, cb) {
-      var $, api_url, div, err, found_tweet_id, found_username, html, new_api_url, proof_text_check, rc, remote_id, u, username, ___iced_passed_deferral, __iced_deferrals, __iced_k;
+      var $, api_url, api_url_matches, author_url_matches, author_username, body_obj, err, new_api_url, proof_text_check, rc, remote_id, schm, tweet_id, tweet_p, u, username, username_from_url, _, ___iced_passed_deferral, __iced_deferrals, __iced_k;
       __iced_k = __iced_k_noop;
       ___iced_passed_deferral = iced.findDeferral(arguments);
       username = _arg.username, api_url = _arg.api_url, proof_text_check = _arg.proof_text_check, remote_id = _arg.remote_id;
@@ -565,48 +567,85 @@
         this.log("null api_url for " + remote_id + "/" + username);
         return cb(err, rc);
       }
-      u = urlmod.parse(api_url);
-      u.host = u.hostname = 'mobile.twitter.com';
+      u = new urlmod.URL('https://api.twitter.com/1/statuses/oembed.json');
+      u.searchParams.set('id', remote_id.toString());
+      u.searchParams.set('omit_script', '1');
       new_api_url = urlmod.format(u);
-      this.log("| rewrite " + api_url + " -> " + new_api_url + " (since twitter was going to redirect us anyways)");
-      api_url = new_api_url;
+      this.log("| use oembed API " + new_api_url + " for tweet at " + api_url + " (remote_id=" + remote_id + ")");
       (function(_this) {
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper.check_status"
           });
           _this._get_url_body({
-            url: api_url
+            url: new_api_url,
+            json: true
           }, __iced_deferrals.defer({
             assign_fn: (function() {
               return function() {
                 err = arguments[0];
                 rc = arguments[1];
-                return html = arguments[2];
+                return body_obj = arguments[2];
               };
             })(),
-            lineno: 321
+            lineno: 331
           }));
           __iced_deferrals._fulfill();
         });
       })(this)((function(_this) {
         return function() {
-          var _ref1;
-          if (rc === v_codes.OK) {
-            $ = _this.libs.cheerio.load(html);
-            div = $('.main-tweet .tweet-text');
-            if (div.length !== 1) {
-              rc = v_codes.FAILED_PARSE;
-            } else {
-              found_username = (_ref1 = $('.main-tweet .username').text()) != null ? _ref1.trim().replace(/^@/g, '') : void 0;
-              found_tweet_id = "" + (div.data('id'));
-              rc = !(sncmp(username, found_username)) ? v_codes.BAD_USERNAME : ("" + remote_id) !== found_tweet_id ? v_codes.BAD_REMOTE_ID : _this.find_sig_in_tweet({
-                tweet_p: div,
-                proof_text_check: proof_text_check
-              });
+          if (!err && rc === v_codes.OK) {
+            schm = schema.dict({
+              url: schema.string().name('url'),
+              author_url: schema.string().name('author_url'),
+              html: schema.string().name('html')
+            }).allow_extra_keys();
+            if (err = schm.check(body_obj)) {
+              return cb(err, v_codes.CONTENT_FAILURE);
+            }
+            if (!(sncmp(body_obj.url, api_url))) {
+              err = new Error("returned url field doesn't match api_url (found: " + body_obj.url + ", expected: " + api_url + ")");
+              return cb(err, v_codes.CONTENT_FAILURE);
+            }
+            api_url_matches = api_url.match(new RegExp("^https://twitter\\.com/([^/]+)/status/(\\d+)(.*)$"));
+            if (!api_url_matches) {
+              err = new Error("api_url field doesn't match regexp, got: " + api_url_matches);
+              return cb(err, v_codes.CONTENT_FAILURE);
+            }
+            _ = api_url_matches[0], username_from_url = api_url_matches[1], tweet_id = api_url_matches[2];
+            if (!(sncmp(username, username_from_url))) {
+              err = new Error("username from api_url didn't match, expected: " + username + ", got: " + username_from_url);
+              return cb(err, v_codes.BAD_USERNAME);
+            }
+            if (tweet_id !== remote_id.toString()) {
+              return cb(err, v_codes.BAD_REMOTE_ID);
+            }
+            author_url_matches = body_obj.author_url.match(new RegExp("^https://twitter\\.com/(.+)$"));
+            if (!author_url_matches) {
+              err = new Error("author_url doesn't match regexp, got: " + body_obj.author_url);
+              return cb(err, v_codes.CONTENT_FAILURE);
+            }
+            _ = author_url_matches[0], author_username = author_url_matches[1];
+            if (!(sncmp(username, author_username))) {
+              err = new Error("username from author_url didn't match, expected: " + username + ", got: " + author_username);
+              return cb(err, v_codes.BAD_USERNAME);
+            }
+            if (body_obj.html.length > 1000) {
+              err = new Error("html is " + body_obj.html.length + " characters, not trying to parse it");
+              return cb(err, v_codes.CONTENT_FAILURE);
+            }
+            $ = _this.libs.cheerio.load(body_obj.html);
+            tweet_p = $('blockquote.twitter-tweet p');
+            if (tweet_p.length !== 1) {
+              err = new Error("failed to find tweet <p> in returned 'html' field");
+              return cb(err, v_codes.FAILED_PARSE);
             }
+            rc = _this.find_sig_in_tweet({
+              tweet_p: tweet_p,
+              proof_text_check: proof_text_check
+            });
           }
           return cb(err, rc);
         };
@@ -624,7 +663,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_bearer_token"
           });
           bt.get(__iced_deferrals.defer({
@@ -634,7 +673,7 @@
                 return tok = arguments[1];
               };
             })(),
-            lineno: 350
+            lineno: 392
           }));
           __iced_deferrals._fulfill();
         });
@@ -656,7 +695,7 @@
         return (function(__iced_k) {
           __iced_deferrals = new iced.Deferrals(__iced_k, {
             parent: ___iced_passed_deferral,
-            filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+            filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
             funcname: "TwitterScraper._get_body_api"
           });
           _this._get_bearer_token(__iced_deferrals.defer({
@@ -667,7 +706,7 @@
                 return tok = arguments[2];
               };
             })(),
-            lineno: 359
+            lineno: 401
           }));
           __iced_deferrals._fulfill();
         });
@@ -687,7 +726,7 @@
               (function(__iced_k) {
                 __iced_deferrals = new iced.Deferrals(__iced_k, {
                   parent: ___iced_passed_deferral,
-                  filename: "/Users/miles/go/src/github.com/keybase/proofs/src/scrapers/twitter.iced",
+                  filename: "/home/zapu/Projects/KB/proofs/src/scrapers/twitter.iced",
                   funcname: "TwitterScraper._get_body_api"
                 });
                 _this._get_url_body(args, __iced_deferrals.defer({
@@ -698,7 +737,7 @@
                       return body = arguments[2];
                     };
                   })(),
-                  lineno: 368
+                  lineno: 410
                 }));
                 __iced_deferrals._fulfill();
               })(__iced_k);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keybase-proofs",
-  "version": "2.3.23",
+  "version": "2.3.25",
   "description": "Publicly-verifiable proofs of identity",
   "main": "lib/main.js",
   "scripts": {
@@ -36,4 +36,4 @@
     "iced-coffee-script": "^108.0.11",
     "iced-test": "0.0.22"
   }
-}
+}

package/src/schema3.iced

@@ -39,6 +39,8 @@ class Node
 class Dict extends Node
   constructor : ({keys}) ->
     @_keys = keys
+    # do not fail if there are extra keys unknown to schema
+    @_allow_extra_keys = false
     super
 
   _check : ({path, obj}) ->
@@ -46,7 +48,9 @@ class Dict extends Node
       return mkerr path, "need a dictionary"
     for k,v of obj
       new_path = path.extend(k)
-      if not (checker = @_keys[k])? then return mkerr new_path, "key is not supported"
+      if not (checker = @_keys[k])?
+        if @_allow_extra_keys then continue
+        return mkerr new_path, "key is not supported"
       if (err = @_check_value { checker, path : new_path, obj : v }) then return err
     for k,v of @_keys
       new_path = path.extend(k)
@@ -62,6 +66,10 @@ class Dict extends Node
   set_key : (k,v) ->
     @_keys[k] = v
 
+  allow_extra_keys : () ->
+    @_allow_extra_keys = true
+    @
+
 class Array extends Node
 
   constructor : ({elem}) ->

package/src/scrapers/base.iced

@@ -5,6 +5,7 @@ pkg = require '../../package.json'
 {space_normalize} = require '../util'
 {b64find} = require '../b64extract'
 urlmod = require 'url'
+{callbackify} = require 'util'
 
 #==============================================================
 
@@ -105,8 +106,12 @@ class BaseScraper
     if rc is v_codes.OK
       try
         if opts.json
-          response.json().then (body) ->
-            cb err, rc, body
+          f = callbackify response.json
+          await f.call response, defer err, body
+          if err
+            @log "| _get_url_body response.json() failed with: #{err.toString()}"
+            rc = v_codes.CONTENT_FAILURE
+          cb err, rc, body
         else
           response.text().then (body) ->
             cb err, rc, body

package/src/scrapers/generic_social.iced

@@ -52,7 +52,7 @@ exports.GenericSocialScraper = class GenericSocialScraper extends BaseScraper
     unless obj? and Array.isArray obj
       err = new Error "did not find proof list on #{check_path.join('.')} in json data"
       return [err, rc]
-    found = obj.find (x) -> sncmp(x.kb_username, kb_username) and sncmp(x.sig_hash, sig_id)
+    found = obj.find (x) -> x and sncmp(x.kb_username, kb_username) and sncmp(x.sig_hash, sig_id)
     if found?
       rc = v_codes.OK
     else

package/src/scrapers/twitter.iced

@@ -5,6 +5,7 @@
 {v_codes} = constants
 {decode_sig} = require('kbpgp').ukm
 urlmod = require 'url'
+schema = require '../schema3'
 
 #================================================================================
 
@@ -312,35 +313,76 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
       @log "null api_url for #{remote_id}/#{username}"
       return cb err, rc
 
-    u = urlmod.parse api_url
-    u.host = u.hostname = 'mobile.twitter.com'
+    # Do not try to fetch api_url which is of form:
+    # "https://twitter.com/tacovontaco/status/673931888088625152"
+    # because Twitter serves a JavaScript-based page there and the contents of
+    # tweet are not available. Instead, construct a new url to oembed API with
+    # the tweet ID. That API doesn't require authentication and claims not to
+    # be rate limited:
+    # https://developer.twitter.com/en/docs/twitter-api/v1/tweets/post-and-engage/api-reference/get-statuses-oembed
+    u = new urlmod.URL('https://api.twitter.com/1/statuses/oembed.json')
+    u.searchParams.set('id', remote_id.toString())
+    # tell twitter not to include their <script> tag in result.
+    u.searchParams.set('omit_script', '1')
+
     new_api_url = urlmod.format u
-    @log "| rewrite #{api_url} -> #{new_api_url} (since twitter was going to redirect us anyways)"
-    api_url = new_api_url
+    @log "| use oembed API #{new_api_url} for tweet at #{api_url} (remote_id=#{remote_id})"
 
     # calls back with a v_code or null if it was ok
-    await @_get_url_body { url : api_url }, defer err, rc, html
-
-    if rc is v_codes.OK
-
-      $ = @libs.cheerio.load html
-      #
-      # only look inside the permalink tweet container
-      #
-      div = $('.main-tweet .tweet-text')
-      if div.length isnt 1
-        rc = v_codes.FAILED_PARSE
-      else
-        #
-        # make sure both the username and tweet id match our query,
-        # in case twitter printed other tweets into the page
-        # inside this container
-        #
-        found_username = $('.main-tweet .username').text()?.trim().replace( /^@/g , '')
-        found_tweet_id = "#{div.data('id')}"
-        rc = if not(sncmp(username, found_username)) then v_codes.BAD_USERNAME
-        else if (("" + remote_id) isnt (found_tweet_id)) then v_codes.BAD_REMOTE_ID
-        else @find_sig_in_tweet { tweet_p : div, proof_text_check }
+    await @_get_url_body { url : new_api_url, json : true }, defer err, rc, body_obj
+
+    if not err and rc is v_codes.OK
+      schm = schema.dict({
+        url: schema.string().name('url')
+        author_url: schema.string().name('author_url')
+        html: schema.string().name('html')
+      }).allow_extra_keys()
+      if err = schm.check body_obj
+        return cb err, v_codes.CONTENT_FAILURE
+
+      # "url" field returned by the API should match api_url.
+      if not(sncmp(body_obj.url, api_url))
+        err = new Error "returned url field doesn't match api_url (found: #{body_obj.url}, expected: #{api_url})"
+        return cb err, v_codes.CONTENT_FAILURE
+
+      # Extract username from URL, do not use "author_name" because it's the
+      # full name.
+      api_url_matches = api_url.match(new RegExp("^https://twitter\\.com/([^/]+)/status/(\\d+)(.*)$"))
+      if not api_url_matches
+        err = new Error "api_url field doesn't match regexp, got: #{api_url_matches}"
+        return cb err, v_codes.CONTENT_FAILURE
+
+      # Check username and tweet ID.
+      [_, username_from_url, tweet_id] = api_url_matches
+      if not(sncmp(username, username_from_url))
+        err = new Error("username from api_url didn't match, expected: #{username}, got: #{username_from_url}")
+        return cb err, v_codes.BAD_USERNAME
+      if tweet_id isnt remote_id.toString()
+        return cb err, v_codes.BAD_REMOTE_ID
+
+      # Check "author_url" field, it contains a link to user's twitter profile.
+      # It should match our username.
+      author_url_matches = body_obj.author_url.match(new RegExp("^https://twitter\\.com/(.+)$"))
+      if not author_url_matches
+        err = new Error("author_url doesn't match regexp, got: #{body_obj.author_url}")
+        return cb err, v_codes.CONTENT_FAILURE
+
+      [_, author_username] = author_url_matches
+      if not(sncmp(username, author_username))
+        err = new Error("username from author_url didn't match, expected: #{username}, got: #{author_username}")
+        return cb err, v_codes.BAD_USERNAME
+
+      if body_obj.html.length > 1000
+        err = new Error("html is #{body_obj.html.length} characters, not trying to parse it")
+        return cb err, v_codes.CONTENT_FAILURE
+
+      $ = @libs.cheerio.load body_obj.html
+      tweet_p = $('blockquote.twitter-tweet p')
+      if tweet_p.length isnt 1
+        err = new Error("failed to find tweet <p> in returned 'html' field")
+        return cb err, v_codes.FAILED_PARSE
+
+      rc = @find_sig_in_tweet { tweet_p, proof_text_check }
 
     cb err, rc