keybase-proofs 2.3.25 → 2.3.28

package/src/scrapers/reddit.iced

@@ -1,18 +1,44 @@
-{sncmp,BaseScraper} = require './base'
+{sncmp,user_agent,BaseScraper,BaseBearerToken} = require './base'
 {constants} = require '../constants'
 {v_codes} = constants
 {proof_text_check_to_med_id} = require '../base'
+urlmod = require 'url'
+{inspect} = require 'util'
 
 #================================================================================
 
 PREFIX = "https://www.reddit.com"
 SUBREDDIT = PREFIX + "/r/keybaseproofs"
+OAUTH_REDDIT_URL = "https://oauth.reddit.com"
+
+REDDIT_USER_AGENT = user_agent + " (by /u/maxtaco)"
+
+#================================================================================
+
+class RedditBearerToken extends BaseBearerToken
+  constructor : ({@base}) ->
+    super {
+      name: "Reddit"
+      @base
+      access_token_url : "#{OAUTH_REDDIT_URL}/api/v1/access_token"
+      scope : [ 'history', 'read' ]
+      user_agent : REDDIT_USER_AGENT
+    }
+
+#================================================================================
+
+_bearer_token = null
+bearer_token = ({base}) ->
+  unless _bearer_token
+    _bearer_token = new RedditBearerToken { base }
+  return _bearer_token
 
 #================================================================================
 
 exports.RedditScraper = class RedditScraper extends BaseScraper
 
   constructor: (opts) ->
+    @auth = opts.auth
     super opts
 
   # ---------------------------------------------------------------------------
@@ -38,8 +64,8 @@ exports.RedditScraper = class RedditScraper extends BaseScraper
     if (err = @_check_args { username, name })?
       return cb err, { rc: v_codes.BAD_ARGS }
 
-    await @_get_url_body
-      url: "#{PREFIX}/user/#{encodeURIComponent(username)}/submitted.json?count=25&cachebust=#{Math.random()}"
+    await @_get_body_api
+      url: "#{OAUTH_REDDIT_URL}/user/#{encodeURIComponent(username)}/submitted.json?count=25&cachebust=#{Math.random()}"
       json: true
     , defer err, rc, json
 
@@ -68,6 +94,9 @@ exports.RedditScraper = class RedditScraper extends BaseScraper
     if not proof_post?
       return cb null, rc: v_codes.NOT_FOUND
 
+    # NOTE: Still return "www.reddit.com" api_url instead of "oauth.reddit.com",
+    # for compatibility with existing rows in the keybase db. `check_status`
+    # method is rewriting www.reddit.com into oauth.reddit.com.
     cb null,
       rc : v_codes.OK
       api_url : PREFIX + proof_post.permalink + ".json"
@@ -109,12 +138,24 @@ exports.RedditScraper = class RedditScraper extends BaseScraper
   # ---------------------------------------------------------------------------
 
   check_status: ({username, api_url, proof_text_check, remote_id}, cb) ->
+    u = urlmod.parse(api_url)
+    unless u
+      return cb new Error "Failed to parse api_url"
+
+    # Host should either be www.reddit.com or oauth.reddit.com.
+    if u.host not in ['www.reddit.com', 'oauth.reddit.com']
+      return cb new Error "Unexpected api_url host: #{host}"
+
+    u = urlmod.format {
+      host : 'oauth.reddit.com'
+      protocol : 'https:'
+      pathname : u.pathname
+    }
 
     # calls back with a v_code or null if it was ok
-    await @_get_url_body {
-      url : api_url + "?cachebust=#{Math.random()}"
+    await @_get_body_api {
+      url : u
       json : true
-      user_agent : "linux:com.github/keybase/proofs:v2.1.41 (by /u/maxtaco)"
     }, defer err, rc, json
 
     rc = if rc isnt v_codes.OK then rc
@@ -122,5 +163,32 @@ exports.RedditScraper = class RedditScraper extends BaseScraper
     else @check_post { post, username, proof_text_check }
     cb err, rc
 
+  # ---------------------------------------------------------------------------
+
+  _get_bearer_token : (cb) ->
+    bt = bearer_token { base : @ }
+    await bt.get defer err, tok
+    rc = if err? then v_codes.AUTH_FAILED else v_codes.OK
+    cb err, rc, tok
+
+  # ---------------------------------------------------------------------------
+
+  _get_body_api : ({url, json}, cb) ->
+    rc = body = err = null
+    await @_get_bearer_token defer err, rc, tok
+    unless err?
+      @log "| HTTP API request for URL '#{url}'"
+      json ?= true
+      args =
+        url : url
+        json : json
+        headers :
+          Authorization : "Bearer #{tok}"
+        method : "get"
+        json : true
+        user_agent : REDDIT_USER_AGENT
+      await @_get_url_body args, defer err, rc, body
+    cb err, rc, body
+
 #================================================================================
 

package/src/scrapers/twitter.iced

@@ -1,7 +1,6 @@
-{sncmp,BaseScraper} = require './base'
+{sncmp,BaseScraper,BaseBearerToken} = require './base'
 {make_ids} = require '../base'
 {constants} = require '../constants'
-{Lock} = require '../util'
 {v_codes} = constants
 {decode_sig} = require('kbpgp').ukm
 urlmod = require 'url'
@@ -17,78 +16,27 @@ ws_normalize = (x) ->
 
 #================================================================================
 
-class BearerToken
-
-  #----------------
-
+class TwitterBearerToken extends BaseBearerToken
   constructor : ({@base}) ->
-    @_tok = null
-    @_created = 0
-    @_lock = new Lock()
-    @auth = @base.auth
-
-  #----------------
-
-  get : (cb) ->
-    await @_lock.acquire defer()
-    err = null
-    now = Math.floor(Date.now() / 1000)
-
-    if not (res = @_tok)? or (now - @_created > @auth.lifespan)
-
-      @base.log "+ Request for bearer token"
-
-      # Very crypto!  Not sure why this is done, but it's done
-      cred = (Buffer.from [ @auth.key, @auth.secret].join(":")).toString('base64')
-
-      req = 'grant_type=client_credentials'
-      opts =
-        url : "https://api.twitter.com/oauth2/token"
-        headers :
-          Authorization : "Basic #{cred}"
-          'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
-          'Content-Length': req.length
-        method : "post"
-        body : req
-      await @base._get_url_body opts, defer err, rc, body
-
-      if err?
-        @base.logl 'error', "In getting bearer_token: #{err.message}"
-      else if (rc isnt v_codes.OK)
-        @base.logl 'error', "HTTP error in getting bearer token: #{rc}"
-        err = new Error "HTTP error: #{rc}"
-      else
-        try
-          body = JSON.parse body
-        catch e
-          @base.logl 'warn', "Could not parse JSON reply: #{e}"
-          err = e
-
-      if err? then # noop
-      else if not (tok = body.access_token)?
-        @base.logl 'warn', "No access token found in reply"
-        err = new Error "Twitter error: no access token"
-      else
-        @_tok = tok
-        @_created = Math.floor(Date.now() / 1000)
-        res = @_tok
-
-      @base.log "- Request for bearer token -> #{err}"
-
-    @_lock.release()
-    cb err, res
+    super {
+      name: "Twitter"
+      @base
+      access_token_url : "https://api.twitter.com/oauth2/token"
+      scope : [ 'history', 'read' ]
+    }
 
 #================================================================================
 
 _bearer_token = null
 bearer_token = ({base}) ->
   unless _bearer_token
-    _bearer_token = new BearerToken { base }
+    _bearer_token = new TwitterBearerToken { base }
   return _bearer_token
 
 #================================================================================
 
 exports.TwitterScraper = class TwitterScraper extends BaseScraper
+  username_regexp = /^[a-z0-9_-]{2,15}$/
 
   constructor: (opts) ->
     @auth = opts.auth
@@ -101,6 +49,8 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
       new Error "Bad args to Twitter proof: no username given"
     else if not (args.name?) or (args.name isnt 'twitter')
       new Error "Bad args to Twitter proof: type is #{args.name}"
+    else if not args.username.match(username_regexp)
+      new Error "Invalid username passed to Twitter proof"
     else
       null
 
@@ -113,25 +63,26 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
 
     return cb(err,out) if (err = @_check_args { username, name })?
 
+    endpoint_name = "/2/tweets/search/recent"
     u = urlmod.format {
       host : "api.twitter.com"
       protocol : "https:"
-      pathname : "/1.1/statuses/user_timeline.json"
+      pathname : endpoint_name
       query :
-        count : 100
-        screen_name : username
+        query : "\"Verifying myself\" \"Keybase.io\" from:#{username}"
     }
-
-    await @_get_body_api { url : u }, defer err, rc, json
+    await @_get_body_api { url : u, endpoint_name }, defer err, rc, json
     @log "| search index #{u} -> #{rc}"
     if rc isnt v_codes.OK then #noop
     else if not json? or (json.length is 0) then rc = v_codes.EMPTY_JSON
+    else if not json.data? then rc = v_codes.INVALID_JSON
     else
-      for {text, id_str},i in json
+      rc = v_codes.NOT_FOUND
+      for {text, id},i in json.data
         if (@find_sig_in_tweet { inside : text, proof_text_check }) is v_codes.OK
           @log "| found valid tweet in stream @ #{i}"
           rc = v_codes.OK
-          remote_id = id_str
+          remote_id = id
           api_url = human_url = @_id_to_url username, remote_id
           out = { remote_id, api_url, human_url }
           break
@@ -397,7 +348,7 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
   # ---------------------------------------------------------------------------
 
   # Only the hunter needs this
-  _get_body_api : ({url}, cb) ->
+  _get_body_api : ({url, endpoint_name}, cb) ->
     rc = body = err = null
     await @_get_bearer_token defer err, rc, tok
     unless err?
@@ -408,6 +359,8 @@ exports.TwitterScraper = class TwitterScraper extends BaseScraper
           Authorization : "Bearer #{tok}"
         method : "get"
         json : true
+        log_ratelimit : endpoint_name?
+        endpoint_name : endpoint_name
       await @_get_url_body args, defer err, rc, body
     cb err, rc, body
 

package/SIGNED.md

@@ -1,131 +0,0 @@
-##### Signed by https://keybase.io/max
-```
------BEGIN PGP SIGNATURE-----
-Comment: GPGTools - https://gpgtools.org
-
-iQEcBAABCgAGBQJWBZ3PAAoJEJgKPw0B/gTfPrkIALMDLUb+CS2oPrFZlILoC31o
-FUKtCiN+hzC9+FgMYp18kWsWDh3DxzqPIRs28mwt9QMQTHdywv6K+nfQw2dE4wns
-1Wi7ZVPbvR5P0/ANIQ0QwetYxbkNow20dU67CRj0aEvvbopQDBgBgyH9b6LjSb+n
-8OTUFLhtiXN0PeZJ5/OyMtq8jyrjT7MEpxAb+8uFt95fPax17TNjCBEF4mlNqvtl
-YIOd/1v6YTlaqbYSzEZoMZsSI12UGK/tLaPG6qYf+5gGi96YjEMuQyDsL7ANQpkG
-jplkuKpykiY1Ndc1B3TOnTK8n+lrRaWesLmHnhldhR047ZNYgETunz+NYks7MHI=
-=96ur
------END PGP SIGNATURE-----
-
-```
-
-<!-- END SIGNATURES -->
-
-### Begin signed statement 
-
-#### Expect
-
-```
-size   exec  file                             contents                                                        
-             ./                                                                                               
-109            .gitignore                     ec278daeb8f83cac2579d262b92ee6d7d872c4d1544e881ba515d8bcc05361ab
-10709          CHANGELOG.md                   8ce81ae853a72be10648f712497199b316dfe57c8519853687cee1b508965af1
-66             CONTIBUTORS.md                 e5a536ada9dc7f25b352f28cc8936db4ded1a0860f6f23ef61d0b14aeae351da
-1483           LICENSE                        333be7050513d91d9e77ca9acb4a91261721f0050209636076ed58676bfc643d
-1383           Makefile                       b90c2b9165497660a610d5ec6177b854aafaa318aa1252af23484c755f1d959a
-43             README.md                      b2db580fc880756ae739bb148570f6afb1c250e5c0d2c5fa56ce41491753ca6c
-               lib/                                                                                           
-2498             alloc.js                     af1347e6f17197fee04fba783839279b07b07f79bf407ddf44e09d360289aa48
-1189             announcement.js              ff01f39dca21c1387faa8b79633dec40414f5645886015a46bd795772987c79e
-1383             auth.js                      86403419686d1cac766c8d3d6c8f6e10176694e33de84faafca652fd01e54e0e
-766              b64extract.js                1466f81bb930ddc439f877ef0cba4d20048f5477aafda294290f0fd0da527073
-33694            base.js                      b57791baad7fd3b648749320875ada1adc6e8d0dc3265c6f2fe4faeb960c05bb
-2043             constants.js                 e88df2ce555fe3fa8f70bbac3ebdca7c169571cbd5055948ecd8215e231d75fb
-1382             cryptocurrency.js            40536713081183e8b698a345df9a288a0967e1c95bd0699fe8ea18b618a6de6a
-1087             device.js                    f77d694ccf19c81b8c1977bb5c0280979a8b15983ced95675d6f5d44cedf67a7
-1294             eldest.js                    878cc36219a2d25c1e1897a672f03b5d97ece05581b6ece7707e78cbb0a8e02e
-909              main.js                      1cc4c7fd9cfee3ce317d8993507a953c8a22803eaea69eb16d4c7be9c0d7731b
-1549             pgp_update.js                c084bee65b6e7cee2d9fd7ddeea107751a319c03831205005ed997fa4840f939
-2578             revoke.js                    57106f23d108ce01a6d781a795417c86a09633af73ff1c9630d3893a8bc47e5f
-                 scrapers/                                                                                    
-6392               base.js                    db98e2349217287c98feba718b0bdd9570c5e1108370942af9ca2a2b59be5ed9
-15450              bitbucket.js               7530f245a6836d29177cae7e2a1e0efa9860e1adc9965731cf1e79c5f82f3f4c
-4137               coinbase.js                5d28f2c525f69e64881d0ae5dd38c535a324f894d98b67d1a5b8217a4c564ce3
-8239               dns.js                     74ced258122595b80997594c4f8fb82c210b5f316848b035b2b66582bb6052e8
-9369               generic_web_site.js        aa3fff8c4d36f86182666642117ec71c4c1574d44eeeba675f698d08fb9a1edd
-9901               github.js                  221586b7fd13ec28a51479319902fdb143a1b37d25dcdb7926826b100917ffbd
-5995               hackernews.js              0e73885c9dba4dbba56fc042afe6002aeae1699094f77555c831d2a15348033c
-8535               reddit.js                  508d1b349ce6a992afe5c882d7c541c1ff85990645343dbfd748c09888d8ce3f
-27418              twitter.js                 fda7930b6de79013601c22d0585833e6c2bf09594ec7890a4852ef3e21d24df5
-1774             sibkey.js                    b434bb5e226214d5113eae44201ead14e7430bc92e56e2446e80d2cf9076ab1d
-10983            subkey.js                    76aca817ffb548ffe3dd668d9c2b189c3c3040300f748c6acf105c5f5a5dae82
-1669             track.js                     b85423eb1309ac58f5cb2b68697f612c091ef2d44fa8cc9e4b85ab6039ae8b9a
-1506             update_passphrase_hash.js    f2cfce382fd677e31a88f3646eb2d90258583a1ef68fe74dcd178a2e178229f6
-1544             util.js                      1760f3955e71bed8a332c02a1c9a96733f9af0e0f297c89ec354f83ed5bf3908
-19530            web_service.js               769a2c81aeb9b76ec627126776c346c20209ed588536dca528a617b5fbcdc553
-744            package.json                   b670f8f7efdd7d69fa92f089729bdfe4486ded548e03b677bab888e3e21f2ab0
-               src/                                                                                           
-2376             alloc.iced                   bce30dc075d16e0551aeb8ce89f2a0515e8b061f7cbd1c9842a93756c028254b
-537              announcement.iced            98ddc17f5c6b66874ed2a273032d5c5e118d1ef3a7f5a08f4c7fb9cf06e811cf
-671              auth.iced                    3f0f211734a2f8408a5f5e1f5c353cad3373f952b01209f0f2d644b681612b0e
-704              b64extract.iced              47b06460fd61d8eb788878566c62086b1fd1f792a1712b3e8736766401bc8fee
-15746            base.iced                    cbd3229cab90232ce33ca7ced8c2d492c2eaa5091a48019486f401f3df689f4a
-2264             constants.iced               ed833b20ade7e8e95d1e4c65577cfb8e7de20b554afaf7aaa7963e86d5f2b370
-609              cryptocurrency.iced          9c839bdd1c12591ca73c6fbc86d44c51f1803718d37286e9eade945bee0761d1
-489              device.iced                  b0c028eb2598c48d9d24be615e066e0a03e8b7b1930a4ca58f29d23517daf0bc
-566              eldest.iced                  8dd63bec35fca4ab933de049888777e957827a862ecf2dec7e91e91e4265917c
-741              main.iced                    0c4af12c6c069e3ece0f446c922ba8f306df4f386e3bee3c4ba852bc92a4df0b
-589              pgp_update.iced              cdc5fed1d47c0bfe144062cf7985d466ced99fb8ec7bd17901d504127359fc64
-788              revoke.iced                  2b94671a923dcdd274d0f769f186f0757a27db64828eaad5fa5eb1a24f9e479f
-                 scrapers/                                                                                    
-4029               base.iced                  e110786284dda6cd78bc0be507ff9ac0add3a128c959c073562416b513d8b2d8
-3967               bitbucket.iced             f22e9c29d781afbecccc0c720513fa63710e3c199cc9780ef34b87a4db0e1fd9
-2370               coinbase.iced              5b1132561f0c33d4c35ba2d7d9c32200e77d83c68a3bce3d770b27085d649bcd
-4081               dns.iced                   773ea8e65f46b2db994e41ed4849c4637200ee2e978f280524d63b06a42ff454
-3249               generic_web_site.iced      7a70be6c922e5adb73459973b81143ec9020b109cdbcc5acf349e47c5ca1f68d
-3305               github.iced                f10410ccca3bb3fc5045024dada97a4974984ee433476bea30c1ab47f870da90
-2909               hackernews.iced            cb14793b3ea889bcbc2121430bc3728655e125806001211864d3c90c1104505a
-4082               reddit.iced                8a4f6902eee7e1609df18bf9a00b5934d2e9a16eaf20bd6231b3d11b2554effe
-11418              twitter.iced               9bbb6cfdc464a5ed3261cc8f1a14a12e18325d6b0edc753c7a9e133b58d9328e
-761              sibkey.iced                  308c4cfef33dbc79809600a46c4f1704d4240be18478d2e5e8f71e523fd6962d
-3503             subkey.iced                  6f41a5aa38813486cbcb5a3c918c2e7aa464665b9dac852f92bb62ffa1e0a3b1
-838              track.iced                   d72d0eb5cda7f0f9800f17225fedf552f45111b802f66c463c4408794bd11aca
-658              update_passphrase_hash.iced  ea87dd15d3a36ee581e48a5436f4c64f38a102906efcdc20bc925b821210bfd9
-1052             util.iced                    88092afaf23bf8dfe42a72252ac156be45e94588911584ecf535d228b108f2b9
-10177            web_service.iced             bb436e165233be85342d0abe01237fcfb5773b0cc050abef49f1f19094abd8f0
-1673           test_scraper.iced              099823ea87768b45dd205bd6dc36773999730dadb26fe3844e0a05f8668bb9d1
-```
-
-#### Ignore
-
-```
-/SIGNED.md
-```
-
-#### Presets
-
-```
-git      # ignore .git and anything as described by .gitignore files
-dropbox  # ignore .dropbox-cache and other Dropbox-related files    
-kb       # ignore anything as described by .kbignore files          
-```
-
-<!-- summarize version = 0.0.9 -->
-
-### End signed statement
-
-<hr>
-
-#### Notes
-
-With keybase you can sign any directory's contents, whether it's a git repo,
-source code distribution, or a personal documents folder. It aims to replace the drudgery of:
-
-  1. comparing a zipped file to a detached statement
-  2. downloading a public key
-  3. confirming it is in fact the author's by reviewing public statements they've made, using it
-
-All in one simple command:
-
-```bash
-keybase dir verify
-```
-
-There are lots of options, including assertions for automating your checks.
-
-For more info, check out https://keybase.io/docs/command_line/code_signing