@juzi/wechaty-puppet-service 1.0.0

package/src/auth/ca.ts

@@ -0,0 +1,137 @@
+/**
+ * Wechaty Certificate Authority Repo:
+ *  https://github.com/wechaty/dotenv/tree/main/ca
+ *
+ * The SSL_ROOT_CERT is a root certificate generated by and for wechaty community.
+ *
+ * Because it's the default root cert for the puppet service,
+ *  so all the Polyglot Wechaty SDK should set this cert to be trusted by default.
+ *
+ * Update:
+ *  - Huan(202108): init, expired in 3650 days (after 2031/07)
+ */
+const TLS_CA_CERT = `-----BEGIN CERTIFICATE-----
+MIIFxTCCA62gAwIBAgIUYddLAoa8JnLzJ80l2u5vGuFsaEIwDQYJKoZIhvcNAQEL
+BQAwcjELMAkGA1UEBhMCVVMxFjAUBgNVBAgMDVNhbiBGcmFuY2lzY28xEjAQBgNV
+BAcMCVBhbG8gQWx0bzEQMA4GA1UECgwHV2VjaGF0eTELMAkGA1UECwwCQ0ExGDAW
+BgNVBAMMD3dlY2hhdHktcm9vdC1jYTAeFw0yMTA4MDkxNTQ4NTJaFw0zMTA4MDcx
+NTQ4NTJaMHIxCzAJBgNVBAYTAlVTMRYwFAYDVQQIDA1TYW4gRnJhbmNpc2NvMRIw
+EAYDVQQHDAlQYWxvIEFsdG8xEDAOBgNVBAoMB1dlY2hhdHkxCzAJBgNVBAsMAkNB
+MRgwFgYDVQQDDA93ZWNoYXR5LXJvb3QtY2EwggIiMA0GCSqGSIb3DQEBAQUAA4IC
+DwAwggIKAoICAQDulLjOZhzQ58TSQ7TfWNYgdtWhlc+5L9MnKb1nznVRhzAkZo3Q
+rPLRW/HDjlv2OEbt4nFLaQgaMmc1oJTUVGDBDlrzesI/lJh7z4eA/B0z8eW7f6Cw
+/TGc8lgzHvq7UIE507QYPhvfSejfW4Prw+90HJnuodriPdMGS0n9AR37JPdQm6sD
+iMFeEvhHmM2SXRo/o7bll8UDZi81DoFu0XuTCx0esfCX1W5QWEmAJ5oAdjWxJ23C
+lxI1+EjwBQKXGqp147VP9+pwpYW5Xxpy870kctPBHKjCAti8Bfo+Y6dyWz2UAd4w
+4BFRD+18C/TgX+ECl1s9fsHMY15JitcSGgAIz8gQX1OelECaTMRTQfNaSnNW4LdS
+sXMQEI9WxAU/W47GCQFmwcJeZvimqDF1QtflHSaARD3O8tlbduYqTR81LJ63bPoy
+9e1pdB6w2bVOTlHunE0YaGSJERALVc1xz40QpPGcZ52mNCb3PBg462RQc77yv/QB
+x/P2RC1y0zDUF2tP9J29gTatWq6+D4MhfEk2flZNyzAgJbDuT6KAIJGzOB1ZJ/MG
+o1gS13eTuZYw24LElrhd1PrR6OHK+lkyYzqUPYMulUg4HzaZIDclfHKwAC4lecKm
+zC5q9jJB4m4SKMKdzxvpIOfdahoqsZMg34l4AavWRqPTpwEU0C0dboNA/QIDAQAB
+o1MwUTAdBgNVHQ4EFgQU0rey3QPklTOgdhMJ9VIA6KbZ5bAwHwYDVR0jBBgwFoAU
+0rey3QPklTOgdhMJ9VIA6KbZ5bAwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
+AQsFAAOCAgEAx2uyShx9kLoB1AJ8x7Vf95v6PX95L/4JkJ1WwzJ9Dlf3BcCI7VH7
+Fp1dnQ6Ig7mFqSBDBAUUBWAptAnuqIDcgehI6XAEKxW8ZZRxD877pUNwZ/45tSC4
+b5U5y9uaiNK7oC3LlDCsB0291b3KSOtevMeDFoh12LcliXAkdIGGTccUxrH+Cyij
+cBOc+EKGJFBdLqcjLDU4M6QdMMMFOdfXyAOSpYuWGYqrxqvxQjAjvianEyMpNZWM
+lajggJqiPhfF67sZTB2yzvRTmtHdUq7x+iNOVonOBcCHu31aGxa9Py91XEr9jaIQ
+EBdl6sycLxKo8mxF/5tyUOns9+919aWNqTOUBmI15D68bqhhOVNyvsb7aVURIt5y
+6A7Sj4gSBR9P22Ba6iFZgbvfLn0zKLzjlBonUGlSPf3rSIYUkawICtDyYPvK5mi3
+mANgIChMiOw6LYCPmmUVVAWU/tDy36kr9ZV9YTIZRYAkWswsJB340whjuzvZUVaG
+DgW45GPR6bGIwlFZeqCwXLput8Z3C8Sw9bE9vjlB2ZCpjPLmWV/WbDlH3J3uDjgt
+9PoALW0sOPhHfYklH4/rrmsSWMYTUuGS/HqxrEER1vpIOOb0hIiAWENDT/mruq22
+VqO8MHX9ebjInSxPmhYOlrSZrOgEcogyMB4Z0SOtKVqPnkWmdR5hatU=
+-----END CERTIFICATE-----`
+
+/**
+ * Huan(202108): This private key is NOT SAFE!
+ *
+ * WARNING: This CA is not safe for production.
+ * **use environment variables to set your safe CA data**
+ *
+ * Our system use this private key for server by default for convience.
+ *  However, everyone can get this key and use it to see the traffic between client and server.
+ *
+ * For security, we should not use this key in production
+ *  by setting it manually by
+ *    either the environment variable `WECHATY_PUPPET_SERVICE_TLS_SERVER_KEY`
+ *    or `options.tlsServerKey`
+ *
+ * So does the below `TLS_SERVER_CERT_UNSAFE`
+ */
+const TLS_INSECURE_SERVER_KEY = `-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAtdFTXAKLW16uqNokJmSowbGtwnCvsPSqIHcdbKgdcuNpaJsZ
+DTeBP0/XHFvnXcekHOyzncYgluxijzMSD1S8AKo3c2fROgem+E+WMSLYAZSTV48p
+uzTRLoypvfhKfqxsrmpct2F6tRTIQ/EABOs0TYP0dY3Nd8NkCEWBmv7ioPDek/a4
+esdisN7R1Ea6jx7ToegSwjkP9aFr2XHxyqR5wjJn/Q6nYZC9A90CKdxJ2WpXtluT
+xFfFfqOhR/1te5/LpqXtqxo2yOwu8k67fHub1FyLu9sAYhcsuSjHVHxbK3nPf0mN
+Gt0RiSwRj84qzbpfwrjMYrAJ3EqKrlxZurmX0wIDAQABAoIBAAcG+SbUPligtzV1
+gPIu78rUuDeMrW20dyLcF7oMYV8AZSGS5Qv6ujcdOd4xuyaHwdMQXvzZHIdYyZJp
+UehfyQhpi80dFRweEZkFUnPBugGNoYg/00gWCYO4EhNylkaBGY5ANCcuUFTRYdAm
+b27BPHtGf1tPyMI5PhOHxDOeaFn6BKB1pcG4mQ+CNieadYxjgPcInh6mAqgJ40cR
+ncWgLgSdChijLlVLW9lFVA+OAqv57vT3xW+Op1r7nBiigj67tka57spZTIEhrLXI
+ZFMyRKQXlxh/l82vLmnYAhvSp/hHbARLwWfQ/znsFvTc/HXvXPocpZ2B9f0tlZ0W
+dqOHSwkCgYEA3zJYAC5Afw3UKuAyApWOyI3AX+noq6ze8B3jFWEPmNdJrZpjLpzp
+mntnWC8Wq0t821uiQXYlGUzF/pIg0rzVdPbc2VTdwKl+iptuCn3fC+LCTJKroRLq
+2a6GDhtmV2g0SEaNdbJt1Zfwr0KyXLNwK+ZdJxhS44vfTCRB4YBsFw0CgYEA0Ioe
+pcRBEyCJud8ZJnSN/HiOQ9kCIsnd8Pk4D7q+DGWY6lLGQddhlkp9Ah7yRIGJVg91
+D6t5BfpiU8DRGFiEGMo+XWEKjLfRTxg03lBQYACJc2crgFRuG8GFuO/WQ1b9ihR5
+nsdLc9cGIm6rFXaUsnLIN9IJhJg4BmFD1U9usl8CgYEArIN+D02wnkOzDRzSqrqs
+bQlbewcRxrfMbS28moa2Bn3Ivf1J0fqIeNYPL9Ldo7KqI+Z0yEIoNKDpnHWYFyrL
+lidE1lrJN6QKYdn3OPbHUqmHYqYvMEWt7mj9xqOY+9BYMNEPf7xVNrXE28IimJI9
+DkF1GMWtM6GmC3Uu0rxvT3UCgYEAgroCylGDpbThAXa8cmHgXCNKs3eHIj2/dn8U
+SK/80RKjUEkBZWbaEvew87Jols9JQ3y/GkqYvEmgd/ZIXWWnsU6e17Ssg1f7ywRW
+qAJa0EOl5oUHPRQwTg/7ftpCS8Zte7CoKQOv5fcmLlGHyBWk01Sm9G8jbk5p2H4C
+ouZ/cysCgYBqZHm6eg0tjwFPJJWgmAMdNvBlnIuW1t5dwa7B2F6ONveUTBBAxGLc
+ZBVdEBseBPki5i7M7eNKNTEA3EM+Cfsfsp5U/S8ntDmzzaMoBhb+jBRor39l3+iG
+qXI72DDvrh802t6KO9W6CQIfpVcxLeOy82RfUP1pHQ/sMPkx89Fd5A==
+-----END RSA PRIVATE KEY-----`
+
+const TLS_INSECURE_SERVER_CERT = `-----BEGIN CERTIFICATE-----
+MIIEVTCCAj0CAQEwDQYJKoZIhvcNAQELBQAwcjELMAkGA1UEBhMCVVMxFjAUBgNV
+BAgMDVNhbiBGcmFuY2lzY28xEjAQBgNVBAcMCVBhbG8gQWx0bzEQMA4GA1UECgwH
+V2VjaGF0eTELMAkGA1UECwwCQ0ExGDAWBgNVBAMMD3dlY2hhdHktcm9vdC1jYTAe
+Fw0yMTA4MjQxODMwMjBaFw0zMTA4MjIxODMwMjBaMG8xCzAJBgNVBAYTAlVTMRYw
+FAYDVQQIDA1TYW4gRnJhbmNpc2NvMRIwEAYDVQQHDAlQYWxvIEFsdG8xEDAOBgNV
+BAoMB1dlY2hhdHkxDzANBgNVBAsMBlB1cHBldDERMA8GA1UEAwwIaW5zZWN1cmUw
+ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC10VNcAotbXq6o2iQmZKjB
+sa3CcK+w9Kogdx1sqB1y42lomxkNN4E/T9ccW+ddx6Qc7LOdxiCW7GKPMxIPVLwA
+qjdzZ9E6B6b4T5YxItgBlJNXjym7NNEujKm9+Ep+rGyualy3YXq1FMhD8QAE6zRN
+g/R1jc13w2QIRYGa/uKg8N6T9rh6x2Kw3tHURrqPHtOh6BLCOQ/1oWvZcfHKpHnC
+Mmf9DqdhkL0D3QIp3EnZale2W5PEV8V+o6FH/W17n8umpe2rGjbI7C7yTrt8e5vU
+XIu72wBiFyy5KMdUfFsrec9/SY0a3RGJLBGPzirNul/CuMxisAncSoquXFm6uZfT
+AgMBAAEwDQYJKoZIhvcNAQELBQADggIBALyPgW0VLlQfkgsNovyLg+zkF7oJZCvM
+HS7m43abZb1H1xUH6Kd/sUFTQCAAPop/n4773iH0KggWtoPjkid1G1s/UWK6A0F1
+IxRp0DYLgZfL/U+PQxe175ViYRLPUKj1YwagjX6HvM5bUMEYDnIypEH2UFIrD39J
+69Q6M8hZ85oFDAo2hRqrjJo66c3+ygmXSCFIL64gsVLZkK3SHRAv3R90+blNgmo5
+Yvh2xqvGuspd1Y3yzeOQreimJkMeDr/t/xucws1TK7fqMjlk/36W4S95xT7EYykf
+rQ+1cDIJvGdVU/lod0/lWcOvqMtyf6wIjzFJaGAoqS5QT2IeeXQYbhq9bZIBQzth
+IfvfdHuijUqOhT8LX8TYXPWVR/UEKItqktdvA7PXuHUdDxU3ldcXsjA+m9jVO81i
+gIOUJQuBR/tImNnLFaTooO6RB71lBB1XCo8HvWPu47MPjxuf/Y+1frPzuP8LFMWj
+bjw0QcwFUik8v+mSiPHhOIfzp0EQlFtlncTr+k0MFuRKokl0Yrs8jXOt30JC4tKS
+GKXupLWnWE3Z15L9uk9zSAskL2T8LwnctaiMP0+mzf8gWchxUaHkk0yGj4gtNVyU
+iJZfrWYwBY9y4SUjp6A7pLspw+i+jIO/EmcX2jbFt1LaajRgEw+uGvNMXhHqtsHC
+WqE+fOGDBUET
+-----END CERTIFICATE-----`
+
+/**
+ * Common Name:
+ *  - Server Name Indication (SNI)
+ *  - Case insensitive
+ *
+ * Wechaty Token format: `${SNI}/${UUIDv4}`
+ *
+ * Huan(202108): the CA generated by Wechaty Puppet Service
+ *  default set to this value:
+ *
+ * grpc.ssl_target_name_override: 'insecure'
+ */
+const TLS_INSECURE_SERVER_CERT_COMMON_NAME = 'insecure' // case insensitive
+
+export {
+  TLS_CA_CERT,
+  TLS_INSECURE_SERVER_CERT_COMMON_NAME,
+  TLS_INSECURE_SERVER_CERT,
+  TLS_INSECURE_SERVER_KEY,
+}

package/src/auth/call-cred.spec.ts

@@ -0,0 +1,25 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+
+import {
+  test,
+  sinon,
+}         from 'tstest'
+
+import { metaGeneratorToken } from './call-cred.js'
+
+test('metaGeneratorToken()', async t => {
+  const TOKEN = 'UUIDv4'
+  const EXPECTED_AUTHORIZATION = `Wechaty ${TOKEN}`
+
+  const sandbox = sinon.createSandbox()
+  const spy = sandbox.spy()
+
+  const metaGenerator = metaGeneratorToken(TOKEN)
+
+  metaGenerator({} as any, spy)
+  t.equal(spy.args[0]![0], null, 'should no error')
+
+  const metadata = spy.args[0]![1]
+  const authorization = metadata.get('Authorization')[0]
+  t.equal(authorization, EXPECTED_AUTHORIZATION, 'should generate authorization in metadata')
+})

package/src/auth/call-cred.ts

@@ -0,0 +1,22 @@
+import {
+  credentials,
+  CallMetadataGenerator,
+  Metadata,
+}                         from './grpc-js.js'
+
+/**
+ * With server authentication SSL/TLS and a custom header with token
+ *  https://grpc.io/docs/guides/auth/#with-server-authentication-ssltls-and-a-custom-header-with-token-1
+ */
+const metaGeneratorToken: (token: string) => CallMetadataGenerator = token => (_params, callback) => {
+  const meta = new Metadata()
+  meta.add('authorization', `Wechaty ${token}`)
+  callback(null, meta)
+}
+
+const callCredToken = (token: string) => credentials.createFromMetadataGenerator(metaGeneratorToken(token))
+
+export {
+  callCredToken,
+  metaGeneratorToken,
+}

package/src/auth/env-vars.ts

@@ -0,0 +1,32 @@
+/**
+ * Environment variables containing newlines in Node?
+ *  `replace(/\\n/g, '\n')`
+ *    https://stackoverflow.com/a/36439803/1123955
+ */
+const WECHATY_PUPPET_SERVICE_TLS_CA_CERT = (v?: string) => v
+ || process.env['WECHATY_PUPPET_SERVICE_TLS_CA_CERT']?.replace(/\\n/g, '\n')
+
+const WECHATY_PUPPET_SERVICE_TLS_SERVER_CERT = (v?: string) => v
+ || process.env['WECHATY_PUPPET_SERVICE_TLS_SERVER_CERT']?.replace(/\\n/g, '\n')
+
+const WECHATY_PUPPET_SERVICE_TLS_SERVER_KEY = (v?: string) => v
+ || process.env['WECHATY_PUPPET_SERVICE_TLS_SERVER_KEY']?.replace(/\\n/g, '\n')
+
+const WECHATY_PUPPET_SERVICE_TLS_SERVER_NAME = (v?: string) => v
+ || process.env['WECHATY_PUPPET_SERVICE_TLS_SERVER_NAME']?.replace(/\\n/g, '\n')
+
+const WECHATY_PUPPET_SERVICE_NO_TLS_INSECURE_SERVER = (v?: boolean) => typeof v === 'undefined'
+  ? process.env['WECHATY_PUPPET_SERVICE_NO_TLS_INSECURE_SERVER'] === 'true'
+  : v
+const WECHATY_PUPPET_SERVICE_NO_TLS_INSECURE_CLIENT = (v?: boolean) => typeof v === 'undefined'
+  ? process.env['WECHATY_PUPPET_SERVICE_NO_TLS_INSECURE_CLIENT'] === 'true'
+  : v
+
+export {
+  WECHATY_PUPPET_SERVICE_NO_TLS_INSECURE_CLIENT,
+  WECHATY_PUPPET_SERVICE_NO_TLS_INSECURE_SERVER,
+  WECHATY_PUPPET_SERVICE_TLS_CA_CERT,
+  WECHATY_PUPPET_SERVICE_TLS_SERVER_CERT,
+  WECHATY_PUPPET_SERVICE_TLS_SERVER_KEY,
+  WECHATY_PUPPET_SERVICE_TLS_SERVER_NAME,
+}

package/src/auth/grpc-js.ts

@@ -0,0 +1,29 @@
+import {
+  credentials,
+  StatusBuilder,
+  UntypedHandleCall,
+  Metadata,
+  UntypedServiceImplementation,
+}                                 from '@grpc/grpc-js'
+import type {
+  sendUnaryData,
+  ServerUnaryCall,
+}                                 from '@grpc/grpc-js/build/src/server-call.js'
+import type {
+  CallMetadataGenerator,
+}                                 from '@grpc/grpc-js/build/src/call-credentials.js'
+import { Status as GrpcStatus }     from '@grpc/grpc-js/build/src/constants.js'
+
+export type {
+  UntypedServiceImplementation,
+  UntypedHandleCall,
+  sendUnaryData,
+  ServerUnaryCall,
+  CallMetadataGenerator,
+}
+export {
+  credentials,
+  GrpcStatus,
+  Metadata,
+  StatusBuilder,
+}

package/src/auth/mod.ts

@@ -0,0 +1,9 @@
+import { authImplToken }                                      from './auth-impl-token.js'
+import { monkeyPatchMetadataFromHttp2Headers }                from './mokey-patch-header-authorization.js'
+import { callCredToken }                                      from './call-cred.js'
+
+export {
+  authImplToken,
+  callCredToken,
+  monkeyPatchMetadataFromHttp2Headers,
+}

package/src/auth/mokey-patch-header-authorization.ts

@@ -0,0 +1,45 @@
+import type http2 from 'http2'
+
+import { log } from '@juzi/wechaty-puppet'
+
+import type {
+  Metadata,
+}             from './grpc-js.js'
+
+/**
+ * Huan(202108): This is for compatible with old clients (version v0.26 or before)
+ *  for non-tls authorization:
+ *    use the HTTP2 header `:authority` as the token.
+ *
+ * This feature is planed to be removed after Dec 31, 2022
+ */
+function monkeyPatchMetadataFromHttp2Headers (
+  MetadataClass: typeof Metadata,
+): () => void {
+  log.verbose('wechaty-puppet-service', 'monkeyPatchMetadataFromHttp2Headers()')
+
+  const fromHttp2Headers = MetadataClass.fromHttp2Headers
+  MetadataClass.fromHttp2Headers = function (
+    headers: http2.IncomingHttpHeaders,
+  ): Metadata {
+    const metadata = fromHttp2Headers.call(MetadataClass, headers)
+
+    if (metadata.get('authorization').length <= 0) {
+      const authority = headers[':authority']
+      const authorization = `Wechaty ${authority}`
+      metadata.set('authorization', authorization)
+    }
+    return metadata
+  }
+
+  /**
+   * un-monkey-patch
+   */
+  return () => {
+    MetadataClass.fromHttp2Headers = fromHttp2Headers
+  }
+}
+
+export {
+  monkeyPatchMetadataFromHttp2Headers,
+}

package/src/auth/monkey-patch-header-authorization.spec.ts

@@ -0,0 +1,26 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+
+import { test } from 'tstest'
+import type http2 from 'http2'
+
+import {
+  Metadata,
+}             from './grpc-js.js'
+
+import { monkeyPatchMetadataFromHttp2Headers } from './mokey-patch-header-authorization.js'
+
+test('monkeyPatchMetadataFromHttp2Headers', async t => {
+  const AUTHORITY = '__authority__'
+  const headers: http2.IncomingHttpHeaders = {
+    ':authority': AUTHORITY,
+  }
+
+  const dispose = monkeyPatchMetadataFromHttp2Headers(Metadata)
+  const meta = Metadata.fromHttp2Headers(headers)
+
+  const authorization = meta.get('authorization')[0]
+  const EXPECTED = `Wechaty ${AUTHORITY}`
+  t.equal(authorization, EXPECTED, 'should get authority from metadata')
+
+  dispose()
+})

package/src/client/duck/actions.ts

@@ -0,0 +1,31 @@
+/**
+ *   Wechaty Open Source Software - https://github.com/wechaty
+ *
+ *   @copyright 2016 Huan LI (李卓桓) <https://github.com/huan>, and
+ *                   Wechaty Contributors <https://github.com/wechaty>.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+ */
+/* eslint-disable sort-keys */
+import {
+  createAction,
+}                   from 'typesafe-actions'
+
+import * as types   from './types.js'
+
+/**
+ * Bug compatible & workaround for Ducks API
+ *  https://github.com/huan/ducks/issues/2
+ */
+export const nop = createAction(types.NOP)()

package/src/client/duck/epic-recover.spec.ts

@@ -0,0 +1,180 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+/* eslint-disable func-call-spacing */
+
+import { test }  from 'tstest'
+
+import {
+  TestScheduler,
+}                     from 'rxjs/testing'
+import {
+  throttleTime,
+}                     from 'rxjs/operators'
+import PuppetMock     from 'wechaty-puppet-mock'
+
+import {
+  Duck as PuppetDuck,
+}                     from 'wechaty-redux'
+
+import {
+  epicRecoverReset$,
+  epicRecoverDing$,
+  monitorHeartbeat$,
+}                     from './epic-recover.js'
+
+/**
+ * RxJS Marble Testing
+ *
+ *  - https://rxjs.dev/guide/testing/marble-testing
+ *  - https://github.com/ReactiveX/rxjs/blob/master/docs_app/content/guide/testing/marble-testing.md
+ *
+ */
+test('Example: marble testing', async t => {
+  const testScheduler = new TestScheduler(t.same)
+
+  testScheduler.run(helpers => {
+    const { cold, time, expectObservable, expectSubscriptions } = helpers
+    const e1       = cold('-a--b--c---|')
+    const e1subs   = '     ^----------!'
+    const t        = time('---|        ')  // t = 3
+    const expected = '     -a-----c---|'
+
+    expectObservable(e1.pipe(throttleTime(t))).toBe(expected)
+    expectSubscriptions(e1.subscriptions).toBe(e1subs)
+  })
+})
+
+test('Example 2: marble subscribe time frame testing', async t => {
+  const testScheduler = new TestScheduler(t.same)
+
+  testScheduler.run(helpers => {
+    const { hot, expectObservable, expectSubscriptions } = helpers
+    const source = hot('  --a--b--c--d--e--f')
+    const subscription = '-----^------!-'
+    const expected = '    -----b--c--d--'
+    expectObservable(source, subscription).toBe(expected)
+    void expectSubscriptions
+    // expectSubscriptions(source.subscriptions).toBe(subscription)
+  })
+})
+
+test('Example 3: subscribe with unsubscribe / complete', async t => {
+  const testScheduler = new TestScheduler(t.same)
+
+  testScheduler.run(({ hot, expectObservable }) => {
+    const values = {
+      a: 0,
+      b: 1,
+      c: 2,
+    }
+    const source = hot('   10ms a 9ms  b 9ms  c-|', values)
+    const subscription1 = '       20ms ^ 9ms  -!-'
+    const subscription2 = '              30ms ^--'
+    const expected1 = '           20ms b 9ms  c--'
+    const expected2 = '                  30ms c-|'
+    expectObservable(source, subscription1).toBe(expected1, values)
+    expectObservable(source, subscription2).toBe(expected2, values)
+  })
+})
+
+test('Example 4: play ground', async t => {
+  const testScheduler = new TestScheduler(t.same)
+
+  testScheduler.run(helpers => {
+    const { hot, cold, expectObservable } = helpers
+    void hot
+    void cold
+    const TIMEOUT = 15
+    const source    = hot(`-----a-----h------h ${TIMEOUT}ms --`)
+    const sub       = `    ------^-------!---- ${TIMEOUT}ms --`
+    const expected  = `    -----------h------- ${TIMEOUT}ms --`
+
+    expectObservable(source, sub).toBe(expected)
+  })
+})
+
+test('monitorHeartbeat$() emit once after lost heartbeat', async t => {
+  const testScheduler = new TestScheduler(t.same)
+
+  const puppet = new PuppetMock() as any
+
+  const TIMEOUT = 15
+
+  testScheduler.run(helpers => {
+    const { hot, expectObservable, expectSubscriptions } = helpers
+
+    const marble = {
+      a: PuppetDuck.actions.STATE_ACTIVATED_EVENT   (puppet.id, true),
+      d: PuppetDuck.actions.DONG_RECEIVED_EVENT     (puppet.id, { data: 'dong' }),
+      e: PuppetDuck.actions.ERROR_RECEIVED_EVENT    (puppet.id, { gerror: `monitorHeartbeat$() TIMEOUT(${TIMEOUT})` }),
+      h: PuppetDuck.actions.HEARTBEAT_RECEIVED_EVENT(puppet.id, { data: 'heartbeat' }),
+    }
+
+    const puppet$ = hot(` -a----h----h ${TIMEOUT}ms       ${TIMEOUT}ms ${TIMEOUT}ms ${TIMEOUT - 1}ms h ${TIMEOUT}ms       ${TIMEOUT - 1}ms d ${TIMEOUT}ms       ------`, marble)
+    const subscription = `^----------- ${TIMEOUT}ms       ${TIMEOUT}ms ${TIMEOUT}ms ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       -----!`
+    const expected  = `   ------------ ${TIMEOUT - 1}ms e ${TIMEOUT}ms ${TIMEOUT}ms ${TIMEOUT}ms       ${TIMEOUT - 1}ms e ${TIMEOUT}ms       ${TIMEOUT - 1}ms e ------`
+
+    expectObservable(
+      monitorHeartbeat$(TIMEOUT)(puppet$),
+      subscription,
+    ).toBe(expected, marble)
+
+    void expectSubscriptions
+    // expectSubscriptions(puppet$.subscriptions).toBe(sub)
+  })
+})
+
+test('epicRecoverDing$() emit periodly', async t => {
+  const testScheduler = new TestScheduler(t.same)
+
+  const puppet = new PuppetMock() as any
+
+  const TIMEOUT = 15
+
+  testScheduler.run(helpers => {
+    const { hot, expectObservable } = helpers
+
+    const marble = {
+      a: PuppetDuck.actions.STATE_ACTIVATED_EVENT   (puppet.id, true),
+      d: PuppetDuck.actions.DONG_RECEIVED_EVENT     (puppet.id, { data: 'dong' }),
+      h: PuppetDuck.actions.HEARTBEAT_RECEIVED_EVENT(puppet.id, { data: 'heartbeat' }),
+      i: PuppetDuck.actions.DING_COMMAND           (puppet.id, 'epicRecoverDing$'),
+    }
+
+    const puppet$ = hot(` -a----h----h ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT - 1}ms h ${TIMEOUT - 1}ms h ${TIMEOUT - 1}ms d ${TIMEOUT - 1}ms d ------`, marble)
+    const subscription = `^----------- ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       -----!`
+    const expected  = `   ------------ ${TIMEOUT - 1}ms i ${TIMEOUT - 1}ms i ${TIMEOUT - 1}ms i ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ------`
+
+    expectObservable(
+      epicRecoverDing$(TIMEOUT)(puppet$),
+      subscription,
+    ).toBe(expected, marble)
+  })
+})
+
+test('epicRecoverReset$() emit periodly', async t => {
+  const testScheduler = new TestScheduler(t.same)
+
+  const puppet = new PuppetMock() as any
+
+  const TIMEOUT = 60
+
+  testScheduler.run(helpers => {
+    const { hot, expectObservable } = helpers
+
+    const marble = {
+      a: PuppetDuck.actions.STATE_ACTIVATED_EVENT   (puppet.id, true),
+      d: PuppetDuck.actions.DONG_RECEIVED_EVENT     (puppet.id, { data: 'dong' }),
+      h: PuppetDuck.actions.HEARTBEAT_RECEIVED_EVENT(puppet.id, { data: 'heartbeat' }),
+      r: PuppetDuck.actions.RESET_COMMAND          (puppet.id, 'epicRecoverReset$'),
+    }
+
+    const puppet$ = hot(` -a----h----h ${TIMEOUT}ms       ${TIMEOUT * 2}ms       ${TIMEOUT * 2}ms       ${TIMEOUT * 2 - 1}ms h ${TIMEOUT - 1}ms h ${TIMEOUT - 1}ms d ${TIMEOUT - 1}ms d ------`, marble)
+    const subscription = `^----------- ${TIMEOUT}ms       ${TIMEOUT * 2}ms       ${TIMEOUT * 2}ms       ${TIMEOUT * 2}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       -----!`
+    const expected  = `   ------------ ${TIMEOUT - 1}ms r ${TIMEOUT * 2 - 1}ms r ${TIMEOUT * 2 - 1}ms r ${TIMEOUT * 2}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ${TIMEOUT}ms       ------`
+
+    expectObservable(
+      epicRecoverReset$(TIMEOUT)(puppet$),
+      subscription,
+    ).toBe(expected, marble)
+  })
+})

package/src/client/duck/epic-recover.ts

@@ -0,0 +1,109 @@
+/**
+ *   Wechaty Open Source Software - https://github.com/wechaty
+ *
+ *   @copyright 2016 Huan LI (李卓桓) <https://github.com/huan>, and
+ *                   Wechaty Contributors <https://github.com/wechaty>.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+ */
+import {
+  Observable,
+  interval,
+  timer,
+}                       from 'rxjs'
+import {
+  debounce,
+  filter,
+  map,
+  switchMap,
+  takeUntil,
+  tap,
+}                       from 'rxjs/operators'
+import {
+  isActionOf,
+}                       from 'typesafe-actions'
+import type {
+  // type Action,
+  AnyAction,
+}                       from 'redux'
+import {
+  Duck as PuppetDuck,
+}                       from 'wechaty-redux'
+import { log }          from '@juzi/wechaty-puppet'
+
+const stateActive$ = (action$: Observable<AnyAction>) => action$.pipe(
+  filter(isActionOf(PuppetDuck.actions.STATE_ACTIVATED_EVENT)),
+  filter(action => action.payload.state === true),
+)
+
+const stateInactive$ = (action$: Observable<AnyAction>) => action$.pipe(
+  filter(isActionOf(PuppetDuck.actions.STATE_INACTIVATED_EVENT)),
+)
+
+const heartbeat$ = (action$: Observable<AnyAction>) => action$.pipe(
+  filter(isActionOf([
+    PuppetDuck.actions.HEARTBEAT_RECEIVED_EVENT,
+    PuppetDuck.actions.DONG_RECEIVED_EVENT,
+  ])),
+)
+
+// Emit once when an active puppet lost heartbeat after a timeout period
+const monitorHeartbeat$ = (timeoutMilliseconds: number) =>
+  (action$: Observable<AnyAction>) =>
+    stateActive$(action$).pipe(
+      switchMap(action => heartbeat$(action$).pipe(
+        debounce(() => interval(timeoutMilliseconds)),
+        tap(() => log.verbose('PuppetService', 'monitorHeartbeat$() %d seconds TIMEOUT',
+          Math.floor(timeoutMilliseconds / 1000),
+        )),
+        map(() => PuppetDuck.actions.ERROR_RECEIVED_EVENT(
+          action.meta.puppetId,
+          { gerror: `monitorHeartbeat$() TIMEOUT(${timeoutMilliseconds})` },
+        )),
+      )),
+    )
+
+const epicRecoverDing$ = (timeoutMilliseconds: number) =>
+  (action$: Observable<AnyAction>) =>
+    monitorHeartbeat$(timeoutMilliseconds)(action$).pipe(
+      switchMap(action => timer(0, Math.floor(timeoutMilliseconds)).pipe(
+        tap(n => log.verbose('PuppetService', 'epicRecoverDing$() actions.ding() emitted #%d', n)),
+        map(() => PuppetDuck.actions.DING_COMMAND(
+          action.meta.puppetId,
+          'epicRecoverDing$',
+        )),
+        takeUntil(heartbeat$(action$)),
+        takeUntil(stateInactive$(action$)),
+      )),
+    )
+
+const epicRecoverReset$ = (timeoutMilliseconds: number) =>
+  (action$: Observable<AnyAction>) =>
+    monitorHeartbeat$(timeoutMilliseconds)(action$).pipe(
+      switchMap(action => timer(0, timeoutMilliseconds * 2).pipe(
+        tap(n => log.verbose('PuppetService', 'epicRecoverReset$() actions.reset() emitted #%d', n)),
+        map(() => PuppetDuck.actions.RESET_COMMAND(
+          action.meta.puppetId,
+          'epicRecoverReset$',
+        )),
+        takeUntil(heartbeat$(action$)),
+        takeUntil(stateInactive$(action$)),
+      )),
+    )
+
+export {
+  monitorHeartbeat$,
+  epicRecoverReset$,
+  epicRecoverDing$,
+}