@activeledger/activerestore 5.0.2 → 5.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/es/interfaces/quick-restore.interface.d.ts +1 -1
  2. package/es/interfaces/quick-restore.interface.d.ts.map +1 -1
  3. package/es/modules/contract/contract.d.ts +1 -2
  4. package/es/modules/contract/contract.d.ts.map +1 -1
  5. package/es/modules/contract/contract.js +36 -27
  6. package/es/modules/contract/contract.js.map +1 -1
  7. package/es/modules/helper/helper.d.ts +3 -0
  8. package/es/modules/helper/helper.d.ts.map +1 -1
  9. package/es/modules/helper/helper.js +18 -0
  10. package/es/modules/helper/helper.js.map +1 -1
  11. package/es/modules/helper/throttle.d.ts +20 -0
  12. package/es/modules/helper/throttle.d.ts.map +1 -0
  13. package/es/modules/helper/throttle.js +54 -0
  14. package/es/modules/helper/throttle.js.map +1 -0
  15. package/es/modules/quick-restore/quick-restore.d.ts +2 -2
  16. package/es/modules/quick-restore/quick-restore.d.ts.map +1 -1
  17. package/es/modules/quick-restore/quick-restore.js +150 -79
  18. package/es/modules/quick-restore/quick-restore.js.map +1 -1
  19. package/lib/interfaces/quick-restore.interface.d.ts +1 -1
  20. package/lib/interfaces/quick-restore.interface.d.ts.map +1 -1
  21. package/lib/modules/contract/contract.d.ts +1 -2
  22. package/lib/modules/contract/contract.d.ts.map +1 -1
  23. package/lib/modules/contract/contract.js +36 -60
  24. package/lib/modules/contract/contract.js.map +1 -1
  25. package/lib/modules/helper/helper.d.ts +3 -0
  26. package/lib/modules/helper/helper.d.ts.map +1 -1
  27. package/lib/modules/helper/helper.js +18 -0
  28. package/lib/modules/helper/helper.js.map +1 -1
  29. package/lib/modules/helper/throttle.d.ts +20 -0
  30. package/lib/modules/helper/throttle.d.ts.map +1 -0
  31. package/lib/modules/helper/throttle.js +58 -0
  32. package/lib/modules/helper/throttle.js.map +1 -0
  33. package/lib/modules/quick-restore/quick-restore.d.ts +2 -2
  34. package/lib/modules/quick-restore/quick-restore.d.ts.map +1 -1
  35. package/lib/modules/quick-restore/quick-restore.js +150 -79
  36. package/lib/modules/quick-restore/quick-restore.js.map +1 -1
  37. package/package.json +7 -5
@@ -9,7 +9,7 @@ export interface IStreamInformation {
9
9
  }
10
10
  export interface INetworkData {
11
11
  documents: any;
12
- volatile: any;
12
+ contracts?: any[];
13
13
  }
14
14
  interface IReductionRevisionData {
15
15
  [revision: string]: number;
@@ -1 +1 @@
1
- {"version":3,"file":"quick-restore.interface.d.ts","sourceRoot":"","sources":["../../src/interfaces/quick-restore.interface.ts"],"names":[],"mappings":"AAuBA,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,cAAc,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,GAAG,CAAC;IACf,QAAQ,EAAE,GAAG,CAAC;CACf;AAED,UAAU,sBAAsB;IAC9B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,sBAAsB,CAAC;CAC5C;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,cAAc,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,EAAE,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC;CAC9C"}
1
+ {"version":3,"file":"quick-restore.interface.d.ts","sourceRoot":"","sources":["../../src/interfaces/quick-restore.interface.ts"],"names":[],"mappings":"AAuBA,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,cAAc,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,GAAG,CAAC;IAEf,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC;CACnB;AAED,UAAU,sBAAsB;IAC9B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,sBAAsB,CAAC;CAC5C;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,cAAc,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,EAAE,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC;CAC9C"}
@@ -1,5 +1,4 @@
1
1
  export declare class Contract {
2
- static rebuild(contract: any): void;
3
- private static transpile;
2
+ static rebuild(contract: any): Promise<void>;
4
3
  }
5
4
  //# sourceMappingURL=contract.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../../src/modules/contract/contract.ts"],"names":[],"mappings":"AAgCA,qBAAa,QAAQ;IAOnB,OAAc,OAAO,CAAC,QAAQ,EAAE,GAAG,GAAG,IAAI,CA6BzC;IAUD,OAAO,CAAC,MAAM,CAAC,SAAS;CAoBzB"}
1
+ {"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../../src/modules/contract/contract.ts"],"names":[],"mappings":"AAqCA,qBAAa,QAAQ;IASnB,OAAoB,OAAO,CAAC,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CA+CxD;CACF"}
@@ -1,33 +1,42 @@
1
- import * as fs from "fs";
2
- import * as ts from "typescript-api";
1
+ import { ActiveLogger } from "@activeledger/activelogger";
2
+ import { ContractFiles, ContractSource, } from "@activeledger/activecontracts/lib/source";
3
+ import { Provider } from "../provider/provider";
3
4
  export class Contract {
4
- static rebuild(contract) {
5
- let versions = Object.keys(contract.contract);
6
- for (let i = 0; i < versions.length; i++) {
7
- const version = versions[i];
8
- const code = Contract.transpile(contract.contract[version]);
9
- if (!fs.existsSync("contracts"))
10
- fs.mkdirSync("contracts");
11
- if (!fs.existsSync(`contracts/${contract.namespace}`))
12
- fs.mkdirSync(`contracts/${contract.namespace}`);
13
- fs.writeFileSync(`contracts/${contract.namespace}/${contract._id}.js`, code);
14
- fs.writeFileSync(`contracts/${contract.namespace}/${contract._id}@${version}.js`, code);
5
+ static async rebuild(contract) {
6
+ const files = new ContractFiles("contracts");
7
+ const fetchUmid = (umid) => Provider.database.get(`${umid}:umid`).catch(() => undefined);
8
+ for (const version of Object.keys(contract.contract)) {
9
+ try {
10
+ const source = await ContractSource.resolve(contract, version, fetchUmid);
11
+ files.write(contract.namespace, contract._id, version, ContractSource.transpile(source.base64, source.target));
12
+ }
13
+ catch (e) {
14
+ ActiveLogger.warn(`Cannot rebuild ${contract._id}@${version}, its file is missing: ${e.message}`);
15
+ }
15
16
  }
16
- }
17
- static transpile(code) {
18
- return ts.transpileModule(Buffer.from(code, "base64").toString(), {
19
- compilerOptions: {
20
- alwaysStrict: true,
21
- strictNullChecks: true,
22
- noImplicitAny: true,
23
- removeComments: true,
24
- module: ts.ModuleKind.CommonJS,
25
- moduleResolution: ts.ModuleResolutionKind.Classic,
26
- target: ts.ScriptTarget.ES2017,
27
- esModuleInterop: false,
28
- ignoreDeprecations: "6.0"
17
+ try {
18
+ const locks = contract.locks;
19
+ const held = [];
20
+ if (locks && locks.all === true)
21
+ held.push("");
22
+ if (locks && Array.isArray(locks.versions)) {
23
+ for (const v of locks.versions) {
24
+ if (typeof v === "string")
25
+ held.push("@" + v);
26
+ }
29
27
  }
30
- }).outputText;
28
+ for (const version of held) {
29
+ try {
30
+ files.lock(contract.namespace, contract._id, version);
31
+ }
32
+ catch (e) {
33
+ ActiveLogger.warn(`Cannot restore the lock on ${contract._id}${version}: ${e.message}`);
34
+ }
35
+ }
36
+ }
37
+ catch (e) {
38
+ ActiveLogger.warn(`Cannot restore the locks on ${contract._id}: ${e.message}`);
39
+ }
31
40
  }
32
41
  }
33
42
  //# sourceMappingURL=contract.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"contract.js","sourceRoot":"","sources":["../../../src/modules/contract/contract.ts"],"names":[],"mappings":"AAuBA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAQrC,MAAM,OAAO,QAAQ;IAOZ,MAAM,CAAC,OAAO,CAAC,QAAa;QAEjC,IAAI,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAG9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAEzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAG5D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC;gBAAE,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;YAG3D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,QAAQ,CAAC,SAAS,EAAE,CAAC;gBACnD,EAAE,CAAC,SAAS,CAAC,aAAa,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;YAGlD,EAAE,CAAC,aAAa,CACd,aAAa,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,GAAG,KAAK,EACpD,IAAI,CACL,CAAC;YAGF,EAAE,CAAC,aAAa,CACd,aAAa,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,GAAG,IAAI,OAAO,KAAK,EAC/D,IAAI,CACL,CAAC;QACJ,CAAC;IACH,CAAC;IAUO,MAAM,CAAC,SAAS,CAAC,IAAY;QAEnC,OAAO,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,QAAQ,EAAE,EAAE;YAChE,eAAe,EAAE;gBACf,YAAY,EAAE,IAAI;gBAClB,gBAAgB,EAAE,IAAI;gBACtB,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,IAAI;gBACpB,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ;gBAC9B,gBAAgB,EAAE,EAAE,CAAC,oBAAoB,CAAC,OAAO;gBACjD,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM;gBAK9B,eAAe,EAAE,KAAK;gBACtB,kBAAkB,EAAE,KAAK;aAC1B;SACF,CAAC,CAAC,UAAU,CAAC;IAChB,CAAC;CACF"}
1
+ {"version":3,"file":"contract.js","sourceRoot":"","sources":["../../../src/modules/contract/contract.ts"],"names":[],"mappings":"AAuBA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EACL,aAAa,EACb,cAAc,GACf,MAAM,0CAA0C,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAShD,MAAM,OAAO,QAAQ;IASZ,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,QAAa;QACvC,MAAM,KAAK,GAAG,IAAI,aAAa,CAAC,WAAW,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,EAAE,CACjC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAE/D,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,CACzC,QAAQ,EACR,OAAO,EACP,SAAS,CACV,CAAC;gBACF,KAAK,CAAC,KAAK,CACT,QAAQ,CAAC,SAAS,EAClB,QAAQ,CAAC,GAAG,EACZ,OAAO,EACP,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CACvD,CAAC;YACJ,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBAChB,YAAY,CAAC,IAAI,CACf,kBAAkB,QAAQ,CAAC,GAAG,IAAI,OAAO,0BAA0B,CAAC,CAAC,OAAO,EAAE,CAC/E,CAAC;YACJ,CAAC;QACH,CAAC;QAGD,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;YAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,IAAI,KAAK,IAAI,KAAK,CAAC,GAAG,KAAK,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/C,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC3C,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;oBAC/B,IAAI,OAAO,CAAC,KAAK,QAAQ;wBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACH,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACxD,CAAC;gBAAC,OAAO,CAAM,EAAE,CAAC;oBAChB,YAAY,CAAC,IAAI,CACf,8BAA8B,QAAQ,CAAC,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,OAAO,EAAE,CACrE,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,YAAY,CAAC,IAAI,CAAC,+BAA+B,QAAQ,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;CACF"}
@@ -1,8 +1,11 @@
1
+ import { IConsensusData, IStreamInformation } from "../../interfaces/quick-restore.interface";
1
2
  export declare class Helper {
2
3
  static verbose: boolean;
3
4
  static output(message: string, other?: any): void;
4
5
  static getNeighbourCount: (dontIncludeSelf: boolean) => number;
5
6
  static metConsensus: (votes: number, dontIncludeSelf?: boolean) => boolean;
7
+ static votesNeeded: () => number;
8
+ static majority(nodes: IStreamInformation[], needed: number): IConsensusData[];
6
9
  static replayEvents(umidDoc: any): Promise<void>;
7
10
  }
8
11
  //# sourceMappingURL=helper.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../../../src/modules/helper/helper.ts"],"names":[],"mappings":"AAgCA,qBAAa,MAAM;IACjB,OAAc,OAAO,EAAE,OAAO,CAAS;IASvC,OAAc,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI,CAIvD;IAQD,OAAc,iBAAiB,oBAAqB,OAAO,YACe;IAQ1E,OAAc,YAAY,UACjB,MAAM,oBACI,OAAO,aAGQ;IAoBlC,OAAoB,YAAY,CAAC,OAAO,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAsB5D;CACF"}
1
+ {"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../../../src/modules/helper/helper.ts"],"names":[],"mappings":"AA0BA,OAAO,EACL,cAAc,EACd,kBAAkB,EACnB,MAAM,0CAA0C,CAAC;AAQlD,qBAAa,MAAM;IACjB,OAAc,OAAO,EAAE,OAAO,CAAS;IASvC,OAAc,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI,CAIvD;IAQD,OAAc,iBAAiB,oBAAqB,OAAO,YACe;IAQ1E,OAAc,YAAY,UACjB,MAAM,oBACI,OAAO,aAGQ;IAYlC,OAAc,WAAW,QAAO,MAAM,CASpC;IAcF,OAAc,QAAQ,CACpB,KAAK,EAAE,kBAAkB,EAAE,EAC3B,MAAM,EAAE,MAAM,GACb,cAAc,EAAE,CAalB;IAoBD,OAAoB,YAAY,CAAC,OAAO,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAsB5D;CACF"}
@@ -1,4 +1,5 @@
1
1
  import { ActiveLogger } from "@activeledger/activelogger";
2
+ import { spiConsensus } from "@activeledger/activesync";
2
3
  import { Provider } from "../provider/provider";
3
4
  export class Helper {
4
5
  static verbose = false;
@@ -10,6 +11,23 @@ export class Helper {
10
11
  static getNeighbourCount = (dontIncludeSelf) => dontIncludeSelf ? Provider.neighbourCount - 1 : Provider.neighbourCount;
11
12
  static metConsensus = (votes, dontIncludeSelf = false) => (votes / Helper.getNeighbourCount(dontIncludeSelf)) * 100 >=
12
13
  Provider.consensusReachedAmount;
14
+ static votesNeeded = () => {
15
+ const nodes = Helper.getNeighbourCount(false);
16
+ if (!(nodes > 0))
17
+ return Infinity;
18
+ const needed = Math.max(1, Math.ceil((Provider.consensusReachedAmount / 100) * nodes - 1));
19
+ return Number.isFinite(needed) && needed > 0 ? needed : Infinity;
20
+ };
21
+ static majority(nodes, needed) {
22
+ const answers = nodes.map((n) => (Array.isArray(n?.streams) ? n.streams : [])
23
+ .filter((s) => s && s.id && s.id !== "undefined" && s.rev && !String(s.id).endsWith(":volatile"))
24
+ .map((s) => ({ _id: s.id, _rev: s.rev })));
25
+ const { winners } = spiConsensus(answers, needed);
26
+ return Object.keys(winners).map((stream) => ({
27
+ stream,
28
+ revision: winners[stream].rev
29
+ }));
30
+ }
13
31
  static async replayEvents(umidDoc) {
14
32
  const events = umidDoc?.events;
15
33
  if (!Array.isArray(events) || !events.length) {
@@ -1 +1 @@
1
- {"version":3,"file":"helper.js","sourceRoot":"","sources":["../../../src/modules/helper/helper.ts"],"names":[],"mappings":"AAuBA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAQhD,MAAM,OAAO,MAAM;IACV,MAAM,CAAC,OAAO,GAAY,KAAK,CAAC;IAShC,MAAM,CAAC,MAAM,CAAC,OAAe,EAAE,KAAW;QAC/C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAQM,MAAM,CAAC,iBAAiB,GAAG,CAAC,eAAwB,EAAE,EAAE,CAC7D,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC;IAQnE,MAAM,CAAC,YAAY,GAAG,CAC3B,KAAa,EACb,eAAe,GAAY,KAAK,EAChC,EAAE,CACF,CAAC,KAAK,GAAG,MAAM,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,GAAG,GAAG;QACzD,QAAQ,CAAC,sBAAsB,CAAC;IAoB3B,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAY;QAC3C,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC/B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7C,OAAO;QACT,CAAC;QAMD,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,YAAY,CAAC,KAAK,CAChB,KAAK,EACL,0BAA0B,KAAK,EAAE,GAAG,sBAAsB,OAAO,EAAE,IAAI,EAAE,KAAK,IAAI,OAAO,EAAE,GAAG,EAAE,CACjG,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
1
+ {"version":3,"file":"helper.js","sourceRoot":"","sources":["../../../src/modules/helper/helper.ts"],"names":[],"mappings":"AAuBA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAYhD,MAAM,OAAO,MAAM;IACV,MAAM,CAAC,OAAO,GAAY,KAAK,CAAC;IAShC,MAAM,CAAC,MAAM,CAAC,OAAe,EAAE,KAAW;QAC/C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAQM,MAAM,CAAC,iBAAiB,GAAG,CAAC,eAAwB,EAAE,EAAE,CAC7D,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC;IAQnE,MAAM,CAAC,YAAY,GAAG,CAC3B,KAAa,EACb,eAAe,GAAY,KAAK,EAChC,EAAE,CACF,CAAC,KAAK,GAAG,MAAM,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,GAAG,GAAG;QACzD,QAAQ,CAAC,sBAAsB,CAAC;IAY3B,MAAM,CAAC,WAAW,GAAG,GAAW,EAAE;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;YAAE,OAAO,QAAQ,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CACrB,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,sBAAsB,GAAG,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAC/D,CAAC;QAEF,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;IACnE,CAAC,CAAC;IAcK,MAAM,CAAC,QAAQ,CACpB,KAA2B,EAC3B,MAAc;QAEd,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9B,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;aAGzC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,KAAK,WAAW,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;aAChG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAC5C,CAAC;QACF,MAAM,EAAE,OAAO,EAAE,GAAG,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC3C,MAAM;YACN,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG;SAC9B,CAAC,CAAC,CAAC;IACN,CAAC;IAoBM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAY;QAC3C,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC/B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7C,OAAO;QACT,CAAC;QAMD,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,YAAY,CAAC,KAAK,CAChB,KAAK,EACL,0BAA0B,KAAK,EAAE,GAAG,sBAAsB,OAAO,EAAE,IAAI,EAAE,KAAK,IAAI,OAAO,EAAE,GAAG,EAAE,CACjG,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,20 @@
1
+ export interface IThrottleOptions {
2
+ pauseMs: number;
3
+ batch: number;
4
+ batchPauseMs: number;
5
+ slowMs: number;
6
+ maxBackoffMs: number;
7
+ }
8
+ export declare const DEFAULT_THROTTLE: IThrottleOptions;
9
+ export declare class Throttle {
10
+ private readonly options;
11
+ private readonly sleep;
12
+ private count;
13
+ private backoff;
14
+ constructor(options: IThrottleOptions | null, sleep?: (ms: number) => Promise<void>);
15
+ static fromConfig(section: any, fast?: boolean, sleep?: (ms: number) => Promise<void>): Throttle;
16
+ get backingOff(): number;
17
+ wait(): Promise<void>;
18
+ record(durationMs: number): void;
19
+ }
20
+ //# sourceMappingURL=throttle.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"throttle.d.ts","sourceRoot":"","sources":["../../../src/modules/helper/throttle.ts"],"names":[],"mappings":"AAWA,MAAM,WAAW,gBAAgB;IAE/B,OAAO,EAAE,MAAM,CAAC;IAEhB,KAAK,EAAE,MAAM,CAAC;IAEd,YAAY,EAAE,MAAM,CAAC;IAErB,MAAM,EAAE,MAAM,CAAC;IAEf,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,eAAO,MAAM,gBAAgB,EAAE,gBAM9B,CAAC;AAEF,qBAAa,QAAQ;IAKjB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IALxB,OAAO,CAAC,KAAK,CAAK;IAClB,OAAO,CAAC,OAAO,CAAK;IAEpB,YACmB,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,KAAK,GAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CACD,EACjD;IAMJ,OAAc,UAAU,CACtB,OAAO,EAAE,GAAG,EACZ,IAAI,UAAQ,EACZ,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GACpC,QAAQ,CAeV;IAGD,IAAW,UAAU,IAAI,MAAM,CAE9B;IAGY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAOjC;IAGM,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAOtC;CACF"}
@@ -0,0 +1,54 @@
1
+ export const DEFAULT_THROTTLE = {
2
+ pauseMs: 5,
3
+ batch: 100,
4
+ batchPauseMs: 200,
5
+ slowMs: 400,
6
+ maxBackoffMs: 5000,
7
+ };
8
+ export class Throttle {
9
+ options;
10
+ sleep;
11
+ count = 0;
12
+ backoff = 0;
13
+ constructor(options, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))) {
14
+ this.options = options;
15
+ this.sleep = sleep;
16
+ }
17
+ static fromConfig(section, fast = false, sleep) {
18
+ if (fast || section?.throttle === false)
19
+ return new Throttle(null, sleep);
20
+ const given = section?.throttle && typeof section.throttle === "object" ? section.throttle : {};
21
+ const pick = (key) => Number.isFinite(given[key]) && given[key] >= 0 ? given[key] : DEFAULT_THROTTLE[key];
22
+ return new Throttle({
23
+ pauseMs: pick("pauseMs"),
24
+ batch: Math.max(1, pick("batch")),
25
+ batchPauseMs: pick("batchPauseMs"),
26
+ slowMs: pick("slowMs"),
27
+ maxBackoffMs: pick("maxBackoffMs"),
28
+ }, sleep);
29
+ }
30
+ get backingOff() {
31
+ return this.backoff;
32
+ }
33
+ async wait() {
34
+ const o = this.options;
35
+ if (!o)
36
+ return;
37
+ this.count++;
38
+ let ms = o.pauseMs + this.backoff;
39
+ if (this.count % o.batch === 0)
40
+ ms += o.batchPauseMs;
41
+ if (ms > 0)
42
+ await this.sleep(ms);
43
+ }
44
+ record(durationMs) {
45
+ const o = this.options;
46
+ if (!o)
47
+ return;
48
+ this.backoff =
49
+ durationMs > o.slowMs
50
+ ? Math.min(o.maxBackoffMs, Math.max(50, this.backoff * 2))
51
+ : Math.floor(this.backoff / 2);
52
+ }
53
+ }
54
+ //# sourceMappingURL=throttle.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"throttle.js","sourceRoot":"","sources":["../../../src/modules/helper/throttle.ts"],"names":[],"mappings":"AAwBA,MAAM,CAAC,MAAM,gBAAgB,GAAqB;IAChD,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,GAAG;IACV,YAAY,EAAE,GAAG;IACjB,MAAM,EAAE,GAAG;IACX,YAAY,EAAE,IAAI;CACnB,CAAC;AAEF,MAAM,OAAO,QAAQ;IAKA,OAAO;IACP,KAAK;IALhB,KAAK,GAAG,CAAC,CAAC;IACV,OAAO,GAAG,CAAC,CAAC;IAEpB,YACmB,OAAgC,EAChC,KAAK,GAAkC,CAAC,EAAE,EAAE,EAAE,CAC7D,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;uBAFlC,OAAO;qBACP,KAAK;IAErB,CAAC;IAMG,MAAM,CAAC,UAAU,CACtB,OAAY,EACZ,IAAI,GAAG,KAAK,EACZ,KAAqC;QAErC,IAAI,IAAI,IAAI,OAAO,EAAE,QAAQ,KAAK,KAAK;YAAE,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,OAAO,EAAE,QAAQ,IAAI,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,GAAG,CAAC,GAA2B,EAAE,EAAE,CAC3C,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACtF,OAAO,IAAI,QAAQ,CACjB;YACE,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;YACxB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACjC,YAAY,EAAE,IAAI,CAAC,cAAc,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC;YACtB,YAAY,EAAE,IAAI,CAAC,cAAc,CAAC;SACnC,EACD,KAAK,CACN,CAAC;IACJ,CAAC;IAGD,IAAW,UAAU;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAGM,KAAK,CAAC,IAAI;QACf,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QACvB,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAClC,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC;YAAE,EAAE,IAAI,CAAC,CAAC,YAAY,CAAC;QACrD,IAAI,EAAE,GAAG,CAAC;YAAE,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAGM,MAAM,CAAC,UAAkB;QAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QACvB,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,IAAI,CAAC,OAAO;YACV,UAAU,GAAG,CAAC,CAAC,MAAM;gBACnB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBAC1D,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IACrC,CAAC;CACF"}
@@ -2,13 +2,13 @@ export declare class QuickRestore {
2
2
  constructor();
3
3
  private isNotThisNode;
4
4
  private runQuickFullRestore;
5
+ private restoreHistory;
5
6
  private fetchDocuments;
7
+ private static isStateDocument;
6
8
  private static adoptDocument;
7
9
  private uploadData;
8
10
  private handleNetworkData;
9
- private checkConsensus;
10
11
  private processRebuildPromises;
11
- private reduceStreamInformation;
12
12
  private getRebuildData;
13
13
  }
14
14
  //# sourceMappingURL=quick-restore.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"quick-restore.d.ts","sourceRoot":"","sources":["../../../src/modules/quick-restore/quick-restore.ts"],"names":[],"mappings":"AA6CA,qBAAa,YAAY;IAIvB,cAEC;IAOD,OAAO,CAAC,aAAa,CACmB;YAQ1B,mBAAmB;IAwCjC,OAAO,CAAC,cAAc;mBAgDD,aAAa;IAiClC,OAAO,CAAC,UAAU;IA2DlB,OAAO,CAAC,iBAAiB;IA0CzB,OAAO,CAAC,cAAc;IA6CtB,OAAO,CAAC,sBAAsB;IA+B9B,OAAO,CAAC,uBAAuB;IAuD/B,OAAO,CAAC,cAAc;CAoCvB"}
1
+ {"version":3,"file":"quick-restore.d.ts","sourceRoot":"","sources":["../../../src/modules/quick-restore/quick-restore.ts"],"names":[],"mappings":"AAwDA,qBAAa,YAAY;IAIvB,cAEC;IAOD,OAAO,CAAC,aAAa,CACmB;YAQ1B,mBAAmB;YA6EnB,cAAc;IAuF5B,OAAO,CAAC,cAAc;IAwHtB,OAAO,CAAC,MAAM,CAAC,eAAe;mBAQT,aAAa;IAiClC,OAAO,CAAC,UAAU;YAuDJ,iBAAiB;IA2C/B,OAAO,CAAC,sBAAsB;IA+B9B,OAAO,CAAC,cAAc;CAoCvB"}
@@ -3,6 +3,14 @@ import { Provider } from "../provider/provider";
3
3
  import { ActiveNetwork } from "@activeledger/activenetwork";
4
4
  import { Helper } from "../helper/helper";
5
5
  import { Contract } from "../contract/contract";
6
+ import { Throttle } from "../helper/throttle";
7
+ import { walkUmidHistory } from "@activeledger/activesync";
8
+ import { ActiveOptions } from "@activeledger/activeoptions";
9
+ const FETCH_ATTEMPTS = 3;
10
+ const FETCH_RETRY_MS = 500;
11
+ const PROGRESS_EVERY = 500;
12
+ const HISTORY_WALK_LIMIT = 1000000;
13
+ const HISTORY_HOP_DELAY_MS = 50;
6
14
  export class QuickRestore {
7
15
  constructor() {
8
16
  this.runQuickFullRestore();
@@ -19,13 +27,90 @@ export class QuickRestore {
19
27
  .map(preparePromise);
20
28
  try {
21
29
  const streamInformation = await this.processRebuildPromises(promises);
22
- const consensusData = await this.checkConsensus(streamInformation);
23
- await this.fetchDocuments(consensusData);
30
+ const needed = Helper.votesNeeded();
31
+ if (!Number.isFinite(needed)) {
32
+ ActiveLogger.error("consensus.reached is not set in the config file, so no revision can be a majority and nothing will be restored");
33
+ }
34
+ const consensusData = Helper.majority(streamInformation, needed);
35
+ const skipHistory = ActiveOptions.get("skip-history", false);
36
+ const historyOnly = ActiveOptions.get("history-only", false);
37
+ if (skipHistory && historyOnly) {
38
+ ActiveLogger.error("--skip-history and --history-only together leave nothing to do");
39
+ return;
40
+ }
41
+ if (!historyOnly)
42
+ await this.fetchDocuments(consensusData);
24
43
  const duration = Date.now() - startTime;
25
- ActiveLogger.info(`Rebuild Complete; Duration: ${duration}ms`);
44
+ ActiveLogger.info(historyOnly
45
+ ? "Rebuild skipped (--history-only)"
46
+ : `Rebuild Complete; Duration: ${duration}ms`);
47
+ await this.restoreHistory(consensusData);
26
48
  }
27
49
  catch (error) {
28
- ActiveLogger.error(error, "There was an error running quick full restore");
50
+ ActiveLogger.error(error, `There was an error running quick full restore: ${error?.message ?? JSON.stringify(error)}`);
51
+ }
52
+ }
53
+ async restoreHistory(consensusData) {
54
+ const section = ActiveOptions.get("restore", {});
55
+ if (section?.history === false ||
56
+ ActiveOptions.get("skip-history", false)) {
57
+ ActiveLogger.info("History pass skipped");
58
+ return;
59
+ }
60
+ const given = section?.history && typeof section.history === "object" ? section.history : {};
61
+ if (!Number.isFinite(given.minAgree) && !Number.isFinite(Helper.votesNeeded())) {
62
+ ActiveLogger.error("consensus.reached is not set in the config file, so the history pass has no bar to hold documents to and was skipped");
63
+ return;
64
+ }
65
+ const ctx = {
66
+ store: Provider.database,
67
+ events: Provider.eventDatabase,
68
+ peers: {
69
+ umid: (umid) => Provider.network.neighbourhood.knockAll(`umid/${umid}`, null, true),
70
+ },
71
+ walkLimit: Number.isFinite(given.walkLimit) ? given.walkLimit : HISTORY_WALK_LIMIT,
72
+ hopDelayMs: Number.isFinite(given.hopDelayMs) ? given.hopDelayMs : HISTORY_HOP_DELAY_MS,
73
+ continuePastHeld: true,
74
+ minAgree: Number.isFinite(given.minAgree) ? given.minAgree : Helper.votesNeeded(),
75
+ };
76
+ const throttle = Throttle.fromConfig(section, ActiveOptions.get("fast", false));
77
+ const streams = consensusData.filter((c) => QuickRestore.isStateDocument(c.stream) && c.stream.indexOf(":") === -1);
78
+ ActiveLogger.info(`History pass: ${streams.length} stream(s)`);
79
+ const started = Date.now();
80
+ let recovered = 0;
81
+ const incomplete = [];
82
+ for (let i = 0; i < streams.length; i++) {
83
+ const id = streams[i].stream;
84
+ if (i && i % PROGRESS_EVERY === 0) {
85
+ ActiveLogger.info(`History progress: ${i}/${streams.length} (${recovered} umid(s) recovered)`);
86
+ }
87
+ let umid;
88
+ try {
89
+ umid = (await Provider.database.get(`${id}:stream`))?.umid;
90
+ }
91
+ catch {
92
+ continue;
93
+ }
94
+ if (!umid)
95
+ continue;
96
+ const fetchStarted = Date.now();
97
+ try {
98
+ const walk = await walkUmidHistory(ctx, id, umid);
99
+ recovered += walk.recovered.length;
100
+ if (!walk.complete)
101
+ incomplete.push(`${id} (${walk.stoppedAt})`);
102
+ if (walk.recovered.length) {
103
+ await throttle.wait();
104
+ throttle.record(Date.now() - fetchStarted);
105
+ }
106
+ }
107
+ catch (error) {
108
+ incomplete.push(`${id} (${error?.message ?? "failed"})`);
109
+ }
110
+ }
111
+ ActiveLogger.info(`History Complete; Duration: ${Date.now() - started}ms; ${recovered} umid(s) recovered`);
112
+ if (incomplete.length) {
113
+ ActiveLogger.warn(`${incomplete.length} stream(s) have a chain that could not be completed, e.g. ${incomplete.slice(0, 3).join("; ")}`);
29
114
  }
30
115
  }
31
116
  fetchDocuments(consensusData) {
@@ -33,13 +118,59 @@ export class QuickRestore {
33
118
  return new Promise(async (resolve, reject) => {
34
119
  try {
35
120
  Helper.output("Fetch documents: Conesensus data", consensusData);
121
+ const contracts = [];
122
+ const unrestored = [];
123
+ const throttle = Throttle.fromConfig(ActiveOptions.get("restore", {}), ActiveOptions.get("fast", false));
124
+ let held = 0;
125
+ let fetched = 0;
36
126
  for (let i = consensusData.length; i--;) {
37
127
  const activity = consensusData[i];
128
+ const done = consensusData.length - i;
129
+ if (done % PROGRESS_EVERY === 0) {
130
+ ActiveLogger.info(`Restore progress: ${done}/${consensusData.length} (${fetched} fetched, ${held} already held)`);
131
+ }
132
+ const local = await Provider.database.get(activity.stream).catch(() => null);
133
+ if (local && local._rev === activity.revision) {
134
+ held++;
135
+ if (typeof local._id === "string" &&
136
+ QuickRestore.isStateDocument(local._id) &&
137
+ local.namespace &&
138
+ local.contract &&
139
+ local.compiled) {
140
+ contracts.push(local);
141
+ }
142
+ continue;
143
+ }
144
+ await throttle.wait();
145
+ const fetchStarted = Date.now();
38
146
  ActiveLogger.info(`[Fetching] ${activity.stream}`);
39
- const streamDataPromise = await streamData(activity);
147
+ let networkData = { documents: [] };
148
+ for (let attempt = 1; attempt <= FETCH_ATTEMPTS; attempt++) {
149
+ const answers = await streamData(activity).catch(() => []);
150
+ networkData = await this.handleNetworkData(answers);
151
+ if (networkData.documents.length)
152
+ break;
153
+ if (attempt < FETCH_ATTEMPTS) {
154
+ await new Promise((r) => setTimeout(r, FETCH_RETRY_MS * attempt));
155
+ }
156
+ }
157
+ if (!networkData.documents.length) {
158
+ unrestored.push(activity.stream);
159
+ ActiveLogger.error(`[Not restored] ${activity.stream} at ${activity.revision}: no neighbour supplied it after ${FETCH_ATTEMPTS} attempts`);
160
+ continue;
161
+ }
40
162
  ActiveLogger.info(`[Fetching Complete] ${activity.stream}`);
41
- const networkData = this.handleNetworkData(streamDataPromise);
42
163
  await this.uploadData(networkData);
164
+ fetched++;
165
+ throttle.record(Date.now() - fetchStarted);
166
+ contracts.push(...(networkData.contracts || []));
167
+ }
168
+ for (const contract of contracts) {
169
+ await Contract.rebuild(contract);
170
+ }
171
+ ActiveLogger.info(`Restored ${fetched} stream(s); ${held} already held at the network's revision`);
172
+ if (unrestored.length) {
173
+ ActiveLogger.error(`${unrestored.length} stream(s) could not be restored, run the restore again to retry them`);
43
174
  }
44
175
  resolve();
45
176
  }
@@ -48,6 +179,11 @@ export class QuickRestore {
48
179
  }
49
180
  });
50
181
  }
182
+ static isStateDocument(id) {
183
+ return (id.indexOf(":stream") === -1 &&
184
+ id.indexOf(":umid") === -1 &&
185
+ id.indexOf("_design") === -1);
186
+ }
51
187
  static async adoptDocument(doc) {
52
188
  let local = null;
53
189
  try {
@@ -74,77 +210,40 @@ export class QuickRestore {
74
210
  try {
75
211
  for (let i = networkData.documents.length; i--;) {
76
212
  const doc = networkData.documents[i];
77
- await QuickRestore.adoptDocument(doc);
78
213
  if (doc._id && doc._id.indexOf(":umid") !== -1) {
79
214
  await Helper.replayEvents(doc);
80
215
  }
216
+ await QuickRestore.adoptDocument(doc);
81
217
  }
82
218
  }
83
219
  catch (error) {
84
220
  ActiveLogger.error("Error occurred in processNetworkData: Uploading documents to database");
85
221
  return reject(error);
86
222
  }
87
- try {
88
- await Provider.database.bulkDocs(networkData.volatile, {
89
- new_edits: true
90
- });
91
- resolve();
92
- }
93
- catch (error) {
94
- ActiveLogger.error("Error occurred in processNetworkData: Uploading volatile to database");
95
- reject(error);
96
- }
223
+ resolve();
97
224
  });
98
225
  }
99
- handleNetworkData(activityData) {
100
- const hasNotErrored = (data) => (!data.error);
226
+ async handleNetworkData(activityData) {
227
+ const hasNotErrored = (data) => !!data && typeof data._id === "string" && !data.error;
101
228
  const isNotStreamOrUmid = (data) => data._id.indexOf(":stream") === -1 &&
102
229
  data._id.indexOf(":umid") === -1 &&
103
230
  data._id.indexOf("_design") === -1;
104
231
  const hasRequiredData = (data) => !!(data.namespace && data.contract && data.compiled);
105
232
  const documents = [];
106
- const volatile = [];
233
+ const contracts = [];
107
234
  for (let r = activityData.length; r--;) {
108
235
  const data = activityData[r];
109
236
  if (hasNotErrored(data)) {
110
237
  documents.push(data);
111
238
  if (isNotStreamOrUmid(data)) {
112
- volatile.push({ _id: `${data._id}:volatile` });
113
239
  hasRequiredData(data)
114
- ? Contract.rebuild(data)
240
+ ? contracts.push(data)
115
241
  : Helper.output("Data does not have required fields", data);
116
242
  }
117
243
  break;
118
244
  }
119
245
  }
120
- return { documents, volatile };
121
- }
122
- checkConsensus(streamInformation) {
123
- const streamIdCorrect = (streamId) => !!(streamId && streamId !== "undefined");
124
- return new Promise((resolve) => {
125
- const consensusReached = [];
126
- const streams = Object.keys(streamInformation);
127
- for (let i = streams.length; i--;) {
128
- const streamId = streams[i];
129
- if (streamIdCorrect(streamId)) {
130
- const stream = streamInformation[streamId];
131
- const revisions = Object.keys(stream);
132
- for (let i = revisions.length; i--;) {
133
- const revision = revisions[i];
134
- const vote = stream[revision];
135
- Helper.output("Check Consensus: Adding", {
136
- stream: streamId,
137
- revision
138
- });
139
- if (Helper.metConsensus(vote)) {
140
- consensusReached.push({ stream: streamId, revision });
141
- break;
142
- }
143
- }
144
- }
145
- }
146
- resolve(consensusReached);
147
- });
246
+ return { documents, contracts };
148
247
  }
149
248
  processRebuildPromises(promises) {
150
249
  const streamDataCorrect = (data) => !!data;
@@ -154,7 +253,7 @@ export class QuickRestore {
154
253
  const cleanedStreamInformation = streamInformation.filter(streamDataCorrect);
155
254
  Helper.output("cleanedStreamInformation", cleanedStreamInformation);
156
255
  Object.keys(cleanedStreamInformation).length > 0
157
- ? resolve(this.reduceStreamInformation(cleanedStreamInformation))
256
+ ? resolve(cleanedStreamInformation)
158
257
  : reject("No data to process");
159
258
  }
160
259
  catch (error) {
@@ -162,34 +261,6 @@ export class QuickRestore {
162
261
  }
163
262
  });
164
263
  }
165
- reduceStreamInformation(streamInformation) {
166
- const haveStream = (stream) => !!reduction[stream.id];
167
- const haveRevision = (stream) => !!reduction[stream.id][stream.rev];
168
- const incrementStreamRevisionCounter = (stream) => reduction[stream.id][stream.rev]++;
169
- const startStreamRevisionCounter = (stream) => (reduction[stream.id][stream.rev] = 1);
170
- const initialiseStreamCounter = (stream) => (reduction[stream.id] = { [stream.rev]: 1 });
171
- let reduction = {};
172
- Helper.output("Stream Information", streamInformation);
173
- for (let i = streamInformation.length; i--;) {
174
- if (streamInformation[i] && streamInformation[i].streams) {
175
- const streams = Object.values(streamInformation[i].streams);
176
- Helper.output(`Reducing ${streams.length} streams`);
177
- for (let s = streams.length; s--;) {
178
- const stream = streams[s];
179
- if (stream) {
180
- Helper.output("Reducing stream", stream);
181
- haveStream(stream)
182
- ? haveRevision(stream)
183
- ? incrementStreamRevisionCounter(stream)
184
- : startStreamRevisionCounter(stream)
185
- : initialiseStreamCounter(stream);
186
- }
187
- }
188
- }
189
- }
190
- Helper.output("Reduction", reduction);
191
- return reduction;
192
- }
193
264
  getRebuildData(node) {
194
265
  return new Promise(async (resolve, reject) => {
195
266
  let streams = [];