@magda/minion-framework 2.0.0-alpha.6 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Crawler.js +6 -3
- package/dist/Crawler.js.map +1 -1
- package/dist/setupWebhookEndpoint.js +1 -1
- package/package.json +5 -5
package/dist/Crawler.js
CHANGED
|
@@ -14,7 +14,7 @@ var __setModuleDefault = this && this.__setModuleDefault || (Object.create ? fun
|
|
|
14
14
|
var __importStar = this && this.__importStar || function (mod) {
|
|
15
15
|
if (mod && mod.__esModule) return mod;
|
|
16
16
|
var result = {};
|
|
17
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
17
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
18
|
__setModuleDefault(result, mod);
|
|
19
19
|
return result;
|
|
20
20
|
};
|
|
@@ -84,8 +84,11 @@ class Crawler {
|
|
|
84
84
|
this.resetCrawler();
|
|
85
85
|
this.isCrawling = true;
|
|
86
86
|
console.info("Crawling existing records in registry");
|
|
87
|
-
const crawlerRecordFetchNumber = ((_a = this.options) === null || _a === void 0 ? void 0 : _a.crawlerRecordFetchNumber) ?
|
|
88
|
-
|
|
87
|
+
const crawlerRecordFetchNumber = ((_a = this.options) === null || _a === void 0 ? void 0 : _a.crawlerRecordFetchNumber) ?
|
|
88
|
+
this.options.crawlerRecordFetchNumber :
|
|
89
|
+
((_c = (_b = this.options) === null || _b === void 0 ? void 0 : _b.argv) === null || _c === void 0 ? void 0 : _c.crawlerRecordFetchNumber) ?
|
|
90
|
+
(_d = this.options) === null || _d === void 0 ? void 0 : _d.argv.crawlerRecordFetchNumber :
|
|
91
|
+
DEFAULT_CRAWLER_RECORD_FETCH_NUMBER;
|
|
89
92
|
const registryPage = AsyncPage_1.default.create(previous => {
|
|
90
93
|
if (previous && previous.hasMore === false) {
|
|
91
94
|
console.info("No more records left");
|
package/dist/Crawler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Crawler.js","sourceRoot":"","sources":["../src/Crawler.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,yGAAiF;AACjF,mFAAgF;AAIhF,qDAAqD;AACrD,MAAM,mCAAmC,GAAG,GAAG,CAAC;AAEhD,MAAM,OAAO;IAST,YAAY,QAAkB,EAAE,OAAsB;QAR9C,eAAU,GAAY,KAAK,CAAC;QAC5B,sBAAiB,GAAW,IAAI,CAAC;QACjC,wBAAmB,GAAW,CAAC,CAAC;QAChC,mBAAc,GAAkB,IAAI,CAAC;QAErC,aAAQ,GAAa,IAAI,CAAC;QAC1B,YAAO,GAAkB,IAAI,CAAC;QAGlC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAEO,YAAY;QAChB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,WAAW;QACd,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;SAChD,CAAC;IACN,CAAC;IAEM,KAAK;QACR,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC,cAAc,CAAC;QAChD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED;;OAEG;IACU,eAAe;;YACxB,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAc;gBAAE,OAAO;YACrD,MAAM,IAAI,CAAC,cAAc,CAAC;QAC9B,CAAC;KAAA;IAEM,YAAY;QACf,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAEa,oBAAoB;;;YAC9B,IAAI;gBACA,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;gBAEtD,MAAM,wBAAwB,GAAG,
|
|
1
|
+
{"version":3,"file":"Crawler.js","sourceRoot":"","sources":["../src/Crawler.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,yGAAiF;AACjF,mFAAgF;AAIhF,qDAAqD;AACrD,MAAM,mCAAmC,GAAG,GAAG,CAAC;AAEhD,MAAM,OAAO;IAST,YAAY,QAAkB,EAAE,OAAsB;QAR9C,eAAU,GAAY,KAAK,CAAC;QAC5B,sBAAiB,GAAW,IAAI,CAAC;QACjC,wBAAmB,GAAW,CAAC,CAAC;QAChC,mBAAc,GAAkB,IAAI,CAAC;QAErC,aAAQ,GAAa,IAAI,CAAC;QAC1B,YAAO,GAAkB,IAAI,CAAC;QAGlC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAEO,YAAY;QAChB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,WAAW;QACd,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;SAChD,CAAC;IACN,CAAC;IAEM,KAAK;QACR,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC,cAAc,CAAC;QAChD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED;;OAEG;IACU,eAAe;;YACxB,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAc;gBAAE,OAAO;YACrD,MAAM,IAAI,CAAC,cAAc,CAAC;QAC9B,CAAC;KAAA;IAEM,YAAY;QACf,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAEa,oBAAoB;;;YAC9B,IAAI;gBACA,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;gBAEtD,MAAM,wBAAwB,GAAG,CAAA,MAAA,IAAI,CAAC,OAAO,0CACvC,wBAAwB;oBAC1B,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,wBAAwB;oBACvC,CAAC,CAAC,CAAA,MAAA,MAAA,IAAI,CAAC,OAAO,0CAAE,IAAI,0CAAE,wBAAwB;wBAC9C,CAAC,CAAC,MAAA,IAAI,CAAC,OAAO,0CAAE,IAAI,CAAC,wBAAwB;wBAC7C,CAAC,CAAC,mCAAmC,CAAC;gBAE1C,MAAM,YAAY,GAAG,mBAAS,CAAC,MAAM,CACjC,CAAC,QAAQ,EAAE,EAAE;oBACT,IAAI,QAAQ,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE;wBACxC,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;wBACrC,oDAAoD;wBACpD,OAAO,SAAS,CAAC;qBACpB;yBAAM;wBACH,OAAO,CAAC,IAAI,CACR,uBAAuB;4BACnB,CAAC,QAAQ,IAAI,QAAQ,CAAC,aAAa;gCAC/B,CAAC,CAAC,QAAQ,CAAC,aAAa;gCACxB,CAAC,CAAC,cAAc,CAAC,CAC5B,CAAC;wBACF,IAAI,CAAC,iBAAiB;4BAClB,QAAQ,IAAI,QAAQ,CAAC,aAAa;gCAC9B,CAAC,CAAC,QAAQ,CAAC,aAAa;gCACxB,CAAC,CAAC,EAAE,CAAC;wBACb,iEAAiE;wBACjE,OAAO,IAAI,CAAC,QAAQ;6BACf,UAAU,CACP,IAAI,CAAC,OAAO,CAAC,OAAO,EACpB,IAAI,CAAC,OAAO,CAAC,eAAe,EAC5B,QAAQ,IAAI,QAAQ,CAAC,aAAa,EAClC,IAAI,EACJ,wBAAkC,CACrC;6BACA,IAAI,CAAC,0BAAgB,CAAC;6BACtB,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;4BACX,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;4BAChD,OAAO,CAAC,IAAI,CACR,WAAW,IAAI,CAAC,OAAO,CAAC,MAAM,UAAU,CAC3C,CAAC;4BACF,OAAO,IAAI,CAAC;wBAChB,CAAC,CAAC,CAAC;qBACV;gBACL,CAAC,CACJ,CAAC,GAAG,CAAC,CAAC,IAAyB,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAEnD,MAAM,wBAAY,CACd,YAAY,EACZ,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,EAC7B,CAAC,MAAc,EAAE,EAAE,CACf,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CACxD,CAAC;aACL;oBAAS;gBACN,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;aAC9B;;KACJ;CACJ;AAED,kBAAe,OAAO,CAAC"}
|
|
@@ -14,7 +14,7 @@ var __setModuleDefault = this && this.__setModuleDefault || (Object.create ? fun
|
|
|
14
14
|
var __importStar = this && this.__importStar || function (mod) {
|
|
15
15
|
if (mod && mod.__esModule) return mod;
|
|
16
16
|
var result = {};
|
|
17
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
17
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
18
|
__setModuleDefault(result, mod);
|
|
19
19
|
return result;
|
|
20
20
|
};
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@magda/minion-framework",
|
|
3
3
|
"description": "MAGDA Minion Framework",
|
|
4
|
-
"version": "2.0.0
|
|
4
|
+
"version": "2.0.0",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"prebuild": "rimraf dist tsconfig.tsbuildinfo",
|
|
7
7
|
"build": "yarn run compile",
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"license": "Apache-2.0",
|
|
15
15
|
"main": "dist",
|
|
16
16
|
"devDependencies": {
|
|
17
|
-
"@magda/scripts": "^2.0.0
|
|
17
|
+
"@magda/scripts": "^2.0.0",
|
|
18
18
|
"@types/chai": "^4.0.1",
|
|
19
19
|
"@types/express": "^4.17.6",
|
|
20
20
|
"@types/isomorphic-fetch": "0.0.34",
|
|
@@ -35,11 +35,11 @@
|
|
|
35
35
|
"sinon": "^2.4.1",
|
|
36
36
|
"supertest": "^3.0.0",
|
|
37
37
|
"testcheck": "^1.0.0-rc.2",
|
|
38
|
-
"typescript": "
|
|
38
|
+
"typescript": "~4.2.4"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@magda/registry-aspects": "^2.0.0
|
|
42
|
-
"@magda/typescript-common": "^2.0.0
|
|
41
|
+
"@magda/registry-aspects": "^2.0.0",
|
|
42
|
+
"@magda/typescript-common": "^2.0.0",
|
|
43
43
|
"body-parser": "^1.17.2",
|
|
44
44
|
"express": "^4.17.1",
|
|
45
45
|
"isomorphic-fetch": "^2.2.1",
|