@magda/minion-framework 2.0.0-alpha.5 → 2.0.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Crawler.js +2 -2
- package/dist/Crawler.js.map +1 -1
- package/dist/commonYargs.d.ts +9 -8
- package/dist/commonYargs.js.map +1 -1
- package/dist/test/Crawler.spec.js +6 -2
- package/dist/test/Crawler.spec.js.map +1 -1
- package/dist/test/fakeArgv.d.ts +2 -1
- package/dist/test/fakeArgv.js.map +1 -1
- package/package.json +4 -4
package/dist/Crawler.js
CHANGED
|
@@ -78,14 +78,14 @@ class Crawler {
|
|
|
78
78
|
return this.isCrawling;
|
|
79
79
|
}
|
|
80
80
|
crawlExistingRecords() {
|
|
81
|
-
var _a;
|
|
81
|
+
var _a, _b, _c, _d;
|
|
82
82
|
return __awaiter(this, void 0, void 0, function* () {
|
|
83
83
|
try {
|
|
84
84
|
this.resetCrawler();
|
|
85
85
|
this.isCrawling = true;
|
|
86
86
|
console.info("Crawling existing records in registry");
|
|
87
87
|
const crawlerRecordFetchNumber = ((_a = this.options) === null || _a === void 0 ? void 0 : _a.crawlerRecordFetchNumber) ? this.options.crawlerRecordFetchNumber :
|
|
88
|
-
DEFAULT_CRAWLER_RECORD_FETCH_NUMBER;
|
|
88
|
+
((_c = (_b = this.options) === null || _b === void 0 ? void 0 : _b.argv) === null || _c === void 0 ? void 0 : _c.crawlerRecordFetchNumber) ? (_d = this.options) === null || _d === void 0 ? void 0 : _d.argv.crawlerRecordFetchNumber : DEFAULT_CRAWLER_RECORD_FETCH_NUMBER;
|
|
89
89
|
const registryPage = AsyncPage_1.default.create(previous => {
|
|
90
90
|
if (previous && previous.hasMore === false) {
|
|
91
91
|
console.info("No more records left");
|
package/dist/Crawler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Crawler.js","sourceRoot":"","sources":["../src/Crawler.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,yGAAiF;AACjF,mFAAgF;AAIhF,qDAAqD;AACrD,MAAM,mCAAmC,GAAG,GAAG,CAAC;AAEhD,MAAM,OAAO;IAST,YAAY,QAAkB,EAAE,OAAsB;QAR9C,eAAU,GAAY,KAAK,CAAC;QAC5B,sBAAiB,GAAW,IAAI,CAAC;QACjC,wBAAmB,GAAW,CAAC,CAAC;QAChC,mBAAc,GAAkB,IAAI,CAAC;QAErC,aAAQ,GAAa,IAAI,CAAC;QAC1B,YAAO,GAAkB,IAAI,CAAC;QAGlC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAEO,YAAY;QAChB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,WAAW;QACd,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;SAChD,CAAC;IACN,CAAC;IAEM,KAAK;QACR,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC,cAAc,CAAC;QAChD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED;;OAEG;IACU,eAAe;;YACxB,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAc;gBAAE,OAAO;YACrD,MAAM,IAAI,CAAC,cAAc,CAAC;QAC9B,CAAC;KAAA;IAEM,YAAY;QACf,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAEa,oBAAoB;;;YAC9B,IAAI;gBACA,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;gBAEtD,MAAM,wBAAwB,GAAG,OAAA,IAAI,CAAC,OAAO,0CACvC,wBAAwB,EAC1B,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,wBAAwB;oBACvC,CAAC,CAAC,mCAAmC,CAAC;gBAE1C,MAAM,YAAY,GAAG,mBAAS,CAAC,MAAM,CACjC,CAAC,QAAQ,EAAE,EAAE;oBACT,IAAI,QAAQ,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE;wBACxC,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;wBACrC,oDAAoD;wBACpD,OAAO,SAAS,CAAC;qBACpB;yBAAM;wBACH,OAAO,CAAC,IAAI,CACR,uBAAuB;4BACnB,CAAC,QAAQ,IAAI,QAAQ,CAAC,aAAa;gCAC/B,CAAC,CAAC,QAAQ,CAAC,aAAa;gCACxB,CAAC,CAAC,cAAc,CAAC,CAC5B,CAAC;wBACF,IAAI,CAAC,iBAAiB;4BAClB,QAAQ,IAAI,QAAQ,CAAC,aAAa;gCAC9B,CAAC,CAAC,QAAQ,CAAC,aAAa;gCACxB,CAAC,CAAC,EAAE,CAAC;wBACb,iEAAiE;wBACjE,OAAO,IAAI,CAAC,QAAQ;6BACf,UAAU,CACP,IAAI,CAAC,OAAO,CAAC,OAAO,EACpB,IAAI,CAAC,OAAO,CAAC,eAAe,EAC5B,QAAQ,IAAI,QAAQ,CAAC,aAAa,EAClC,IAAI,EACJ,
|
|
1
|
+
{"version":3,"file":"Crawler.js","sourceRoot":"","sources":["../src/Crawler.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,yGAAiF;AACjF,mFAAgF;AAIhF,qDAAqD;AACrD,MAAM,mCAAmC,GAAG,GAAG,CAAC;AAEhD,MAAM,OAAO;IAST,YAAY,QAAkB,EAAE,OAAsB;QAR9C,eAAU,GAAY,KAAK,CAAC;QAC5B,sBAAiB,GAAW,IAAI,CAAC;QACjC,wBAAmB,GAAW,CAAC,CAAC;QAChC,mBAAc,GAAkB,IAAI,CAAC;QAErC,aAAQ,GAAa,IAAI,CAAC;QAC1B,YAAO,GAAkB,IAAI,CAAC;QAGlC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAEO,YAAY;QAChB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,WAAW;QACd,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;SAChD,CAAC;IACN,CAAC;IAEM,KAAK;QACR,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC,cAAc,CAAC;QAChD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED;;OAEG;IACU,eAAe;;YACxB,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAc;gBAAE,OAAO;YACrD,MAAM,IAAI,CAAC,cAAc,CAAC;QAC9B,CAAC;KAAA;IAEM,YAAY;QACf,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAEa,oBAAoB;;;YAC9B,IAAI;gBACA,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;gBAEtD,MAAM,wBAAwB,GAAG,OAAA,IAAI,CAAC,OAAO,0CACvC,wBAAwB,EAC1B,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,wBAAwB;oBACvC,CAAC,CAAC,aAAA,IAAI,CAAC,OAAO,0CAAE,IAAI,0CAAE,wBAAwB,EAC9C,CAAC,OAAC,IAAI,CAAC,OAAO,0CAAE,IAAI,CAAC,wBAAwB,CAC7C,CAAC,CAAC,mCAAmC,CAAC;gBAE1C,MAAM,YAAY,GAAG,mBAAS,CAAC,MAAM,CACjC,CAAC,QAAQ,EAAE,EAAE;oBACT,IAAI,QAAQ,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE;wBACxC,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;wBACrC,oDAAoD;wBACpD,OAAO,SAAS,CAAC;qBACpB;yBAAM;wBACH,OAAO,CAAC,IAAI,CACR,uBAAuB;4BACnB,CAAC,QAAQ,IAAI,QAAQ,CAAC,aAAa;gCAC/B,CAAC,CAAC,QAAQ,CAAC,aAAa;gCACxB,CAAC,CAAC,cAAc,CAAC,CAC5B,CAAC;wBACF,IAAI,CAAC,iBAAiB;4BAClB,QAAQ,IAAI,QAAQ,CAAC,aAAa;gCAC9B,CAAC,CAAC,QAAQ,CAAC,aAAa;gCACxB,CAAC,CAAC,EAAE,CAAC;wBACb,iEAAiE;wBACjE,OAAO,IAAI,CAAC,QAAQ;6BACf,UAAU,CACP,IAAI,CAAC,OAAO,CAAC,OAAO,EACpB,IAAI,CAAC,OAAO,CAAC,eAAe,EAC5B,QAAQ,IAAI,QAAQ,CAAC,aAAa,EAClC,IAAI,EACJ,wBAAkC,CACrC;6BACA,IAAI,CAAC,0BAAgB,CAAC;6BACtB,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;4BACX,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;4BAChD,OAAO,CAAC,IAAI,CACR,WAAW,IAAI,CAAC,OAAO,CAAC,MAAM,UAAU,CAC3C,CAAC;4BACF,OAAO,IAAI,CAAC;wBAChB,CAAC,CAAC,CAAC;qBACV;gBACL,CAAC,CACJ,CAAC,GAAG,CAAC,CAAC,IAAyB,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAEnD,MAAM,wBAAY,CACd,YAAY,EACZ,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,EAC7B,CAAC,MAAc,EAAE,EAAE,CACf,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CACxD,CAAC;aACL;oBAAS;gBACN,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;aAC9B;;KACJ;CACJ;AAED,kBAAe,OAAO,CAAC"}
|
package/dist/commonYargs.d.ts
CHANGED
|
@@ -7,14 +7,15 @@ export declare type MinionArguments = {
|
|
|
7
7
|
registryUrl: string;
|
|
8
8
|
enableMultiTenant: boolean;
|
|
9
9
|
tenantUrl: string;
|
|
10
|
-
retries: string | number;
|
|
10
|
+
retries: string | number;
|
|
11
|
+
crawlerRecordFetchNumber?: string | number;};
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
14
|
+
* Builds an argv object that will accept command line arguments used by all common argv minions.
|
|
15
|
+
*
|
|
16
|
+
* @param id
|
|
17
|
+
* @param defaultPort
|
|
18
|
+
* @param defaultInternalUrl
|
|
19
|
+
* @param additions
|
|
20
|
+
*/
|
|
20
21
|
export default function commonYargs<T extends MinionArguments = MinionArguments>(defaultPort: number, defaultInternalUrl: string, additions?: (a: yargs.Argv<MinionArguments>) => yargs.Argv<T>): T;
|
package/dist/commonYargs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"commonYargs.js","sourceRoot":"","sources":["../src/commonYargs.ts"],"names":[],"mappings":";;;;;AAAA,kDAA0B;AAE1B,wHAAgG;
|
|
1
|
+
{"version":3,"file":"commonYargs.js","sourceRoot":"","sources":["../src/commonYargs.ts"],"names":[],"mappings":";;;;;AAAA,kDAA0B;AAE1B,wHAAgG;AAchG;;;;;;;GAOG;AACH,SAAwB,WAAW,CAG/B,WAAmB,EACnB,kBAA0B,EAC1B,YAA+D,CAAC,CAAC,EAAE,EAAE,CACjE,CAAkB;IAEtB,MAAM,IAAI,GAAG,eAAK;SACb,MAAM,EAAE;SACR,IAAI,EAAE;SACN,MAAM,CAAC,YAAY,EAAE;QAClB,QAAQ,EAAE,qDAAqD;QAC/D,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,WAAW;KAChD,CAAC;SACD,MAAM,CAAC,aAAa,EAAE;QACnB,QAAQ,EAAE,uCAAuC;QACjD,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,kBAAkB;KAC1D,CAAC;SACD,MAAM,CAAC,WAAW,EAAE;QACjB,QAAQ,EAAE,mDAAmD;QAC7D,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,IAAI;QACZ,OAAO,EACH,OAAO,CAAC,GAAG,CAAC,UAAU;YACtB,OAAO,CAAC,GAAG,CAAC,4BAA4B;KAC/C,CAAC;SACD,MAAM,CAAC,QAAQ,EAAE;QACd,QAAQ,EACJ,uEAAuE;QAC3E,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,IAAI;QACZ,OAAO,EACH,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,yBAAyB;KACnE,CAAC;SACD,MAAM,CAAC,aAAa,EAAE;QACnB,QAAQ,EAAE,+BAA+B;QACzC,IAAI,EAAE,QAAQ;QACd,OAAO,EACH,OAAO,CAAC,GAAG,CAAC,YAAY;YACxB,OAAO,CAAC,GAAG,CAAC,8BAA8B;YAC1C,0BAA0B;KACjC,CAAC;SACD,MAAM,CAAC,mBAAmB,EAAE;QACzB,QAAQ,EAAE,sCAAsC;QAChD,IAAI,EAAE,SAAS;QACf,OAAO,EACH,OAAO,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,WAAW;YACnD,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,WAAW,EAAE,KAAK,MAAM;gBACvD,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,KAAK;YACX,CAAC,CAAC,OAAO,OAAO,CAAC,GAAG;iBACb,qCAAqC,KAAK,WAAW;gBAC5D,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,qCAAqC;oBAC/C,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,KAAK;gBACX,CAAC,CAAC,KAAK;KAClB,CAAC;SACD,MAAM,CAAC,WAAW,EAAE;QACjB,QAAQ,EAAE,qCAAqC;QAC/C,IAAI,EAAE,QAAQ;QACd,OAAO,EACH,OAAO,CAAC,GAAG,CAAC,UAAU;YACtB,OAAO,CAAC,GAAG,CAAC,4BAA4B;YACxC,0BAA0B;KACjC,CAAC;SACD,MAAM,CAAC,SAAS,EAAE;QACf,QAAQ,EAAE,mDAAmD;QAC7D,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE;KACrC,CAAC;SACD,MAAM,CAAC,0BAA0B,EAAE;QAChC,QAAQ,EAAE,mDAAmD;QAC7D,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,GAAG;KAC1D,CAAC,CAAC;IAEP,MAAM,WAAW,GAAG,gCAAsB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IACjE,OAAO,WAAW,CAAC;AACvB,CAAC;AAjFD,8BAiFC"}
|
|
@@ -23,6 +23,7 @@ const fakeArgv_1 = __importDefault(require("./fakeArgv"));
|
|
|
23
23
|
const baseSpec_1 = __importDefault(require("./baseSpec"));
|
|
24
24
|
const Crawler_1 = __importDefault(require("../Crawler"));
|
|
25
25
|
const TenantConsts_1 = require("magda-typescript-common/src/registry/TenantConsts");
|
|
26
|
+
const DEFAULT_CRAWLER_RECORD_FETCH_NUMBER = 198;
|
|
26
27
|
baseSpec_1.default("Crawler", (expressApp, _expressServer, listenPort, beforeEachProperty) => {
|
|
27
28
|
//--- avoid different property test hit into the same registry domain
|
|
28
29
|
let registryDomainCounter = 0;
|
|
@@ -68,7 +69,8 @@ baseSpec_1.default("Crawler", (expressApp, _expressServer, listenPort, beforeEac
|
|
|
68
69
|
tenantUrl,
|
|
69
70
|
jwtSecret,
|
|
70
71
|
userId,
|
|
71
|
-
listenPort: listenPort()
|
|
72
|
+
listenPort: listenPort(),
|
|
73
|
+
crawlerRecordFetchNumber: DEFAULT_CRAWLER_RECORD_FETCH_NUMBER
|
|
72
74
|
}),
|
|
73
75
|
id: "id",
|
|
74
76
|
aspects: [],
|
|
@@ -123,8 +125,10 @@ baseSpec_1.default("Crawler", (expressApp, _expressServer, listenPort, beforeEac
|
|
|
123
125
|
? parseInt(params.pageToken, 10)
|
|
124
126
|
: 0;
|
|
125
127
|
const limit = parseInt(params.limit, 10);
|
|
126
|
-
|
|
128
|
+
chai_1.expect(limit).to.equal(DEFAULT_CRAWLER_RECORD_FETCH_NUMBER);
|
|
129
|
+
if (limit < 1) {
|
|
127
130
|
throw new Error("Invalid limit param received!");
|
|
131
|
+
}
|
|
128
132
|
if (pageIdx >= this.registryRecords.length)
|
|
129
133
|
return [
|
|
130
134
|
200,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Crawler.spec.js","sourceRoot":"","sources":["../../src/test/Crawler.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAEA,gDAAwB;AACxB,oDAAuB;AACvB,+BAA8B;AAC9B,gEAAuC;AACvC,6HAAqF;AAGrF,8EAAqF;AACrF,yFAA4D;AAG5D,0DAAkC;AAClC,0DAAkC;AAClC,yDAAiC;AACjC,oFAA0F;AAE1F,kBAAQ,CACJ,SAAS,EACT,CACI,UAAiC,EACjC,cAA4B,EAC5B,UAAwB,EACxB,kBAA8B,EAChC,EAAE;IACA,qEAAqE;IACrE,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAE9B,uDAAuD;IACvD,SAAe,eAAe,CAC1B,0BAAkC,EAClC,MAAc,EACd,SAAiB,EACjB,WAAmB,EACnB,KAAc,EACd,iBAA0B;IAC1B,uEAAuE;IACvE,OAAkB,EAClB,aAGkB,EAClB,iBAA4D,EAC5D,oBAAmC;;YAEnC,kBAAkB,EAAE,CAAC;YAErB,MAAM,WAAW,GAAG,UAAU,MAAM,MAAM,CAAC;YAC3C,MAAM,cAAc,GAAG,WAAW,GAAG,qBAAqB,CAAC;YAC3D,MAAM,YAAY,GAAG,SAAS,GAAG,qBAAqB,CAAC;YACvD,qBAAqB,EAAE,CAAC;YACxB,MAAM,WAAW,GAAG,UAAU,cAAc,SAAS,CAAC;YACtD,MAAM,SAAS,GAAG,UAAU,YAAY,SAAS,CAAC;YAClD,MAAM,aAAa,GAAG,cAAI,CAAC,WAAW,CAAC,CAAC;YACxC,MAAM,QAAQ,GAAG,oCAAqB,CAAC;YAEvC,MAAM,MAAM,GAAG,sCAAsC,CAAC;YACtD,MAAM,QAAQ,GAAG,IAAI,kCAAQ,CAAC;gBAC1B,OAAO,EAAE,WAAW;gBACpB,SAAS,EAAE,SAAS;gBACpB,MAAM,EAAE,MAAM;gBACd,QAAQ,EAAE,QAAQ;aACrB,CAAC,CAAC;YAEH,IAAI,OAAO,GAAQ;gBACf,0BAA0B;gBAC1B,MAAM;gBACN,SAAS;gBACT,MAAM;gBACN,WAAW;gBACX,KAAK;gBACL,aAAa;gBACb,QAAQ;aACX,CAAC;YAEF,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;gBAC9B,OAAO,mCAAQ,OAAO,GAAK,GAAG,CAAE,CAAC;YAErC,MAAM,OAAO,GAAkB;gBAC3B,IAAI,EAAE,kBAAQ,CAAC;oBACX,WAAW;oBACX,WAAW;oBACX,iBAAiB;oBACjB,SAAS;oBACT,SAAS;oBACT,MAAM;oBACN,UAAU,EAAE,UAAU,EAAE;
|
|
1
|
+
{"version":3,"file":"Crawler.spec.js","sourceRoot":"","sources":["../../src/test/Crawler.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAEA,gDAAwB;AACxB,oDAAuB;AACvB,+BAA8B;AAC9B,gEAAuC;AACvC,6HAAqF;AAGrF,8EAAqF;AACrF,yFAA4D;AAG5D,0DAAkC;AAClC,0DAAkC;AAClC,yDAAiC;AACjC,oFAA0F;AAE1F,MAAM,mCAAmC,GAAG,GAAG,CAAC;AAEhD,kBAAQ,CACJ,SAAS,EACT,CACI,UAAiC,EACjC,cAA4B,EAC5B,UAAwB,EACxB,kBAA8B,EAChC,EAAE;IACA,qEAAqE;IACrE,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAE9B,uDAAuD;IACvD,SAAe,eAAe,CAC1B,0BAAkC,EAClC,MAAc,EACd,SAAiB,EACjB,WAAmB,EACnB,KAAc,EACd,iBAA0B;IAC1B,uEAAuE;IACvE,OAAkB,EAClB,aAGkB,EAClB,iBAA4D,EAC5D,oBAAmC;;YAEnC,kBAAkB,EAAE,CAAC;YAErB,MAAM,WAAW,GAAG,UAAU,MAAM,MAAM,CAAC;YAC3C,MAAM,cAAc,GAAG,WAAW,GAAG,qBAAqB,CAAC;YAC3D,MAAM,YAAY,GAAG,SAAS,GAAG,qBAAqB,CAAC;YACvD,qBAAqB,EAAE,CAAC;YACxB,MAAM,WAAW,GAAG,UAAU,cAAc,SAAS,CAAC;YACtD,MAAM,SAAS,GAAG,UAAU,YAAY,SAAS,CAAC;YAClD,MAAM,aAAa,GAAG,cAAI,CAAC,WAAW,CAAC,CAAC;YACxC,MAAM,QAAQ,GAAG,oCAAqB,CAAC;YAEvC,MAAM,MAAM,GAAG,sCAAsC,CAAC;YACtD,MAAM,QAAQ,GAAG,IAAI,kCAAQ,CAAC;gBAC1B,OAAO,EAAE,WAAW;gBACpB,SAAS,EAAE,SAAS;gBACpB,MAAM,EAAE,MAAM;gBACd,QAAQ,EAAE,QAAQ;aACrB,CAAC,CAAC;YAEH,IAAI,OAAO,GAAQ;gBACf,0BAA0B;gBAC1B,MAAM;gBACN,SAAS;gBACT,MAAM;gBACN,WAAW;gBACX,KAAK;gBACL,aAAa;gBACb,QAAQ;aACX,CAAC;YAEF,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;gBAC9B,OAAO,mCAAQ,OAAO,GAAK,GAAG,CAAE,CAAC;YAErC,MAAM,OAAO,GAAkB;gBAC3B,IAAI,EAAE,kBAAQ,CAAC;oBACX,WAAW;oBACX,WAAW;oBACX,iBAAiB;oBACjB,SAAS;oBACT,SAAS;oBACT,MAAM;oBACN,UAAU,EAAE,UAAU,EAAE;oBACxB,wBAAwB,EAAE,mCAAmC;iBAChE,CAAC;gBACF,EAAE,EAAE,IAAI;gBACR,OAAO,EAAE,EAAE;gBACX,eAAe,EAAE,EAAE;gBACnB,eAAe,EAAE,EAAE;gBACnB,KAAK;gBACL,OAAO,EAAE,UAAU;gBACnB,WAAW,EAAE,WAAW;gBACxB,aAAa,EAAE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC;aAC7C,CAAC;YAEF,aAAa;iBACR,OAAO,EAAE;iBACT,GAAG,CAAC,UAAU,CAAC;iBACf,KAAK,CAAC,IAAI,CAAC;iBACX,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YAE5C,MAAM,OAAO,GAAG,IAAI,iBAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC/C,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC;YAE1B,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,aAAa,CAAC,IAAI,EAAE,CAAC;YACrB,OAAO,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,CAAC;KAAA;IAED,SAAS,gBAAgB;IACrB,uEAAuE;IACvE,OAAkB,EAClB,aAGkB,EAClB,iBAA4D,EAC5D,oBAAmC;QAEnC,OAAO,kBAAG,CAAC,MAAM,CACb,kBAAG,CAAC,MAAM,CACN,kBAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EACZ,mCAAqB,EACrB,mCAAqB,EACrB,kBAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAClB,kBAAG,CAAC,IAAI,EACR,kBAAG,CAAC,IAAI,EACR,gBAAC,CAAC,YAAY,CACV,eAAe,EACf,OAAO,EACP,aAAa,EACb,iBAAiB,EACjB,oBAAoB,CACvB,CACJ,CACJ,CAAC;IACN,CAAC;IAED,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACjE,OAAO,gBAAgB,CACnB;YACI,YAAY;YACZ,oFAAoF;YACpF,MAAM,gBAAgB,GAAa,IAAI,KAAK,CACxC,IAAI,CAAC,0BAA0B,CAClC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACV,MAAM,eAAe,GAAa,gBAAgB,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;gBACZ,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC;gBACf,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;gBACX,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,oCAAqB;aAClC,CAAC,CACL,CAAC;YACF,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACjD,CAAC,EACD,UAGI,WAAmB,EACnB,QAAkB;YAElB,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACzC,IAAI,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE;gBAChD,MAAM,IAAI,KAAK,CACX,sCAAsC,GAAG,EAAE,CAC9C,CAAC;aACL;YACD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC7B,CAAC,EACD,UAAqB,GAAW,EAAE,WAAmB;YACjD,sBAAsB;YACtB,MAAM,MAAM,GAAG,sBAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;YAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS;gBAC5B,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAmB,EAAE,EAAE,CAAC;gBAC1C,CAAC,CAAC,CAAC,CAAC;YACR,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAe,EAAE,EAAE,CAAC,CAAC;YACnD,aAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAC5D,IAAI,KAAK,GAAG,CAAC,EAAE;gBACX,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;aACpD;YACD,IAAI,OAAO,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM;gBACtC,OAAO;oBACH,GAAG;oBACH;wBACI,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;wBACvC,OAAO,EAAE,KAAK;wBACd,OAAO,EAAE,IAAI,KAAK,EAAE;qBACvB;iBACJ,CAAC;YACN,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CACzC,OAAO,EACP,OAAO,GAAG,KAAK,GAAG,CAAC,CACtB,CAAC;YAEF,MAAM,OAAO,GAAQ;gBACjB,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACvC,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,UAAU;aACtB,CAAC;YAEF,MAAM,aAAa,GAAG,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC;YAElD,IAAI,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE;gBAC7C,OAAO,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;aACjD;iBAAM;gBACH,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC;aAC3B;YAED,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC1B,CAAC,EACD;YACI,yBAAyB;YACzB,OAAO,CACH,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAC3B,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,CAC5B,KAAK,CAAC,CAAC,CACX,CAAC;QACN,CAAC,CACJ,CAAC;IACN,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAElB,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACjF,OAAO,gBAAgB,CACnB;YACI,YAAY;YACZ,MAAM,yBAAyB,GAAG,CAAC,CAAC;YAEpC,MAAM,eAAe,GAAa,IAAI,KAAK,CACvC,IAAI,CAAC,0BAA0B,CAClC;iBACI,IAAI,CAAC,CAAC,CAAC;iBACP,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;gBACjB,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC;gBACf,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;gBACX,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,oCAAqB;aAClC,CAAC,CAAC,CAAC;YACR,OAAO;gBACH,yBAAyB;gBACzB,eAAe;aAClB,CAAC;QACN,CAAC,EACD,UAGI,WAAmB,EACnB,QAAkB;YAElB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC7B,CAAC,EACD,UAAqB,GAAW,EAAE,WAAmB;YACjD,sBAAsB;YACtB,MAAM,MAAM,GAAG,sBAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;YAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS;gBAC5B,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAmB,EAAE,EAAE,CAAC;gBAC1C,CAAC,CAAC,CAAC,CAAC;YACR,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAe,EAAE,EAAE,CAAC,CAAC;YACnD,IAAI,KAAK,GAAG,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;YACrD,IAAI,OAAO,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE;gBACxC,OAAO;oBACH,GAAG;oBACH;wBACI,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;wBACvC,OAAO,EAAE,KAAK;wBACd,OAAO,EAAE,EAAE;qBACd;iBACJ,CAAC;aACL;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CACzC,OAAO,EACP,OAAO,GAAG,KAAK,GAAG,CAAC,CACtB,CAAC;YACF,MAAM,sBAAsB,GAAG,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC;YAC3D,MAAM,iBAAiB,GAAG,MAAM,CAAC,sBAAsB,CAAC,CAAC;YACzD,IAAI,CAAC,yBAAyB,IAAI,UAAU,CAAC,MAAM,CAAC;YACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC5C,aAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;gBAC3B,UAAU,EAAE,IAAI;gBAChB,iBAAiB,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjD,mBAAmB,EAAE,OAAO;aAC/B,CAAC,CAAC;YACH,OAAO;gBACH,GAAG;gBACH;oBACI,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;oBACvC,OAAO,EAAE,IAAI;oBACb,aAAa,EAAE,iBAAiB;oBAChC,OAAO,EAAE,UAAU;iBACtB;aACJ,CAAC;QACN,CAAC,EACD;YACI,yBAAyB;YACzB,OAAO,IAAI,CAAC;QAChB,CAAC,CACJ,CAAC;IACN,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AACvB,CAAC,CACJ,CAAC"}
|
package/dist/test/fakeArgv.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fakeArgv.js","sourceRoot":"","sources":["../../src/test/fakeArgv.ts"],"names":[],"mappings":";;AAEA,SAAwB,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"fakeArgv.js","sourceRoot":"","sources":["../../src/test/fakeArgv.ts"],"names":[],"mappings":";;AAEA,SAAwB,QAAQ,CAAC,OAShC;IACG,uCACO,OAAO,KACV,OAAO,EAAE,CAAC,IACZ;AACN,CAAC;AAdD,2BAcC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@magda/minion-framework",
|
|
3
3
|
"description": "MAGDA Minion Framework",
|
|
4
|
-
"version": "2.0.0-alpha.
|
|
4
|
+
"version": "2.0.0-alpha.8",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"prebuild": "rimraf dist tsconfig.tsbuildinfo",
|
|
7
7
|
"build": "yarn run compile",
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"license": "Apache-2.0",
|
|
15
15
|
"main": "dist",
|
|
16
16
|
"devDependencies": {
|
|
17
|
-
"@magda/scripts": "^2.0.0-alpha.
|
|
17
|
+
"@magda/scripts": "^2.0.0-alpha.8",
|
|
18
18
|
"@types/chai": "^4.0.1",
|
|
19
19
|
"@types/express": "^4.17.6",
|
|
20
20
|
"@types/isomorphic-fetch": "0.0.34",
|
|
@@ -38,8 +38,8 @@
|
|
|
38
38
|
"typescript": "^3.7.2"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@magda/registry-aspects": "^2.0.0-alpha.
|
|
42
|
-
"@magda/typescript-common": "^2.0.0-alpha.
|
|
41
|
+
"@magda/registry-aspects": "^2.0.0-alpha.8",
|
|
42
|
+
"@magda/typescript-common": "^2.0.0-alpha.8",
|
|
43
43
|
"body-parser": "^1.17.2",
|
|
44
44
|
"express": "^4.17.1",
|
|
45
45
|
"isomorphic-fetch": "^2.2.1",
|