@magda/typescript-common 1.1.0-alpha.2 → 1.1.0-alpha.3

package/dist/AspectBuilder.d.ts

@@ -0,0 +1,7 @@
+import { AspectDefinition } from "./generated/registry/api";
+interface AspectBuilder {
+    aspectDefinition: AspectDefinition;
+    builderFunctionString: string;
+    setupFunctionString?: string;
+}
+export default AspectBuilder;

package/dist/AspectBuilder.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=AspectBuilder.js.map

package/dist/AspectBuilder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AspectBuilder.js","sourceRoot":"","sources":["../src/AspectBuilder.ts"],"names":[],"mappings":""}

package/dist/AspectCreationFailure.d.ts

@@ -0,0 +1,5 @@
+export default class AspectCreationFailure {
+    readonly id: string;
+    readonly error: Error;
+    constructor(id: string, error: Error);
+}

package/dist/AspectCreationFailure.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+class AspectCreationFailure {
+    constructor(id, error) {
+        this.id = id;
+        this.error = error;
+    }
+}
+exports.default = AspectCreationFailure;
+//# sourceMappingURL=AspectCreationFailure.js.map

package/dist/AspectCreationFailure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AspectCreationFailure.js","sourceRoot":"","sources":["../src/AspectCreationFailure.ts"],"names":[],"mappings":";;AAAA,MAAqB,qBAAqB;IACtC,YAAqB,EAAU,EAAW,KAAY;QAAjC,OAAE,GAAF,EAAE,CAAQ;QAAW,UAAK,GAAL,KAAK,CAAO;IAAG,CAAC;CAC7D;AAFD,wCAEC"}

package/dist/AsyncPage.d.ts

@@ -0,0 +1,18 @@
+export interface CreateAsyncPage<T> {
+    (): Promise<AsyncPage<T>>;
+}
+export default class AsyncPage<T> {
+    readonly requestNextPage: CreateAsyncPage<T>;
+    readonly data: T;
+    readonly hasData: boolean;
+    static create<T>(next: (data: T) => Promise<T>): AsyncPage<T>;
+    static single<T>(value: T): AsyncPage<T>;
+    static singlePromise<T>(valuePromise: Promise<T>): AsyncPage<T>;
+    static none<T>(): AsyncPage<T>;
+    constructor(data: T, hasData: boolean, requestNextPage: CreateAsyncPage<T>);
+    map<TResult>(selector: (data: T) => TResult): AsyncPage<TResult>;
+    forEach(callback: (data: T) => void): Promise<void>;
+    take(n: number): AsyncPage<T>;
+}
+export declare function forEachAsync<T>(page: AsyncPage<T[]>, maxConcurrency: number, callbackFn: (data: T) => Promise<void>): Promise<void>;
+export declare function asyncPageToArray<T>(page: AsyncPage<T[]>): Promise<T[]>;

package/dist/AsyncPage.js

@@ -0,0 +1,140 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.asyncPageToArray = exports.forEachAsync = void 0;
+class AsyncPage {
+    constructor(data, hasData, requestNextPage) {
+        this.data = data;
+        this.hasData = hasData;
+        this.requestNextPage = requestNextPage;
+    }
+    static create(next) {
+        const nextPage = (lastPage) => {
+            const nextPromise = next(lastPage);
+            if (!nextPromise) {
+                // There is no next page.
+                return Promise.resolve(new AsyncPage(undefined, false, undefined));
+            }
+            return nextPromise.then((thisPage) => {
+                return new AsyncPage(thisPage, true, () => nextPage(thisPage));
+            });
+        };
+        return new AsyncPage(undefined, false, () => nextPage(undefined));
+    }
+    static single(value) {
+        return new AsyncPage(value, true, undefined);
+    }
+    static singlePromise(valuePromise) {
+        return AsyncPage.create((current) => current ? undefined : valuePromise);
+    }
+    static none() {
+        return new AsyncPage(undefined, false, undefined);
+    }
+    map(selector) {
+        function loadNextPage(pager) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const nextPage = yield pager.requestNextPage();
+                const data = nextPage.hasData ? selector(nextPage.data) : undefined;
+                const loadNextPageFunction = nextPage.requestNextPage
+                    ? () => loadNextPage(nextPage)
+                    : undefined;
+                return new AsyncPage(data, nextPage.hasData, loadNextPageFunction);
+            });
+        }
+        return new AsyncPage(this.hasData ? selector(this.data) : undefined, this.hasData, this.requestNextPage ? () => loadNextPage(this) : undefined);
+    }
+    forEach(callback) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let current = this;
+            while (current) {
+                if (current.hasData) {
+                    callback(current.data);
+                }
+                if (current.requestNextPage) {
+                    current = yield current.requestNextPage();
+                }
+                else {
+                    current = undefined;
+                }
+            }
+        });
+    }
+    take(n) {
+        if (n === 0) {
+            return AsyncPage.none();
+        }
+        const nextN = this.hasData ? n - 1 : n;
+        return new AsyncPage(this.data, this.hasData, this.requestNextPage
+            ? () => this.requestNextPage().then((page) => page.take(nextN))
+            : undefined);
+    }
+}
+exports.default = AsyncPage;
+function forEachAsync(page, maxConcurrency, callbackFn) {
+    let currentPromise = undefined;
+    let currentPage = page;
+    let currentIndex = 0;
+    let resultIndex = 0;
+    function getNext() {
+        return __awaiter(this, void 0, void 0, function* () {
+            while (currentPage.requestNextPage &&
+                (!currentPage.hasData || currentIndex >= currentPage.data.length)) {
+                if (!currentPromise) {
+                    currentPromise = currentPage
+                        .requestNextPage()
+                        .then((nextPage) => {
+                        currentIndex = 0;
+                        currentPromise = undefined;
+                        currentPage = nextPage;
+                        return nextPage;
+                    });
+                }
+                currentPage = yield currentPromise;
+            }
+            if (!currentPage.hasData || currentIndex >= currentPage.data.length) {
+                return undefined;
+            }
+            return {
+                value: currentPage.data[currentIndex++],
+                index: resultIndex++
+            };
+        });
+    }
+    function callNext() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const next = yield getNext();
+            if (!next) {
+                return;
+            }
+            yield callbackFn(next.value);
+            // Start the next value.
+            return callNext();
+        });
+    }
+    const promises = [];
+    for (var i = 0; i < maxConcurrency; ++i) {
+        promises.push(callNext());
+    }
+    return Promise.all(promises).then((results) => {
+        return;
+    });
+}
+exports.forEachAsync = forEachAsync;
+function asyncPageToArray(page) {
+    const result = new Array();
+    return page
+        .forEach((value) => {
+        result.push(...value);
+    })
+        .then(() => result);
+}
+exports.asyncPageToArray = asyncPageToArray;
+//# sourceMappingURL=AsyncPage.js.map

package/dist/AsyncPage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AsyncPage.js","sourceRoot":"","sources":["../src/AsyncPage.ts"],"names":[],"mappings":";;;;;;;;;;;;AAIA,MAAqB,SAAS;IAsC1B,YACI,IAAO,EACP,OAAgB,EAChB,eAAmC;QAEnC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAzCD,MAAM,CAAC,MAAM,CAAI,IAA6B;QAC1C,MAAM,QAAQ,GAA2C,CACrD,QAAW,EACb,EAAE;YACA,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,CAAC,WAAW,EAAE;gBACd,yBAAyB;gBACzB,OAAO,OAAO,CAAC,OAAO,CAClB,IAAI,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAC7C,CAAC;aACL;YACD,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACjC,OAAO,IAAI,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YACnE,CAAC,CAAC,CAAC;QACP,CAAC,CAAC;QAEF,OAAO,IAAI,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,CAAC,MAAM,CAAI,KAAQ;QACrB,OAAO,IAAI,SAAS,CAAI,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,CAAC,aAAa,CAAI,YAAwB;QAC5C,OAAO,SAAS,CAAC,MAAM,CAAI,CAAC,OAAO,EAAE,EAAE,CACnC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CACrC,CAAC;IACN,CAAC;IAED,MAAM,CAAC,IAAI;QACP,OAAO,IAAI,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IACtD,CAAC;IAYD,GAAG,CAAU,QAA8B;QACvC,SAAe,YAAY,CACvB,KAAmB;;gBAEnB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC;gBAE/C,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACpE,MAAM,oBAAoB,GAAG,QAAQ,CAAC,eAAe;oBACjD,CAAC,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC;oBAC9B,CAAC,CAAC,SAAS,CAAC;gBAEhB,OAAO,IAAI,SAAS,CAChB,IAAI,EACJ,QAAQ,CAAC,OAAO,EAChB,oBAAoB,CACvB,CAAC;YACN,CAAC;SAAA;QAED,OAAO,IAAI,SAAS,CAChB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAC9C,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAC9D,CAAC;IACN,CAAC;IAEK,OAAO,CAAC,QAA2B;;YACrC,IAAI,OAAO,GAAiB,IAAI,CAAC;YACjC,OAAO,OAAO,EAAE;gBACZ,IAAI,OAAO,CAAC,OAAO,EAAE;oBACjB,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;iBAC1B;gBAED,IAAI,OAAO,CAAC,eAAe,EAAE;oBACzB,OAAO,GAAG,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC;iBAC7C;qBAAM;oBACH,OAAO,GAAG,SAAS,CAAC;iBACvB;aACJ;QACL,CAAC;KAAA;IAED,IAAI,CAAC,CAAS;QACV,IAAI,CAAC,KAAK,CAAC,EAAE;YACT,OAAO,SAAS,CAAC,IAAI,EAAK,CAAC;SAC9B;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,SAAS,CAChB,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,eAAe;YAChB,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/D,CAAC,CAAC,SAAS,CAClB,CAAC;IACN,CAAC;CACJ;AArGD,4BAqGC;AAED,SAAgB,YAAY,CACxB,IAAoB,EACpB,cAAsB,EACtB,UAAsC;IAEtC,IAAI,cAAc,GAA4B,SAAS,CAAC;IACxD,IAAI,WAAW,GAAG,IAAI,CAAC;IACvB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,SAAe,OAAO;;YAClB,OACI,WAAW,CAAC,eAAe;gBAC3B,CAAC,CAAC,WAAW,CAAC,OAAO,IAAI,YAAY,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EACnE;gBACE,IAAI,CAAC,cAAc,EAAE;oBACjB,cAAc,GAAG,WAAW;yBACvB,eAAe,EAAE;yBACjB,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;wBACf,YAAY,GAAG,CAAC,CAAC;wBACjB,cAAc,GAAG,SAAS,CAAC;wBAC3B,WAAW,GAAG,QAAQ,CAAC;wBACvB,OAAO,QAAQ,CAAC;oBACpB,CAAC,CAAC,CAAC;iBACV;gBAED,WAAW,GAAG,MAAM,cAAc,CAAC;aACtC;YAED,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,YAAY,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE;gBACjE,OAAO,SAAS,CAAC;aACpB;YAED,OAAO;gBACH,KAAK,EAAE,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvC,KAAK,EAAE,WAAW,EAAE;aACvB,CAAC;QACN,CAAC;KAAA;IAED,SAAe,QAAQ;;YACnB,MAAM,IAAI,GAAG,MAAM,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,EAAE;gBACP,OAAO;aACV;YAED,MAAM,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE7B,wBAAwB;YACxB,OAAO,QAAQ,EAAE,CAAC;QACtB,CAAC;KAAA;IAED,MAAM,QAAQ,GAAG,EAAE,CAAC;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,EAAE,EAAE,CAAC,EAAE;QACrC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;KAC7B;IAED,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAC1C,OAAO;IACX,CAAC,CAAC,CAAC;AACP,CAAC;AA5DD,oCA4DC;AAED,SAAgB,gBAAgB,CAAI,IAAoB;IACpD,MAAM,MAAM,GAAG,IAAI,KAAK,EAAK,CAAC;IAC9B,OAAO,IAAI;SACN,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;IAC1B,CAAC,CAAC;SACD,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAPD,4CAOC"}

package/dist/ConnectionResult.d.ts

@@ -0,0 +1,16 @@
+import AspectCreationFailure from "./AspectCreationFailure";
+import RecordCreationFailure from "./RecordCreationFailure";
+export default class ConnectionResult {
+    aspectDefinitionsConnected: number;
+    organizationsConnected: number;
+    datasetsConnected: number;
+    distributionsConnected: number;
+    recordsTrimmed: number;
+    trimStillProcessing: boolean;
+    aspectDefinitionFailures: AspectCreationFailure[];
+    organizationFailures: RecordCreationFailure[];
+    datasetFailures: RecordCreationFailure[];
+    distributionFailures: RecordCreationFailure[];
+    summarize(): string;
+    static combine(...results: ConnectionResult[]): ConnectionResult;
+}

package/dist/ConnectionResult.js

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+class ConnectionResult {
+    constructor() {
+        this.aspectDefinitionsConnected = 0;
+        this.organizationsConnected = 0;
+        this.datasetsConnected = 0;
+        this.distributionsConnected = 0;
+        this.recordsTrimmed = 0;
+        this.trimStillProcessing = false;
+        this.aspectDefinitionFailures = Array();
+        this.organizationFailures = Array();
+        this.datasetFailures = Array();
+        this.distributionFailures = Array();
+    }
+    summarize() {
+        let result = "";
+        result +=
+            "Aspect Definitions Connected: " +
+                this.aspectDefinitionsConnected +
+                "\n";
+        result += "Datasets Connected: " + this.datasetsConnected + "\n";
+        result +=
+            "Distributions Connected: " + this.distributionsConnected + "\n";
+        result +=
+            "Organizations Connected: " + this.organizationsConnected + "\n";
+        result += "Records Trimmed: " + this.recordsTrimmed + "\n";
+        if (this.trimStillProcessing) {
+            result += "(trim still processing) \n";
+        }
+        if (this.aspectDefinitionFailures.length > 0) {
+            result +=
+                "Aspect Definition Failures:\n" +
+                    JSON.stringify(this.aspectDefinitionFailures, undefined, "  ") +
+                    "\n";
+        }
+        if (this.organizationFailures.length > 0) {
+            result +=
+                "Organization Failures:\n" +
+                    JSON.stringify(this.organizationFailures, undefined, "  ") +
+                    "\n";
+        }
+        if (this.datasetFailures.length > 0) {
+            result +=
+                "Dataset Failures:\n" +
+                    JSON.stringify(this.datasetFailures, undefined, "  ") +
+                    "\n";
+        }
+        if (this.distributionFailures.length > 0) {
+            result +=
+                "Distribution Failures:\n" +
+                    JSON.stringify(this.distributionFailures, undefined, "  ") +
+                    "\n";
+        }
+        return result;
+    }
+    static combine(...results) {
+        const total = new ConnectionResult();
+        results.forEach((result) => {
+            total.aspectDefinitionsConnected +=
+                result.aspectDefinitionsConnected;
+            total.organizationsConnected += result.organizationsConnected;
+            total.datasetsConnected += result.datasetsConnected;
+            total.distributionsConnected += result.distributionsConnected;
+            total.recordsTrimmed += result.recordsTrimmed;
+            total.trimStillProcessing =
+                result.trimStillProcessing || total.trimStillProcessing;
+            total.aspectDefinitionFailures.push(...result.aspectDefinitionFailures);
+            total.organizationFailures.push(...result.organizationFailures);
+            total.datasetFailures.push(...result.datasetFailures);
+            total.distributionFailures.push(...result.distributionFailures);
+        });
+        return total;
+    }
+}
+exports.default = ConnectionResult;
+//# sourceMappingURL=ConnectionResult.js.map

package/dist/ConnectionResult.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConnectionResult.js","sourceRoot":"","sources":["../src/ConnectionResult.ts"],"names":[],"mappings":";;AAGA,MAAqB,gBAAgB;IAArC;QACW,+BAA0B,GAAG,CAAC,CAAC;QAC/B,2BAAsB,GAAG,CAAC,CAAC;QAC3B,sBAAiB,GAAG,CAAC,CAAC;QACtB,2BAAsB,GAAG,CAAC,CAAC;QAC3B,mBAAc,GAAG,CAAC,CAAC;QACnB,wBAAmB,GAAG,KAAK,CAAC;QAE5B,6BAAwB,GAAG,KAAK,EAAyB,CAAC;QAC1D,yBAAoB,GAAG,KAAK,EAAyB,CAAC;QACtD,oBAAe,GAAG,KAAK,EAAyB,CAAC;QACjD,yBAAoB,GAAG,KAAK,EAAyB,CAAC;IAsEjE,CAAC;IApEU,SAAS;QACZ,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,MAAM;YACF,gCAAgC;gBAChC,IAAI,CAAC,0BAA0B;gBAC/B,IAAI,CAAC;QACT,MAAM,IAAI,sBAAsB,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QACjE,MAAM;YACF,2BAA2B,GAAG,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACrE,MAAM;YACF,2BAA2B,GAAG,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACrE,MAAM,IAAI,mBAAmB,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3D,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAC1B,MAAM,IAAI,4BAA4B,CAAC;SAC1C;QAED,IAAI,IAAI,CAAC,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1C,MAAM;gBACF,+BAA+B;oBAC/B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,wBAAwB,EAAE,SAAS,EAAE,IAAI,CAAC;oBAC9D,IAAI,CAAC;SACZ;QACD,IAAI,IAAI,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;YACtC,MAAM;gBACF,0BAA0B;oBAC1B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,oBAAoB,EAAE,SAAS,EAAE,IAAI,CAAC;oBAC1D,IAAI,CAAC;SACZ;QACD,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE;YACjC,MAAM;gBACF,qBAAqB;oBACrB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,EAAE,IAAI,CAAC;oBACrD,IAAI,CAAC;SACZ;QACD,IAAI,IAAI,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;YACtC,MAAM;gBACF,0BAA0B;oBAC1B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,oBAAoB,EAAE,SAAS,EAAE,IAAI,CAAC;oBAC1D,IAAI,CAAC;SACZ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,GAAG,OAA2B;QAChD,MAAM,KAAK,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAErC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACvB,KAAK,CAAC,0BAA0B;gBAC5B,MAAM,CAAC,0BAA0B,CAAC;YACtC,KAAK,CAAC,sBAAsB,IAAI,MAAM,CAAC,sBAAsB,CAAC;YAC9D,KAAK,CAAC,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,CAAC;YACpD,KAAK,CAAC,sBAAsB,IAAI,MAAM,CAAC,sBAAsB,CAAC;YAC9D,KAAK,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,CAAC;YAC9C,KAAK,CAAC,mBAAmB;gBACrB,MAAM,CAAC,mBAAmB,IAAI,KAAK,CAAC,mBAAmB,CAAC;YAE5D,KAAK,CAAC,wBAAwB,CAAC,IAAI,CAC/B,GAAG,MAAM,CAAC,wBAAwB,CACrC,CAAC;YACF,KAAK,CAAC,oBAAoB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;YAChE,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;YACtD,KAAK,CAAC,oBAAoB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ;AAjFD,mCAiFC"}

package/dist/ConnectorRecordId.d.ts

@@ -0,0 +1,15 @@
+export declare const RecordTypeMapping: {
+    Organization: string;
+    Dataset: string;
+    Distribution: string;
+    Group: string;
+};
+export declare type RecordType = "Organization" | "Dataset" | "Distribution" | "Group";
+export default class ConnectorRecordId {
+    readonly id: string;
+    readonly type: RecordType;
+    readonly sourceId: string;
+    constructor(id: string, type: RecordType, sourceId: string);
+    toString(): string;
+    private get typeId();
+}

package/dist/ConnectorRecordId.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecordTypeMapping = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+exports.RecordTypeMapping = {
+    Organization: "org",
+    Dataset: "ds",
+    Distribution: "dist",
+    Group: "gp"
+};
+class ConnectorRecordId {
+    constructor(id, type, sourceId) {
+        this.id = id;
+        this.type = type;
+        this.sourceId = sourceId;
+    }
+    toString() {
+        const id = [this.typeId, this.sourceId, this.id].join("-");
+        if (id.length > 100) {
+            // ID is too long, so hash the source record ID.
+            const hashedId = crypto_1.default
+                .createHash("sha256")
+                .update(this.id, "utf8")
+                .digest("hex");
+            const hashedFullId = [this.typeId, this.sourceId, hashedId].join("-");
+            if (hashedFullId.length > 100) {
+                // It's _still_ too long, probably because the sourceId is excessively long.
+                // So hash the full ID, including the sourceId and typeId.
+                return crypto_1.default
+                    .createHash("sha256")
+                    .update(id, "utf8")
+                    .digest("hex");
+            }
+            return hashedFullId;
+        }
+        return id;
+    }
+    get typeId() {
+        return exports.RecordTypeMapping[this.type];
+    }
+}
+exports.default = ConnectorRecordId;
+//# sourceMappingURL=ConnectorRecordId.js.map

package/dist/ConnectorRecordId.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConnectorRecordId.js","sourceRoot":"","sources":["../src/ConnectorRecordId.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAEf,QAAA,iBAAiB,GAAG;IAC7B,YAAY,EAAE,KAAK;IACnB,OAAO,EAAE,IAAI;IACb,YAAY,EAAE,MAAM;IACpB,KAAK,EAAE,IAAI;CACd,CAAC;AAIF,MAAqB,iBAAiB;IAClC,YACa,EAAU,EACV,IAAgB,EAChB,QAAgB;QAFhB,OAAE,GAAF,EAAE,CAAQ;QACV,SAAI,GAAJ,IAAI,CAAY;QAChB,aAAQ,GAAR,QAAQ,CAAQ;IAC1B,CAAC;IAEJ,QAAQ;QACJ,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3D,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE;YACjB,gDAAgD;YAChD,MAAM,QAAQ,GAAG,gBAAM;iBAClB,UAAU,CAAC,QAAQ,CAAC;iBACpB,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC;iBACvB,MAAM,CAAC,KAAK,CAAC,CAAC;YACnB,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,IAAI,CAC5D,GAAG,CACN,CAAC;YACF,IAAI,YAAY,CAAC,MAAM,GAAG,GAAG,EAAE;gBAC3B,4EAA4E;gBAC5E,0DAA0D;gBAC1D,OAAO,gBAAM;qBACR,UAAU,CAAC,QAAQ,CAAC;qBACpB,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC;qBAClB,MAAM,CAAC,KAAK,CAAC,CAAC;aACtB;YACD,OAAO,YAAY,CAAC;SACvB;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAED,IAAY,MAAM;QACd,OAAO,yBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;CACJ;AAlCD,oCAkCC"}

package/dist/JsonConnector.d.ts

@@ -0,0 +1,195 @@
+import { Record } from "./generated/registry/api";
+import AsyncPage from "./AsyncPage";
+import { RecordType } from "./ConnectorRecordId";
+import ConnectionResult from "./ConnectionResult";
+import JsonTransformer from "./JsonTransformer";
+import Registry from "./registry/AuthorizedRegistryClient";
+/**
+ * A base class for connectors for most any JSON-based catalog source.
+ */
+export default class JsonConnector {
+    readonly source: ConnectorSource;
+    readonly transformer: JsonTransformer;
+    readonly registry: Registry;
+    readonly maxConcurrency: number;
+    readonly sourceTag?: string;
+    readonly configData?: JsonConnectorConfig;
+    constructor({ source, transformer, registry, maxConcurrency, sourceTag }: JsonConnectorOptions);
+    readConfigData(): JsonConnectorConfig;
+    createAspectDefinitions(): Promise<ConnectionResult>;
+    createOrganization(organizationJson: object): Promise<Record | Error>;
+    createDataset(datasetJson: object): Promise<Record | Error>;
+    createDistribution(distributionJson: object, datasetJson: object): Promise<Record | Error>;
+    createOrganizations(): Promise<ConnectionResult>;
+    createDatasetsAndDistributions(): Promise<ConnectionResult>;
+    trimRecords(): Promise<ConnectionResult>;
+    /**
+     * Runs the connector, creating aspect definitions, organizations, datasets, and distributions in the
+     * registry as necessary.
+     *
+     * @returns {Promise<ConnectionResult>}
+     * @memberof JsonConnector
+     */
+    run(): Promise<ConnectionResult>;
+    runInteractive(options: JsonConnectorRunInteractiveOptions): void;
+    private shutdownOnIdle;
+    putRecord(record: Record, recordType: RecordType): Promise<Record | Error>;
+    /**
+     * Copy `extras` from connector config data to
+     * records `source` aspect
+     */
+    private attachConnectorDataToSource;
+    private attachConnectorPresetAspects;
+}
+export interface ConnectorSource {
+    /**
+     * The ID of the source. This is used to prefix IDs of datasets, distributions, and organizations
+     * found in this source.
+     */
+    readonly id: string;
+    /**
+     * The user-friendly name of the source.
+     */
+    readonly name: string;
+    /**
+     * This field is not compulsory and JsonConnector will try to locate its value from commandline parameters
+     * before use ConnectorSource.extras as backup --- more for test cases
+     */
+    readonly extras?: JsonConnectorConfigExtraMetaData;
+    /**
+     * This field is not compulsory and JsonConnector will try to locate its value from commandline parameters
+     * before use ConnectorSource.presetRecordAspects as backup --- more for test cases
+     */
+    readonly presetRecordAspects?: JsonConnectorConfigPresetAspect[];
+    /**
+     * Get all of the datasets as pages of objects.
+     *
+     * @returns {AsyncPage<any[]>} A page of datasets.
+     */
+    getJsonDatasets(): AsyncPage<any[]>;
+    /**
+     * Get a particular dataset given its ID.
+     *
+     * @param {string} id The ID of the dataset.
+     * @returns {Promise<any>} The dataset object with the given ID.
+     */
+    getJsonDataset(id: string): Promise<any>;
+    /**
+     * Search datasets for those that have a particular case-insensitive string
+     * in their title.
+     *
+     * @param {string} title The string to search for the in the title.
+     * @param {number} maxResults The maximum number of results to return.
+     * @returns {AsyncPage<any[]>} A page of matching datasets.
+     */
+    searchDatasetsByTitle(title: string, maxResults: number): AsyncPage<any[]>;
+    /**
+     * Gets the distributions of a given dataset.
+     *
+     * @param {object} dataset The dataset.
+     * @returns {AsyncPage<any[]>} A page of distributions of the dataset.
+     */
+    getJsonDistributions(dataset: any): AsyncPage<any[]>;
+    /**
+     * True if the source provides organizations as first-class objects that can be enumerated and retrieved
+     * by ID.  False if organizations are just fields on datasets or distributions, or if they're not
+     * available at all.
+     */
+    readonly hasFirstClassOrganizations: boolean;
+    /**
+     * Enumerates first-class organizations.  If {@link hasFirstClassOrganizations} is false, this
+     * method returns undefined.
+     *
+     * @returns {AsyncPage<any[]>} A page of organizations, or undefined if first-class organizations are not available.
+     */
+    getJsonFirstClassOrganizations(): AsyncPage<any[]>;
+    /**
+     * Gets a first-class organization by ID. If {@link hasFirstClassOrganizations} is false, this
+     * method returns undefined.
+     *
+     * @param {string} id The ID of the organization to retrieve.
+     * @returns {Promise<any>} A promise for the organization, or undefined if first-class organizations are not available.
+     */
+    getJsonFirstClassOrganization(id: string): Promise<any>;
+    /**
+     * Search first-class organizations for those that have a particular case-insensitive string
+     * in their title.
+     *
+     * @param {string} title The string to search for the in the title.
+     * @param {number} maxResults The maximum number of results to return.
+     * @returns {AsyncPage<any[]>} A page of matching organizations, or undefined if first-class organizations are not available.
+     */
+    searchFirstClassOrganizationsByTitle(title: string, maxResults: number): AsyncPage<any[]>;
+    /**
+     * Gets the ID of the publisher of this dataset.  This method will return undefined if {@link hasFirstClassOrganizations}
+     * is false because non-first-class organizations do not have IDs.
+     *
+     * @param {any} dataset The dataset from which to get the publisher ID.
+     * @returns {string} The ID of the dataset's publisher.
+     */
+    getJsonDatasetPublisherId(dataset: any): string;
+    /**
+     * Gets the publisher organization of this dataset.
+     *
+     * @param {any} dataset The dataset from which to get the publisher.
+     * @returns {Promise<object>} A promise for the organization that published this dataset.
+     */
+    getJsonDatasetPublisher(dataset: any): Promise<any>;
+}
+export interface JsonConnectorOptions {
+    source: ConnectorSource;
+    transformer: JsonTransformer;
+    registry: Registry;
+    maxConcurrency?: number;
+    sourceTag?: string;
+}
+export interface JsonConnectorRunInteractiveOptions {
+    timeoutSeconds: number;
+    listenPort: number;
+    transformerOptions: any;
+}
+/**
+ * Connector extra metadata
+ * Will be copied to records' source aspect automatically
+ */
+export declare type JsonConnectorConfigExtraMetaData = {
+    [k: string]: any;
+};
+/**
+ * Any aspects that will be `preset` on any records created by the connector
+ *
+ * opType: operation type; Describe how to add the aspect to the record
+ * - MERGE_LEFT: merge `presetAspect` with records aspects.
+ *   i.e. `presetAspect` will be overwritten by records aspects data if any
+ * - MEREG_RIGHT: merge records aspects with `presetAspect`.
+ *   i.e. records aspects data (if any) will be overwritten by `presetAspect`
+ * - REPLACE: `presetAspect` will replace any existing records aspect
+ * - FILL: `presetAspect` will be added if no existing aspect
+ * Default value (If not specified) will be `MERGE_LEFT`
+ *
+ * recordType:
+ * Describes which type of records this aspect will be added to;
+ * If this field is omitted, the aspect will be added to every records.
+ *
+ * data: Object; aspect data
+ */
+export declare type JsonConnectorConfigPresetAspect = {
+    id: string;
+    recordType?: RecordType;
+    opType?: "MERGE_LEFT" | "REPLACE" | "FILL" | "MERGE_RIGHT";
+    data: {
+        [k: string]: any;
+    };
+};
+export interface JsonConnectorConfig {
+    id: string;
+    name: string;
+    sourceUrl?: string;
+    pageSize?: number;
+    schedule?: string;
+    ignoreHarvestSources?: string[];
+    allowedOrganisationNames?: string[];
+    ignoreOrganisationNames?: string[];
+    extras?: JsonConnectorConfigExtraMetaData;
+    presetRecordAspects?: JsonConnectorConfigPresetAspect[];
+}