@magda/connector-sdk 2.3.2-alpha.0 → 2.3.2-alpha.2

package/dist/index.d.ts

@@ -218,6 +218,9 @@ export declare class ConnectionResult {
     distributionsConnected: number;
     recordsTrimmed: number;
     trimStillProcessing: boolean;
+    organizationsSkiped: number;
+    datasetsSkiped: number;
+    distributionsSkiped: number;
     aspectDefinitionFailures: AspectCreationFailure[];
     organizationFailures: RecordCreationFailure[];
     datasetFailures: RecordCreationFailure[];
@@ -255,6 +258,11 @@ export declare interface ConnectorSource {
      * before use ConnectorSource.presetRecordAspects as backup --- more for test cases
      */
     readonly presetRecordAspects?: JsonConnectorConfigPresetAspect[];
+    /**
+     * This field is not compulsory and JsonConnector will try to locate its value from commandline parameters
+     * before use ConnectorSource.customJsFilterCode as backup --- more for test cases
+     */
+    readonly customJsFilterCode?: string;
     /**
      * Get all of the datasets as pages of objects.
      *
@@ -398,7 +406,9 @@ export declare class JsonConnector {
     readonly maxConcurrency: number;
     readonly sourceTag?: string;
     readonly configData?: JsonConnectorConfig;
+    private readonly recordFilterFunction;
     constructor({ source, transformer, registry, maxConcurrency, sourceTag }: JsonConnectorOptions);
+    createRecordFilterFunction(): RecordFilterFunctionType;
     readConfigData(): JsonConnectorConfig;
     createAspectDefinitions(): Promise<ConnectionResult>;
     createOrganization(organizationJson: object): Promise<Record_2 | Error>;
@@ -437,6 +447,13 @@ export declare interface JsonConnectorConfig {
     ignoreOrganisationNames?: string[];
     extras?: JsonConnectorConfigExtraMetaData;
     presetRecordAspects?: JsonConnectorConfigPresetAspect[];
+    /**
+     * Custom JS code to filter out records
+     * The the following variables are available in scope:
+     * - jsonData: the original record json data before transformation
+     * - type: a string represent the type of the record. e.g. "Organization" | "Dataset" | "Distribution"
+     */
+    customJsFilterCode?: string;
 }
 
 /**
@@ -450,6 +467,8 @@ declare type JsonConnectorConfigExtraMetaData = {
 /**
  * Any aspects that will be `preset` on any records created by the connector
  *
+ * id: aspect Id
+ *
  * opType: operation type; Describe how to add the aspect to the record
  * - MERGE_LEFT: merge `presetAspect` with records aspects.
  *   i.e. `presetAspect` will be overwritten by records aspects data if any
@@ -740,6 +759,8 @@ declare class RecordCreationFailure {
     constructor(id: ConnectorRecordId, parentId: ConnectorRecordId, error: Error);
 }
 
+declare type RecordFilterFunctionType = (jsonData: any, type: "Dataset" | "Distribution" | "Organization") => boolean;
+
 declare class RecordHistoryApi {
     protected basePath: string;
     protected defaultHeaders: any;

package/dist/index.js

@@ -23547,6 +23547,12 @@ class ConnectionResult {
         this.distributionsConnected = 0;
         this.recordsTrimmed = 0;
         this.trimStillProcessing = false;
+        // skipped organizations due to user supplied record filter function
+        this.organizationsSkiped = 0;
+        // skipped dataset records due to user supplied record filter function
+        this.datasetsSkiped = 0;
+        // skipped distribution records due to user supplied record filter function
+        this.distributionsSkiped = 0;
         this.aspectDefinitionFailures = Array();
         this.organizationFailures = Array();
         this.datasetFailures = Array();
@@ -23563,6 +23569,9 @@ class ConnectionResult {
             "Distributions Connected: " + this.distributionsConnected + "\n";
         result +=
             "Organizations Connected: " + this.organizationsConnected + "\n";
+        result += "Datasets Skipped: " + this.datasetsSkiped + "\n";
+        result += "Distributions Skipped: " + this.distributionsSkiped + "\n";
+        result += "Organizations Skipped: " + this.organizationsSkiped + "\n";
         result += "Records Trimmed: " + this.recordsTrimmed + "\n";
         if (this.trimStillProcessing) {
             result += "(trim still processing) \n";
@@ -23601,6 +23610,9 @@ class ConnectionResult {
             total.organizationsConnected += result.organizationsConnected;
             total.datasetsConnected += result.datasetsConnected;
             total.distributionsConnected += result.distributionsConnected;
+            total.organizationsSkiped += result.organizationsSkiped;
+            total.datasetsSkiped += result.datasetsSkiped;
+            total.distributionsSkiped += result.distributionsSkiped;
             total.recordsTrimmed += result.recordsTrimmed;
             total.trimStillProcessing =
                 result.trimStillProcessing || total.trimStillProcessing;
@@ -54146,8 +54158,19 @@ class JsonConnector {
         this.maxConcurrency = maxConcurrency;
         this.sourceTag = sourceTag;
         this.configData = this.readConfigData();
+        this.recordFilterFunction = this.createRecordFilterFunction();
+    }
+    createRecordFilterFunction() {
+        var _a;
+        if (!((_a = this.configData) === null || _a === void 0 ? void 0 : _a.customJsFilterCode)) {
+            return () => true;
+        }
+        const code = this.configData.customJsFilterCode;
+        const filterFunction = new Function("jsonData", "type", code);
+        return filterFunction;
     }
     readConfigData() {
+        var _a, _b;
         try {
             const argv = yargs_1.parse(process_1.default.argv);
             if (!argv) {
@@ -54166,6 +54189,9 @@ class JsonConnector {
                     if (this.source.presetRecordAspects) {
                         configData.presetRecordAspects = this.source.presetRecordAspects;
                     }
+                    if ((_a = this.source) === null || _a === void 0 ? void 0 : _a.customJsFilterCode) {
+                        configData.customJsFilterCode = (_b = this.source) === null || _b === void 0 ? void 0 : _b.customJsFilterCode;
+                    }
                 }
                 else {
                     configData = {
@@ -54178,6 +54204,9 @@ class JsonConnector {
                     if (argv.presetRecordAspects) {
                         configData.presetRecordAspects = argv.presetRecordAspects;
                     }
+                    if (argv.customJsFilterCode) {
+                        configData.customJsFilterCode = argv.customJsFilterCode;
+                    }
                 }
             }
             else {
@@ -54245,6 +54274,10 @@ class JsonConnector {
                     if (!organization) {
                         return;
                     }
+                    if (this.recordFilterFunction(organization, "Organization") === false) {
+                        result.organizationsSkiped++;
+                        return;
+                    }
                     const recordOrError = yield this.createOrganization(organization);
                     if (recordOrError instanceof Error) {
                         result.organizationFailures.push(new RecordCreationFailure_1.default(this.transformer.getIdFromJsonOrganization(organization, this.source.id), undefined, recordOrError));
@@ -54262,11 +54295,19 @@ class JsonConnector {
             const result = new ConnectionResult_1.default();
             const datasets = this.source.getJsonDatasets();
             yield AsyncPage_1.forEachAsync(datasets, this.maxConcurrency, (dataset) => __awaiter(this, void 0, void 0, function* () {
+                if (this.recordFilterFunction(dataset, "Dataset") === false) {
+                    result.datasetsSkiped++;
+                    return;
+                }
                 const record = this.transformer.datasetJsonToRecord(dataset);
                 const distributions = this.source.getJsonDistributions(dataset);
                 if (distributions) {
                     const distributionIds = [];
                     yield AsyncPage_1.forEachAsync(distributions, 1, (distribution) => __awaiter(this, void 0, void 0, function* () {
+                        if (this.recordFilterFunction(distribution, "Distribution") === false) {
+                            result.distributionsSkiped++;
+                            return;
+                        }
                         const recordOrError = yield this.createDistribution(distribution, dataset);
                         if (recordOrError instanceof Error) {
                             result.distributionFailures.push(new RecordCreationFailure_1.default(this.transformer.getIdFromJsonDistribution(distribution, dataset, this.source.id), this.transformer.getIdFromJsonDataset(dataset, this.source.id), recordOrError));
@@ -54293,15 +54334,20 @@ class JsonConnector {
                     if (publisher) {
                         const publisherId = this.transformer.getIdFromJsonOrganization(publisher, this.source.id);
                         if (publisherId) {
-                            const recordOrError = yield this.createOrganization(publisher);
-                            if (recordOrError instanceof Error) {
-                                result.organizationFailures.push(new RecordCreationFailure_1.default(publisherId, undefined, recordOrError));
+                            if (this.recordFilterFunction(publisher, "Organization") === false) {
+                                result.distributionsSkiped++;
                             }
                             else {
-                                record.aspects["dataset-publisher"] = {
-                                    publisher: publisherId.toString()
-                                };
-                                ++result.organizationsConnected;
+                                const recordOrError = yield this.createOrganization(publisher);
+                                if (recordOrError instanceof Error) {
+                                    result.organizationFailures.push(new RecordCreationFailure_1.default(publisherId, undefined, recordOrError));
+                                }
+                                else {
+                                    record.aspects["dataset-publisher"] = {
+                                        publisher: publisherId.toString()
+                                    };
+                                    ++result.organizationsConnected;
+                                }
                             }
                         }
                     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@magda/connector-sdk",
   "description": "MAGDA Connector SDK",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.2-alpha.2",
   "scripts": {
     "prebuild": "rimraf dist tsconfig.tsbuildinfo",
     "compile": "webpack && webpack --env.target=web",
@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "browser": "dist/index-web.js",
   "devDependencies": {
-    "@magda/typescript-common": "^2.3.2-alpha.0",
+    "@magda/typescript-common": "^2.3.2-alpha.2",
     "@microsoft/api-extractor": "7.15.2",
     "ts-loader": "^6.2.1",
     "typescript": "~4.2.4",