@magda/connector-sdk 2.3.1 → 2.3.2-alpha.1

package/dist/index.d.ts

@@ -218,6 +218,9 @@ export declare class ConnectionResult {
     distributionsConnected: number;
     recordsTrimmed: number;
     trimStillProcessing: boolean;
+    organizationsSkiped: number;
+    datasetsSkiped: number;
+    distributionsSkiped: number;
     aspectDefinitionFailures: AspectCreationFailure[];
     organizationFailures: RecordCreationFailure[];
     datasetFailures: RecordCreationFailure[];
@@ -255,6 +258,11 @@ export declare interface ConnectorSource {
      * before use ConnectorSource.presetRecordAspects as backup --- more for test cases
      */
     readonly presetRecordAspects?: JsonConnectorConfigPresetAspect[];
+    /**
+     * This field is not compulsory and JsonConnector will try to locate its value from commandline parameters
+     * before use ConnectorSource.customJsFilterCode as backup --- more for test cases
+     */
+    readonly customJsFilterCode?: string;
     /**
      * Get all of the datasets as pages of objects.
      *
@@ -398,7 +406,9 @@ export declare class JsonConnector {
     readonly maxConcurrency: number;
     readonly sourceTag?: string;
     readonly configData?: JsonConnectorConfig;
+    private readonly recordFilterFunction;
     constructor({ source, transformer, registry, maxConcurrency, sourceTag }: JsonConnectorOptions);
+    createRecordFilterFunction(): RecordFilterFunctionType;
     readConfigData(): JsonConnectorConfig;
     createAspectDefinitions(): Promise<ConnectionResult>;
     createOrganization(organizationJson: object): Promise<Record_2 | Error>;
@@ -437,6 +447,13 @@ export declare interface JsonConnectorConfig {
     ignoreOrganisationNames?: string[];
     extras?: JsonConnectorConfigExtraMetaData;
     presetRecordAspects?: JsonConnectorConfigPresetAspect[];
+    /**
+     * Custom JS code to filter out records
+     * The the following variables are available in scope:
+     * - jsonData: the original record json data before transformation
+     * - type: a string represent the type of the record. e.g. "Organization" | "Dataset" | "Distribution"
+     */
+    customJsFilterCode?: string;
 }
 
 /**
@@ -740,6 +757,8 @@ declare class RecordCreationFailure {
     constructor(id: ConnectorRecordId, parentId: ConnectorRecordId, error: Error);
 }
 
+declare type RecordFilterFunctionType = (jsonData: any, type: "Dataset" | "Distribution" | "Organization") => boolean;
+
 declare class RecordHistoryApi {
     protected basePath: string;
     protected defaultHeaders: any;
@@ -839,9 +858,10 @@ declare class RecordsApi {
      * @param orderByDir Specify the order by direction. Either &#x60;asc&#x60; or &#x60;desc&#x60;
      * @param orderNullFirst Specify whether nulls appear before (&#x60;true&#x60;) or after (&#x60;false&#x60;) non-null values in the sort ordering.
      * @param reversePageTokenOrder When pagination via pageToken, by default, records with smaller pageToken (i.e. older records) will be returned first. When this parameter is set to &#x60;true&#x60;, higher pageToken records (newer records) will be returned.
+     * @param q full text search query
      * @param xMagdaSession Magda internal session id
      */
-    getAll(xMagdaTenantId: number, aspect?: Array<string>, optionalAspect?: Array<string>, pageToken?: string, start?: number, limit?: number, dereference?: boolean, aspectQuery?: Array<string>, aspectOrQuery?: Array<string>, orderBy?: string, orderByDir?: string, orderNullFirst?: boolean, reversePageTokenOrder?: boolean, xMagdaSession?: string): Promise<{
+    getAll(xMagdaTenantId: number, aspect?: Array<string>, optionalAspect?: Array<string>, pageToken?: string, start?: number, limit?: number, dereference?: boolean, aspectQuery?: Array<string>, aspectOrQuery?: Array<string>, orderBy?: string, orderByDir?: string, orderNullFirst?: boolean, reversePageTokenOrder?: boolean, q?: string, xMagdaSession?: string): Promise<{
         response: http.IncomingMessage;
         body: Array<Record_2>;
     }>;
@@ -853,9 +873,10 @@ declare class RecordsApi {
      * @param start The index of the first record to retrieve.  When possible, specify pageToken instead as it will result in better performance.  If this parameter and pageToken are both specified, this parameter is interpreted as the index after the pageToken of the first record to retrieve.
      * @param limit The maximum number of records to receive.  The response will include a token that can be passed as the pageToken parameter to a future request to continue receiving results where this query leaves off.
      * @param reversePageTokenOrder When pagination via pageToken, by default, records with smaller pageToken (i.e. older records) will be returned first. When this parameter is set to &#x60;true&#x60;, higher pageToken records (newer records) will be returned.
+     * @param q full text search query
      * @param xMagdaSession Magda internal session id
      */
-    getAllSummary(xMagdaTenantId: number, pageToken?: string, start?: number, limit?: number, reversePageTokenOrder?: boolean, xMagdaSession?: string): Promise<{
+    getAllSummary(xMagdaTenantId: number, pageToken?: string, start?: number, limit?: number, reversePageTokenOrder?: boolean, q?: string, xMagdaSession?: string): Promise<{
         response: http.IncomingMessage;
         body: Array<RecordSummary>;
     }>;
@@ -902,9 +923,10 @@ declare class RecordsApi {
      * @param aspect The aspects for which to retrieve data, specified as multiple occurrences of this query parameter.  Only records that have all of these aspects will be included in the response.
      * @param aspectQuery Filter the records returned by a value within the aspect JSON. Expressed as &#39;aspectId.path.to.field:value&#39;, url encoded. NOTE: This is an early stage API and may change greatly in the future
      * @param aspectOrQuery Filter the records returned by a value within the aspect JSON. Expressed as &#39;aspectId.path.to.field:value&#39;, url encoded. Queries passing via this parameter will be grouped with OR logic.
+     * @param q full text search query
      * @param xMagdaSession Magda internal session id
      */
-    getCount(xMagdaTenantId: number, aspect?: Array<string>, aspectQuery?: Array<string>, aspectOrQuery?: Array<string>, xMagdaSession?: string): Promise<{
+    getCount(xMagdaTenantId: number, aspect?: Array<string>, aspectQuery?: Array<string>, aspectOrQuery?: Array<string>, q?: string, xMagdaSession?: string): Promise<{
         response: http.IncomingMessage;
         body: CountResponse;
     }>;
@@ -1035,7 +1057,7 @@ declare class RegistryClient {
     getRecord(id: string, aspect?: Array<string>, optionalAspect?: Array<string>, dereference?: boolean): Promise<Record_2 | ServerError>;
     getRecordAspect(id: string, aspectId: string): Promise<any | ServerError>;
     getRecordInFull(id: string): Promise<Record_2>;
-    getRecords<I extends Record_2>(aspect?: Array<string>, optionalAspect?: Array<string>, pageToken?: string, dereference?: boolean, limit?: number, aspectQueries?: string[], aspectOrQuery?: string[], orderBy?: string, orderByDir?: string, orderNullFirst?: boolean, reversePageTokenOrder?: boolean): Promise<RecordsPage<I> | ServerError>;
+    getRecords<I extends Record_2>(aspect?: Array<string>, optionalAspect?: Array<string>, pageToken?: string, dereference?: boolean, limit?: number, aspectQueries?: string[], aspectOrQuery?: string[], orderBy?: string, orderByDir?: string, orderNullFirst?: boolean, reversePageTokenOrder?: boolean, q?: string): Promise<RecordsPage<I> | ServerError>;
     getRecordsPageTokens(aspect?: Array<string>, limit?: number): Promise<string[] | ServerError>;
 }
 

package/dist/index.js

@@ -16906,9 +16906,10 @@ class RecordsApi {
      * @param orderByDir Specify the order by direction. Either `asc` or `desc`
      * @param orderNullFirst Specify whether nulls appear before (`true`) or after (`false`) non-null values in the sort ordering.
      * @param reversePageTokenOrder When pagination via pageToken, by default, records with smaller pageToken (i.e. older records) will be returned first. When this parameter is set to `true`, higher pageToken records (newer records) will be returned.
+     * @param q full text search query
      * @param xMagdaSession Magda internal session id
      */
-    getAll(xMagdaTenantId, aspect, optionalAspect, pageToken, start, limit, dereference, aspectQuery, aspectOrQuery, orderBy, orderByDir, orderNullFirst, reversePageTokenOrder, xMagdaSession) {
+    getAll(xMagdaTenantId, aspect, optionalAspect, pageToken, start, limit, dereference, aspectQuery, aspectOrQuery, orderBy, orderByDir, orderNullFirst, reversePageTokenOrder, q, xMagdaSession) {
         const localVarPath = this.basePath + '/records';
         let queryParameters = {};
         let headerParams = Object.assign({}, this.defaultHeaders);
@@ -16953,6 +16954,9 @@ class RecordsApi {
         if (reversePageTokenOrder !== undefined) {
             queryParameters['reversePageTokenOrder'] = reversePageTokenOrder;
         }
+        if (q !== undefined) {
+            queryParameters['q'] = q;
+        }
         headerParams['X-Magda-Tenant-Id'] = xMagdaTenantId;
         headerParams['X-Magda-Session'] = xMagdaSession;
         let useFormData = false;
@@ -16997,9 +17001,10 @@ class RecordsApi {
      * @param start The index of the first record to retrieve.  When possible, specify pageToken instead as it will result in better performance.  If this parameter and pageToken are both specified, this parameter is interpreted as the index after the pageToken of the first record to retrieve.
      * @param limit The maximum number of records to receive.  The response will include a token that can be passed as the pageToken parameter to a future request to continue receiving results where this query leaves off.
      * @param reversePageTokenOrder When pagination via pageToken, by default, records with smaller pageToken (i.e. older records) will be returned first. When this parameter is set to `true`, higher pageToken records (newer records) will be returned.
+     * @param q full text search query
      * @param xMagdaSession Magda internal session id
      */
-    getAllSummary(xMagdaTenantId, pageToken, start, limit, reversePageTokenOrder, xMagdaSession) {
+    getAllSummary(xMagdaTenantId, pageToken, start, limit, reversePageTokenOrder, q, xMagdaSession) {
         const localVarPath = this.basePath + '/records/summary';
         let queryParameters = {};
         let headerParams = Object.assign({}, this.defaultHeaders);
@@ -17020,6 +17025,9 @@ class RecordsApi {
         if (reversePageTokenOrder !== undefined) {
             queryParameters['reversePageTokenOrder'] = reversePageTokenOrder;
         }
+        if (q !== undefined) {
+            queryParameters['q'] = q;
+        }
         headerParams['X-Magda-Tenant-Id'] = xMagdaTenantId;
         headerParams['X-Magda-Session'] = xMagdaSession;
         let useFormData = false;
@@ -17246,9 +17254,10 @@ class RecordsApi {
      * @param aspect The aspects for which to retrieve data, specified as multiple occurrences of this query parameter.  Only records that have all of these aspects will be included in the response.
      * @param aspectQuery Filter the records returned by a value within the aspect JSON. Expressed as 'aspectId.path.to.field:value', url encoded. NOTE: This is an early stage API and may change greatly in the future
      * @param aspectOrQuery Filter the records returned by a value within the aspect JSON. Expressed as 'aspectId.path.to.field:value', url encoded. Queries passing via this parameter will be grouped with OR logic.
+     * @param q full text search query
      * @param xMagdaSession Magda internal session id
      */
-    getCount(xMagdaTenantId, aspect, aspectQuery, aspectOrQuery, xMagdaSession) {
+    getCount(xMagdaTenantId, aspect, aspectQuery, aspectOrQuery, q, xMagdaSession) {
         const localVarPath = this.basePath + '/records/count';
         let queryParameters = {};
         let headerParams = Object.assign({}, this.defaultHeaders);
@@ -17266,6 +17275,9 @@ class RecordsApi {
         if (aspectOrQuery !== undefined) {
             queryParameters['aspectOrQuery'] = aspectOrQuery;
         }
+        if (q !== undefined) {
+            queryParameters['q'] = q;
+        }
         headerParams['X-Magda-Tenant-Id'] = xMagdaTenantId;
         headerParams['X-Magda-Session'] = xMagdaSession;
         let useFormData = false;
@@ -23535,6 +23547,12 @@ class ConnectionResult {
         this.distributionsConnected = 0;
         this.recordsTrimmed = 0;
         this.trimStillProcessing = false;
+        // skipped organizations due to user supplied record filter function
+        this.organizationsSkiped = 0;
+        // skipped dataset records due to user supplied record filter function
+        this.datasetsSkiped = 0;
+        // skipped distribution records due to user supplied record filter function
+        this.distributionsSkiped = 0;
         this.aspectDefinitionFailures = Array();
         this.organizationFailures = Array();
         this.datasetFailures = Array();
@@ -23551,6 +23569,9 @@ class ConnectionResult {
             "Distributions Connected: " + this.distributionsConnected + "\n";
         result +=
             "Organizations Connected: " + this.organizationsConnected + "\n";
+        result += "Datasets Skipped: " + this.datasetsSkiped + "\n";
+        result += "Distributions Skipped: " + this.distributionsSkiped + "\n";
+        result += "Organizations Skipped: " + this.organizationsSkiped + "\n";
         result += "Records Trimmed: " + this.recordsTrimmed + "\n";
         if (this.trimStillProcessing) {
             result += "(trim still processing) \n";
@@ -23589,6 +23610,9 @@ class ConnectionResult {
             total.organizationsConnected += result.organizationsConnected;
             total.datasetsConnected += result.datasetsConnected;
             total.distributionsConnected += result.distributionsConnected;
+            total.organizationsSkiped += result.organizationsSkiped;
+            total.datasetsSkiped += result.datasetsSkiped;
+            total.distributionsSkiped += result.distributionsSkiped;
             total.recordsTrimmed += result.recordsTrimmed;
             total.trimStillProcessing =
                 result.trimStillProcessing || total.trimStillProcessing;
@@ -29850,8 +29874,8 @@ class RegistryClient {
             }
         });
     }
-    getRecords(aspect, optionalAspect, pageToken, dereference, limit, aspectQueries, aspectOrQuery, orderBy, orderByDir, orderNullFirst, reversePageTokenOrder) {
-        const operation = (pageToken) => () => this.recordsApi.getAll(this.tenantId, aspect, optionalAspect, pageToken, undefined, limit, dereference, aspectQueries, aspectOrQuery, orderBy, orderByDir, orderNullFirst, reversePageTokenOrder, this.jwt);
+    getRecords(aspect, optionalAspect, pageToken, dereference, limit, aspectQueries, aspectOrQuery, orderBy, orderByDir, orderNullFirst, reversePageTokenOrder, q) {
+        const operation = (pageToken) => () => this.recordsApi.getAll(this.tenantId, aspect, optionalAspect, pageToken, undefined, limit, dereference, aspectQueries, aspectOrQuery, orderBy, orderByDir, orderNullFirst, reversePageTokenOrder, q, this.jwt);
         return retry_1.default(operation(pageToken), this.secondsBetweenRetries, this.maxRetries, (e, retriesLeft) => console.log(formatServiceError_1.default("Failed to GET records.", e, retriesLeft)))
             .then((result) => result.body)
             .catch(exports.toServerError("getRecords"));
@@ -54134,8 +54158,19 @@ class JsonConnector {
         this.maxConcurrency = maxConcurrency;
         this.sourceTag = sourceTag;
         this.configData = this.readConfigData();
+        this.recordFilterFunction = this.createRecordFilterFunction();
+    }
+    createRecordFilterFunction() {
+        var _a;
+        if (!((_a = this.configData) === null || _a === void 0 ? void 0 : _a.customJsFilterCode)) {
+            return () => true;
+        }
+        const code = this.configData.customJsFilterCode;
+        const filterFunction = new Function("jsonData", "type", code);
+        return filterFunction;
     }
     readConfigData() {
+        var _a, _b;
         try {
             const argv = yargs_1.parse(process_1.default.argv);
             if (!argv) {
@@ -54154,6 +54189,9 @@ class JsonConnector {
                     if (this.source.presetRecordAspects) {
                         configData.presetRecordAspects = this.source.presetRecordAspects;
                     }
+                    if ((_a = this.source) === null || _a === void 0 ? void 0 : _a.customJsFilterCode) {
+                        configData.customJsFilterCode = (_b = this.source) === null || _b === void 0 ? void 0 : _b.customJsFilterCode;
+                    }
                 }
                 else {
                     configData = {
@@ -54166,6 +54204,9 @@ class JsonConnector {
                     if (argv.presetRecordAspects) {
                         configData.presetRecordAspects = argv.presetRecordAspects;
                     }
+                    if (argv.customJsFilterCode) {
+                        configData.customJsFilterCode = argv.customJsFilterCode;
+                    }
                 }
             }
             else {
@@ -54233,6 +54274,10 @@ class JsonConnector {
                     if (!organization) {
                         return;
                     }
+                    if (this.recordFilterFunction(organization, "Organization") === false) {
+                        result.organizationsSkiped++;
+                        return;
+                    }
                     const recordOrError = yield this.createOrganization(organization);
                     if (recordOrError instanceof Error) {
                         result.organizationFailures.push(new RecordCreationFailure_1.default(this.transformer.getIdFromJsonOrganization(organization, this.source.id), undefined, recordOrError));
@@ -54250,11 +54295,19 @@ class JsonConnector {
             const result = new ConnectionResult_1.default();
             const datasets = this.source.getJsonDatasets();
             yield AsyncPage_1.forEachAsync(datasets, this.maxConcurrency, (dataset) => __awaiter(this, void 0, void 0, function* () {
+                if (this.recordFilterFunction(dataset, "Dataset") === false) {
+                    result.datasetsSkiped++;
+                    return;
+                }
                 const record = this.transformer.datasetJsonToRecord(dataset);
                 const distributions = this.source.getJsonDistributions(dataset);
                 if (distributions) {
                     const distributionIds = [];
                     yield AsyncPage_1.forEachAsync(distributions, 1, (distribution) => __awaiter(this, void 0, void 0, function* () {
+                        if (this.recordFilterFunction(distribution, "Distribution") === false) {
+                            result.distributionsSkiped++;
+                            return;
+                        }
                         const recordOrError = yield this.createDistribution(distribution, dataset);
                         if (recordOrError instanceof Error) {
                             result.distributionFailures.push(new RecordCreationFailure_1.default(this.transformer.getIdFromJsonDistribution(distribution, dataset, this.source.id), this.transformer.getIdFromJsonDataset(dataset, this.source.id), recordOrError));
@@ -54281,15 +54334,20 @@ class JsonConnector {
                     if (publisher) {
                         const publisherId = this.transformer.getIdFromJsonOrganization(publisher, this.source.id);
                         if (publisherId) {
-                            const recordOrError = yield this.createOrganization(publisher);
-                            if (recordOrError instanceof Error) {
-                                result.organizationFailures.push(new RecordCreationFailure_1.default(publisherId, undefined, recordOrError));
+                            if (this.recordFilterFunction(publisher, "Organization") === false) {
+                                result.distributionsSkiped++;
                             }
                             else {
-                                record.aspects["dataset-publisher"] = {
-                                    publisher: publisherId.toString()
-                                };
-                                ++result.organizationsConnected;
+                                const recordOrError = yield this.createOrganization(publisher);
+                                if (recordOrError instanceof Error) {
+                                    result.organizationFailures.push(new RecordCreationFailure_1.default(publisherId, undefined, recordOrError));
+                                }
+                                else {
+                                    record.aspects["dataset-publisher"] = {
+                                        publisher: publisherId.toString()
+                                    };
+                                    ++result.organizationsConnected;
+                                }
                             }
                         }
                     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@magda/connector-sdk",
   "description": "MAGDA Connector SDK",
-  "version": "2.3.1",
+  "version": "2.3.2-alpha.1",
   "scripts": {
     "prebuild": "rimraf dist tsconfig.tsbuildinfo",
     "compile": "webpack && webpack --env.target=web",
@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "browser": "dist/index-web.js",
   "devDependencies": {
-    "@magda/typescript-common": "^2.3.1",
+    "@magda/typescript-common": "^2.3.2-alpha.1",
     "@microsoft/api-extractor": "7.15.2",
     "ts-loader": "^6.2.1",
     "typescript": "~4.2.4",