@transcend-io/cli 4.24.3 → 4.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -2
- package/build/cli-manual-enrichment-push-identifiers.d.ts +3 -0
- package/build/cli-manual-enrichment-push-identifiers.d.ts.map +1 -0
- package/build/cli-manual-enrichment-push-identifiers.js +49 -0
- package/build/cli-manual-enrichment-push-identifiers.js.map +1 -0
- package/build/manual-enrichment/enrichPrivacyRequest.d.ts +17 -0
- package/build/manual-enrichment/enrichPrivacyRequest.d.ts.map +1 -0
- package/build/manual-enrichment/enrichPrivacyRequest.js +72 -0
- package/build/manual-enrichment/enrichPrivacyRequest.js.map +1 -0
- package/build/manual-enrichment/index.d.ts +2 -0
- package/build/manual-enrichment/index.d.ts.map +1 -1
- package/build/manual-enrichment/index.js +2 -0
- package/build/manual-enrichment/index.js.map +1 -1
- package/build/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.d.ts +21 -0
- package/build/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.d.ts.map +1 -0
- package/build/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.js +34 -0
- package/build/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.js.map +1 -0
- package/build/requests/mapCsvRowsToRequestInputs.d.ts.map +1 -1
- package/build/requests/mapCsvRowsToRequestInputs.js +16 -11
- package/build/requests/mapCsvRowsToRequestInputs.js.map +1 -1
- package/build/requests/readCsv.d.ts.map +1 -1
- package/build/requests/readCsv.js +8 -1
- package/build/requests/readCsv.js.map +1 -1
- package/build/tsbuildinfo +1 -1
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -41,10 +41,14 @@
|
|
|
41
41
|
- [Authentication](#authentication-7)
|
|
42
42
|
- [Arguments](#arguments-7)
|
|
43
43
|
- [Usage](#usage-8)
|
|
44
|
-
- [tr-
|
|
44
|
+
- [tr-manual-enrichment-push-identifiers](#tr-manual-enrichment-push-identifiers)
|
|
45
45
|
- [Authentication](#authentication-8)
|
|
46
46
|
- [Arguments](#arguments-8)
|
|
47
47
|
- [Usage](#usage-9)
|
|
48
|
+
- [tr-mark-request-data-silos-completed](#tr-mark-request-data-silos-completed)
|
|
49
|
+
- [Authentication](#authentication-9)
|
|
50
|
+
- [Arguments](#arguments-9)
|
|
51
|
+
- [Usage](#usage-10)
|
|
48
52
|
|
|
49
53
|
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
|
|
50
54
|
|
|
@@ -785,7 +789,9 @@ yarn tr-cron-mark-identifiers-completed --auth=$TRANSCEND_API_KEY --dataSiloId=7
|
|
|
785
789
|
|
|
786
790
|
### tr-manual-enrichment-pull-identifiers
|
|
787
791
|
|
|
788
|
-
This command pulls down the set of privacy requests that are currently pending manual enrichment.
|
|
792
|
+
This command pulls down the set of privacy requests that are currently pending manual enrichment.
|
|
793
|
+
|
|
794
|
+
This is useful for the following workflow:
|
|
789
795
|
|
|
790
796
|
1. Pull identifiers to CSV:
|
|
791
797
|
`yarn tr-manual-enrichment-pull-identifiers --file=./enrichment-requests.csv`
|
|
@@ -842,6 +848,67 @@ With specific concurrency
|
|
|
842
848
|
yarn tr-manual-enrichment-push-identifiers --auth=$TRANSCEND_API_KEY --concurrency=200
|
|
843
849
|
```
|
|
844
850
|
|
|
851
|
+
### tr-manual-enrichment-push-identifiers
|
|
852
|
+
|
|
853
|
+
This command push up a set of identifiers for a set of requests pending manual enrichment.
|
|
854
|
+
|
|
855
|
+
This is useful for the following workflow:
|
|
856
|
+
|
|
857
|
+
1. Pull identifiers to CSV:
|
|
858
|
+
`yarn tr-manual-enrichment-pull-identifiers --file=./enrichment-requests.csv`
|
|
859
|
+
2. Fill out the CSV with additional identifiers
|
|
860
|
+
3. Push updated back to Transcend
|
|
861
|
+
`yarn tr-manual-enrichment-push-identifiers --file=./enrichment-requests.csv`
|
|
862
|
+
|
|
863
|
+
#### Authentication
|
|
864
|
+
|
|
865
|
+
In order to use this cli, you will first need to generate an API key on the Transcend Admin Dashboard (https://app.transcend.io/infrastructure/api-keys).
|
|
866
|
+
|
|
867
|
+
The API key must have the following scopes:
|
|
868
|
+
|
|
869
|
+
- "Manage Request Identity Verification"
|
|
870
|
+
|
|
871
|
+
#### Arguments
|
|
872
|
+
|
|
873
|
+
| Argument | Description | Type | Default | Required |
|
|
874
|
+
| ------------ | ------------------------------------------------------------------------------------ | ------------------ | ----------------------------------- | -------- |
|
|
875
|
+
| auth | The Transcend API capable of pull request information. | string | N/A | true |
|
|
876
|
+
| enricherId | The ID of the Request Enricher to upload to | string - UUID | N/A | true |
|
|
877
|
+
| sombraAuth | The sombra internal key, use for additional authentication when self-hosting sombra. | string | N/A | false |
|
|
878
|
+
| transcendUrl | URL of the Transcend backend. Use https://api.us.transcend.io for US hosting. | string - URL | https://api.transcend.io | false |
|
|
879
|
+
| file | Path to the CSV file where requests will be written to. | string - file-path | ./manual-enrichment-identifiers.csv | false |
|
|
880
|
+
| concurrency | The concurrency to use when uploading requests in parallel. | number | 100 | false |
|
|
881
|
+
|
|
882
|
+
### Usage
|
|
883
|
+
|
|
884
|
+
```sh
|
|
885
|
+
yarn tr-manual-enrichment-push-identifiers --auth=$TRANSCEND_API_KEY --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf
|
|
886
|
+
```
|
|
887
|
+
|
|
888
|
+
Pull to a specific file location
|
|
889
|
+
|
|
890
|
+
```sh
|
|
891
|
+
yarn tr-manual-enrichment-push-identifiers --auth=$TRANSCEND_API_KEY --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf --file=/Users/transcend/Desktop/test.csv
|
|
892
|
+
```
|
|
893
|
+
|
|
894
|
+
For US hosted infrastructure
|
|
895
|
+
|
|
896
|
+
```sh
|
|
897
|
+
yarn tr-manual-enrichment-push-identifiers --auth=$TRANSCEND_API_KEY --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf --transcendUrl=https://api.us.transcend.io
|
|
898
|
+
```
|
|
899
|
+
|
|
900
|
+
With Sombra authentication
|
|
901
|
+
|
|
902
|
+
```sh
|
|
903
|
+
yarn tr-manual-enrichment-push-identifiers --auth=$TRANSCEND_API_KEY --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf --sombraAuth=$SOMBRA_INTERNAL_KEY
|
|
904
|
+
```
|
|
905
|
+
|
|
906
|
+
With specific concurrency
|
|
907
|
+
|
|
908
|
+
```sh
|
|
909
|
+
yarn tr-manual-enrichment-push-identifiers --auth=$TRANSCEND_API_KEY --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf --concurrency=200
|
|
910
|
+
```
|
|
911
|
+
|
|
845
912
|
### tr-mark-request-data-silos-completed
|
|
846
913
|
|
|
847
914
|
This command takes in a CSV of Request IDs as well as a Data Silo ID and marks all associated privacy request jobs as completed.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-manual-enrichment-push-identifiers.d.ts","sourceRoot":"","sources":["../src/cli-manual-enrichment-push-identifiers.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
4
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
5
|
+
};
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
const yargs_parser_1 = __importDefault(require("yargs-parser"));
|
|
8
|
+
const colors_1 = __importDefault(require("colors"));
|
|
9
|
+
const logger_1 = require("./logger");
|
|
10
|
+
const manual_enrichment_1 = require("./manual-enrichment");
|
|
11
|
+
/**
|
|
12
|
+
* Push the the set of manually enriched requests back into the Transcend dashboard
|
|
13
|
+
*
|
|
14
|
+
* Requires an API key with scope to Manage Request Identity Verification
|
|
15
|
+
*
|
|
16
|
+
* Dev Usage:
|
|
17
|
+
* yarn ts-node ./src/cli-manual-enrichment-push-identifiers.ts --auth=asd123 \
|
|
18
|
+
* --file=/Users/michaelfarrell/Desktop/test.csv
|
|
19
|
+
*
|
|
20
|
+
* Standard usage:
|
|
21
|
+
* yarn tr-manual-enrichment-push-identifiers --auth=asd123 \
|
|
22
|
+
* --file=/Users/michaelfarrell/Desktop/test.csv
|
|
23
|
+
*/
|
|
24
|
+
async function main() {
|
|
25
|
+
// Parse command line arguments
|
|
26
|
+
const { file = './manual-enrichment-identifiers.csv', transcendUrl = 'https://api.transcend.io', auth, enricherId, sombraAuth, concurrency = '100', } = (0, yargs_parser_1.default)(process.argv.slice(2));
|
|
27
|
+
// Ensure auth is passed
|
|
28
|
+
if (!auth) {
|
|
29
|
+
logger_1.logger.error(colors_1.default.red('A Transcend API key must be provided. You can specify using --auth=asd123'));
|
|
30
|
+
process.exit(1);
|
|
31
|
+
}
|
|
32
|
+
// Ensure enricherId is provided
|
|
33
|
+
if (!enricherId) {
|
|
34
|
+
logger_1.logger.error(colors_1.default.red('An Enricher ID must be provided. ' +
|
|
35
|
+
'You can specify using --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf'));
|
|
36
|
+
process.exit(1);
|
|
37
|
+
}
|
|
38
|
+
// Pull manual enrichment identifiers
|
|
39
|
+
await (0, manual_enrichment_1.pushManualEnrichmentIdentifiersFromCsv)({
|
|
40
|
+
file,
|
|
41
|
+
transcendUrl,
|
|
42
|
+
enricherId,
|
|
43
|
+
concurrency: parseInt(concurrency, 10),
|
|
44
|
+
auth,
|
|
45
|
+
sombraAuth,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
main();
|
|
49
|
+
//# sourceMappingURL=cli-manual-enrichment-push-identifiers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-manual-enrichment-push-identifiers.js","sourceRoot":"","sources":["../src/cli-manual-enrichment-push-identifiers.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,2DAA6E;AAE7E;;;;;;;;;;;;GAYG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,qCAAqC,EAC5C,YAAY,GAAG,0BAA0B,EACzC,IAAI,EACJ,UAAU,EACV,UAAU,EACV,WAAW,GAAG,KAAK,GACpB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,2EAA2E,CAC5E,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,gCAAgC;IAChC,IAAI,CAAC,UAAU,EAAE;QACf,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,mCAAmC;YACjC,yEAAyE,CAC5E,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,qCAAqC;IACrC,MAAM,IAAA,0DAAsC,EAAC;QAC3C,IAAI;QACJ,YAAY;QACZ,UAAU;QACV,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;QACtC,IAAI;QACJ,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { Got } from 'got';
|
|
2
|
+
import * as t from 'io-ts';
|
|
3
|
+
/**
|
|
4
|
+
* Minimal set required to mark as completed
|
|
5
|
+
*/
|
|
6
|
+
export declare const EnrichPrivacyRequest: t.RecordC<t.StringC, t.StringC>;
|
|
7
|
+
/** Type override */
|
|
8
|
+
export declare type EnrichPrivacyRequest = t.TypeOf<typeof EnrichPrivacyRequest>;
|
|
9
|
+
/**
|
|
10
|
+
* Upload identifiers to a privacy request or mark request as
|
|
11
|
+
*
|
|
12
|
+
* @param sombra - Sombra instance configured to make requests
|
|
13
|
+
* @param request - Request to enricher
|
|
14
|
+
* @param enricherId - The ID of the enricher being uploaded to
|
|
15
|
+
*/
|
|
16
|
+
export declare function enrichPrivacyRequest(sombra: Got, { id, ...rest }: EnrichPrivacyRequest, enricherId: string): Promise<void>;
|
|
17
|
+
//# sourceMappingURL=enrichPrivacyRequest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enrichPrivacyRequest.d.ts","sourceRoot":"","sources":["../../src/manual-enrichment/enrichPrivacyRequest.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC/B,OAAO,KAAK,CAAC,MAAM,OAAO,CAAC;AAM3B;;GAEG;AACH,eAAO,MAAM,oBAAoB,iCAA+B,CAAC;AAEjE,oBAAoB;AACpB,oBAAY,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAEzE;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,GAAG,EACX,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE,oBAAoB,EACrC,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CAmCf"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
|
5
|
+
}) : (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
o[k2] = m[k];
|
|
8
|
+
}));
|
|
9
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
10
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
11
|
+
}) : function(o, v) {
|
|
12
|
+
o["default"] = v;
|
|
13
|
+
});
|
|
14
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
15
|
+
if (mod && mod.__esModule) return mod;
|
|
16
|
+
var result = {};
|
|
17
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
|
+
__setModuleDefault(result, mod);
|
|
19
|
+
return result;
|
|
20
|
+
};
|
|
21
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
22
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
23
|
+
};
|
|
24
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
+
exports.enrichPrivacyRequest = exports.EnrichPrivacyRequest = void 0;
|
|
26
|
+
const t = __importStar(require("io-ts"));
|
|
27
|
+
const logger_1 = require("../logger");
|
|
28
|
+
const uniq_1 = __importDefault(require("lodash/uniq"));
|
|
29
|
+
const colors_1 = __importDefault(require("colors"));
|
|
30
|
+
const splitCsvToList_1 = require("../requests/splitCsvToList");
|
|
31
|
+
/**
|
|
32
|
+
* Minimal set required to mark as completed
|
|
33
|
+
*/
|
|
34
|
+
exports.EnrichPrivacyRequest = t.record(t.string, t.string);
|
|
35
|
+
/**
|
|
36
|
+
* Upload identifiers to a privacy request or mark request as
|
|
37
|
+
*
|
|
38
|
+
* @param sombra - Sombra instance configured to make requests
|
|
39
|
+
* @param request - Request to enricher
|
|
40
|
+
* @param enricherId - The ID of the enricher being uploaded to
|
|
41
|
+
*/
|
|
42
|
+
async function enrichPrivacyRequest(sombra, { id, ...rest }, enricherId) {
|
|
43
|
+
// Pull out the identifiers
|
|
44
|
+
const enrichedIdentifiers = Object.entries(rest).reduce((acc, [key, value]) => {
|
|
45
|
+
const values = (0, uniq_1.default)((0, splitCsvToList_1.splitCsvToList)(value));
|
|
46
|
+
return values.length === 0
|
|
47
|
+
? acc
|
|
48
|
+
: Object.assign(acc, {
|
|
49
|
+
[key]: (0, splitCsvToList_1.splitCsvToList)(value).map((val) => ({ value: val })),
|
|
50
|
+
});
|
|
51
|
+
}, {});
|
|
52
|
+
// Make the GraphQL request
|
|
53
|
+
try {
|
|
54
|
+
await sombra
|
|
55
|
+
.post('v1/enrich-identifiers', {
|
|
56
|
+
headers: {
|
|
57
|
+
'x-transcend-request-id': id,
|
|
58
|
+
'x-transcend-enricher-id': enricherId,
|
|
59
|
+
},
|
|
60
|
+
json: {
|
|
61
|
+
enrichedIdentifiers,
|
|
62
|
+
},
|
|
63
|
+
})
|
|
64
|
+
.json();
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
logger_1.logger.error(colors_1.default.red(`Failed to enricher identifiers for request with id: ${id} - ${err.message} - ${err.response.body}`));
|
|
68
|
+
throw err;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
exports.enrichPrivacyRequest = enrichPrivacyRequest;
|
|
72
|
+
//# sourceMappingURL=enrichPrivacyRequest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enrichPrivacyRequest.js","sourceRoot":"","sources":["../../src/manual-enrichment/enrichPrivacyRequest.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AACA,yCAA2B;AAC3B,sCAAmC;AACnC,uDAA+B;AAC/B,oDAA4B;AAC5B,+DAA4D;AAE5D;;GAEG;AACU,QAAA,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;AAKjE;;;;;;GAMG;AACI,KAAK,UAAU,oBAAoB,CACxC,MAAW,EACX,EAAE,EAAE,EAAE,GAAG,IAAI,EAAwB,EACrC,UAAkB;IAElB,2BAA2B;IAC3B,MAAM,mBAAmB,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,CACrD,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACpB,MAAM,MAAM,GAAG,IAAA,cAAI,EAAC,IAAA,+BAAc,EAAC,KAAK,CAAC,CAAC,CAAC;QAC3C,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC;YACxB,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;gBACjB,CAAC,GAAG,CAAC,EAAE,IAAA,+BAAc,EAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;aAC5D,CAAC,CAAC;IACT,CAAC,EACD,EAA8B,CAC/B,CAAC;IAEF,2BAA2B;IAC3B,IAAI;QACF,MAAM,MAAM;aACT,IAAI,CAAC,uBAAuB,EAAE;YAC7B,OAAO,EAAE;gBACP,wBAAwB,EAAE,EAAE;gBAC5B,yBAAyB,EAAE,UAAU;aACtC;YACD,IAAI,EAAE;gBACJ,mBAAmB;aACpB;SACF,CAAC;aACD,IAAI,EAAE,CAAC;KACX;IAAC,OAAO,GAAG,EAAE;QACZ,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uDAAuD,EAAE,MAAM,GAAG,CAAC,OAAO,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CACpG,CACF,CAAC;QACF,MAAM,GAAG,CAAC;KACX;AACH,CAAC;AAvCD,oDAuCC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/manual-enrichment/index.ts"],"names":[],"mappings":"AAAA,cAAc,wCAAwC,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/manual-enrichment/index.ts"],"names":[],"mappings":"AAAA,cAAc,wCAAwC,CAAC;AACvD,cAAc,0CAA0C,CAAC;AACzD,cAAc,wBAAwB,CAAC"}
|
|
@@ -11,4 +11,6 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
11
11
|
};
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
__exportStar(require("./pullManualEnrichmentIdentifiersToCsv"), exports);
|
|
14
|
+
__exportStar(require("./pushManualEnrichmentIdentifiersFromCsv"), exports);
|
|
15
|
+
__exportStar(require("./enrichPrivacyRequest"), exports);
|
|
14
16
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/manual-enrichment/index.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,yEAAuD"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/manual-enrichment/index.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,yEAAuD;AACvD,2EAAyD;AACzD,yDAAuC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Push a CSV of enriched requests back into Transcend
|
|
3
|
+
*
|
|
4
|
+
* @param options - Options
|
|
5
|
+
* @returns Number of items processed
|
|
6
|
+
*/
|
|
7
|
+
export declare function pushManualEnrichmentIdentifiersFromCsv({ file, auth, sombraAuth, enricherId, concurrency, transcendUrl, }: {
|
|
8
|
+
/** CSV file path */
|
|
9
|
+
file: string;
|
|
10
|
+
/** Transcend API key authentication */
|
|
11
|
+
auth: string;
|
|
12
|
+
/** ID of enricher being uploaded to */
|
|
13
|
+
enricherId: string;
|
|
14
|
+
/** Sombra API key authentication */
|
|
15
|
+
sombraAuth?: string;
|
|
16
|
+
/** Concurrency */
|
|
17
|
+
concurrency?: number;
|
|
18
|
+
/** API URL for Transcend backend */
|
|
19
|
+
transcendUrl?: string;
|
|
20
|
+
}): Promise<number>;
|
|
21
|
+
//# sourceMappingURL=pushManualEnrichmentIdentifiersFromCsv.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pushManualEnrichmentIdentifiersFromCsv.d.ts","sourceRoot":"","sources":["../../src/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts"],"names":[],"mappings":"AAUA;;;;;GAKG;AACH,wBAAsB,sCAAsC,CAAC,EAC3D,IAAI,EACJ,IAAI,EACJ,UAAU,EACV,UAAU,EACV,WAAiB,EACjB,YAAyC,GAC1C,EAAE;IACD,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,oCAAoC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,OAAO,CAAC,MAAM,CAAC,CAsBlB"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.pushManualEnrichmentIdentifiersFromCsv = void 0;
|
|
7
|
+
const colors_1 = __importDefault(require("colors"));
|
|
8
|
+
const bluebird_1 = require("bluebird");
|
|
9
|
+
const logger_1 = require("../logger");
|
|
10
|
+
const graphql_1 = require("../graphql");
|
|
11
|
+
const enrichPrivacyRequest_1 = require("./enrichPrivacyRequest");
|
|
12
|
+
const requests_1 = require("../requests");
|
|
13
|
+
/**
|
|
14
|
+
* Push a CSV of enriched requests back into Transcend
|
|
15
|
+
*
|
|
16
|
+
* @param options - Options
|
|
17
|
+
* @returns Number of items processed
|
|
18
|
+
*/
|
|
19
|
+
async function pushManualEnrichmentIdentifiersFromCsv({ file, auth, sombraAuth, enricherId, concurrency = 100, transcendUrl = 'https://api.transcend.io', }) {
|
|
20
|
+
// Create sombra instance to communicate with
|
|
21
|
+
const sombra = await (0, graphql_1.createSombraGotInstance)(transcendUrl, auth, sombraAuth);
|
|
22
|
+
// Read from CSV
|
|
23
|
+
logger_1.logger.info(colors_1.default.magenta(`Reading "${file}" from disk`));
|
|
24
|
+
const activeResults = (0, requests_1.readCsv)(file, enrichPrivacyRequest_1.EnrichPrivacyRequest);
|
|
25
|
+
// Notify Transcend
|
|
26
|
+
logger_1.logger.info(colors_1.default.magenta(`Enriching "${activeResults.length}" privacy requests.`));
|
|
27
|
+
await (0, bluebird_1.map)(activeResults, async (request) => {
|
|
28
|
+
await (0, enrichPrivacyRequest_1.enrichPrivacyRequest)(sombra, request, enricherId);
|
|
29
|
+
}, { concurrency });
|
|
30
|
+
logger_1.logger.info(colors_1.default.green('Successfully notified Transcend!'));
|
|
31
|
+
return activeResults.length;
|
|
32
|
+
}
|
|
33
|
+
exports.pushManualEnrichmentIdentifiersFromCsv = pushManualEnrichmentIdentifiersFromCsv;
|
|
34
|
+
//# sourceMappingURL=pushManualEnrichmentIdentifiersFromCsv.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pushManualEnrichmentIdentifiersFromCsv.js","sourceRoot":"","sources":["../../src/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,uCAA+B;AAC/B,sCAAmC;AACnC,wCAAqD;AACrD,iEAGgC;AAChC,0CAAsC;AAEtC;;;;;GAKG;AACI,KAAK,UAAU,sCAAsC,CAAC,EAC3D,IAAI,EACJ,IAAI,EACJ,UAAU,EACV,UAAU,EACV,WAAW,GAAG,GAAG,EACjB,YAAY,GAAG,0BAA0B,GAc1C;IACC,6CAA6C;IAC7C,MAAM,MAAM,GAAG,MAAM,IAAA,iCAAuB,EAAC,YAAY,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAE7E,gBAAgB;IAChB,eAAM,CAAC,IAAI,CAAC,gBAAM,CAAC,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,IAAA,kBAAO,EAAC,IAAI,EAAE,2CAAoB,CAAC,CAAC;IAE1D,mBAAmB;IACnB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CAAC,cAAc,aAAa,CAAC,MAAM,qBAAqB,CAAC,CACxE,CAAC;IACF,MAAM,IAAA,cAAG,EACP,aAAa,EACb,KAAK,EAAE,OAAO,EAAE,EAAE;QAChB,MAAM,IAAA,2CAAoB,EAAC,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAC1D,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAC;IAEF,eAAM,CAAC,IAAI,CAAC,gBAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC,CAAC;IAC9D,OAAO,aAAa,CAAC,MAAM,CAAC;AAC9B,CAAC;AA1CD,wFA0CC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mapCsvRowsToRequestInputs.d.ts","sourceRoot":"","sources":["../../src/requests/mapCsvRowsToRequestInputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAEL,sBAAsB,EACtB,aAAa,EACb,cAAc,EACd,cAAc,EACd,yBAAyB,EAC1B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,OAAO,EACL,eAAe,EAKhB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D;;GAEG;AACH,oBAAY,wBAAwB,GAAG;KACpC,CAAC,IAAI,cAAc,CAAC,CAAC,EAAE;QACtB,yBAAyB;QACzB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,0BAA0B;QAC1B,KAAK,EAAE,MAAM,CAAC;KACf,EAAE;CACJ,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,oBAAoB;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,wBAAwB,EAAE,wBAAwB,CAAC;IACnD,+BAA+B;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,mCAAmC;IACnC,WAAW,EAAE,aAAa,CAAC;IAC3B,2BAA2B;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc;IACd,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,2BAA2B;IAC3B,kBAAkB,CAAC,EAAE,yBAAyB,CAAC;IAC/C,uBAAuB;IACvB,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAC9B,uDAAuD;IACvD,MAAM,CAAC,EAAE,sBAAsB,CAAC;IAChC,4CAA4C;IAC5C,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,kCAAkC;IAClC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,6BAA6B;IAC7B,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,cAAc,EAC9B,uBAAuB,EAAE,MAAM,GAC9B,MAAM,
|
|
1
|
+
{"version":3,"file":"mapCsvRowsToRequestInputs.d.ts","sourceRoot":"","sources":["../../src/requests/mapCsvRowsToRequestInputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAEL,sBAAsB,EACtB,aAAa,EACb,cAAc,EACd,cAAc,EACd,yBAAyB,EAC1B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,OAAO,EACL,eAAe,EAKhB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D;;GAEG;AACH,oBAAY,wBAAwB,GAAG;KACpC,CAAC,IAAI,cAAc,CAAC,CAAC,EAAE;QACtB,yBAAyB;QACzB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,0BAA0B;QAC1B,KAAK,EAAE,MAAM,CAAC;KACf,EAAE;CACJ,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,oBAAoB;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,wBAAwB,EAAE,wBAAwB,CAAC;IACnD,+BAA+B;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,mCAAmC;IACnC,WAAW,EAAE,aAAa,CAAC;IAC3B,2BAA2B;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc;IACd,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,2BAA2B;IAC3B,kBAAkB,CAAC,EAAE,yBAAyB,CAAC;IAC/C,uBAAuB;IACvB,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAC9B,uDAAuD;IACvD,MAAM,CAAC,EAAE,sBAAsB,CAAC;IAChC,4CAA4C;IAC5C,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,kCAAkC;IAClC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,6BAA6B;IAC7B,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,cAAc,EAC9B,uBAAuB,EAAE,MAAM,GAC9B,MAAM,CAsBR;AAED;;;;;;;;;GASG;AACH,wBAAgB,yBAAyB,CACvC,aAAa,EAAE,WAAW,EAAE,EAC5B,KAAK,EAAE,cAAc,CAAC,OAAO,eAAe,CAAC,EAC7C,EACE,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,uBAA6B,GAC9B,EAAE;IACD,2BAA2B;IAC3B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,8BAA8B;IAC9B,aAAa,EAAE,aAAa,CAAC;IAC7B,kCAAkC;IAClC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,iCAAiC;IACjC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,6BAA6B;IAC7B,oBAAoB,EAAE,YAAY,EAAE,CAAC;CACtC,GACA,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,mBAAmB,CAAC,EAAE,CAiJjD"}
|
|
@@ -26,9 +26,11 @@ function normalizeIdentifierValue(identifierValue, identifierType, defaultPhoneC
|
|
|
26
26
|
.replace(/[:]/g, '')
|
|
27
27
|
.replace(/[]/g, '')
|
|
28
28
|
.replace(/[A-Za-z]/g, '');
|
|
29
|
-
return normalized
|
|
30
|
-
?
|
|
31
|
-
:
|
|
29
|
+
return !normalized
|
|
30
|
+
? ''
|
|
31
|
+
: normalized.startsWith('+')
|
|
32
|
+
? normalized
|
|
33
|
+
: `+${defaultPhoneCountryCode}${normalized}`;
|
|
32
34
|
}
|
|
33
35
|
return identifierValue;
|
|
34
36
|
}
|
|
@@ -61,15 +63,18 @@ function mapCsvRowsToRequestInputs(requestInputs, state, { columnNameMap, identi
|
|
|
61
63
|
// Only add the identifier if the value exists
|
|
62
64
|
const identifierValue = input[columnName];
|
|
63
65
|
if (identifierValue) {
|
|
64
|
-
|
|
65
|
-
if (
|
|
66
|
-
|
|
66
|
+
const normalized = normalizeIdentifierValue(identifierValue, identifierType, defaultPhoneCountryCode);
|
|
67
|
+
if (normalized) {
|
|
68
|
+
// Initialize
|
|
69
|
+
if (!attestedExtraIdentifiers[identifierType]) {
|
|
70
|
+
attestedExtraIdentifiers[identifierType] = [];
|
|
71
|
+
}
|
|
72
|
+
// Add the identifier
|
|
73
|
+
attestedExtraIdentifiers[identifierType].push({
|
|
74
|
+
value: normalized,
|
|
75
|
+
name: identifierName,
|
|
76
|
+
});
|
|
67
77
|
}
|
|
68
|
-
// Add the identifier
|
|
69
|
-
attestedExtraIdentifiers[identifierType].push({
|
|
70
|
-
value: normalizeIdentifierValue(identifierValue, identifierType, defaultPhoneCountryCode),
|
|
71
|
-
name: identifierName,
|
|
72
|
-
});
|
|
73
78
|
}
|
|
74
79
|
});
|
|
75
80
|
// The extra attributes to upload for this request
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mapCsvRowsToRequestInputs.js","sourceRoot":"","sources":["../../src/requests/mapCsvRowsToRequestInputs.ts"],"names":[],"mappings":";;;AAEA,+DAOqC;AAGrC,2CAMqB;AAGrB,qDAAkD;AA4ClD;;;;;;;GAOG;AACH,SAAgB,wBAAwB,CACtC,eAAuB,EACvB,cAA8B,EAC9B,uBAA+B;IAE/B,kBAAkB;IAClB,IAAI,cAAc,KAAK,8BAAc,CAAC,KAAK,EAAE;QAC3C,OAAO,eAAe,CAAC,WAAW,EAAE,CAAC;KACtC;IAED,yBAAyB;IACzB,IAAI,cAAc,KAAK,8BAAc,CAAC,KAAK,EAAE;QAC3C,MAAM,UAAU,GAAG,eAAe;aAC/B,OAAO,CAAC,sCAAsB,EAAE,EAAE,CAAC;aACnC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;aACpB,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;aACnB,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;aACnB,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;aACpB,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC5B,OAAO,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC;
|
|
1
|
+
{"version":3,"file":"mapCsvRowsToRequestInputs.js","sourceRoot":"","sources":["../../src/requests/mapCsvRowsToRequestInputs.ts"],"names":[],"mappings":";;;AAEA,+DAOqC;AAGrC,2CAMqB;AAGrB,qDAAkD;AA4ClD;;;;;;;GAOG;AACH,SAAgB,wBAAwB,CACtC,eAAuB,EACvB,cAA8B,EAC9B,uBAA+B;IAE/B,kBAAkB;IAClB,IAAI,cAAc,KAAK,8BAAc,CAAC,KAAK,EAAE;QAC3C,OAAO,eAAe,CAAC,WAAW,EAAE,CAAC;KACtC;IAED,yBAAyB;IACzB,IAAI,cAAc,KAAK,8BAAc,CAAC,KAAK,EAAE;QAC3C,MAAM,UAAU,GAAG,eAAe;aAC/B,OAAO,CAAC,sCAAsB,EAAE,EAAE,CAAC;aACnC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;aACpB,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;aACnB,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;aACnB,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;aACpB,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC5B,OAAO,CAAC,UAAU;YAChB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC;gBAC5B,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,IAAI,uBAAuB,GAAG,UAAU,EAAE,CAAC;KAChD;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AA1BD,4DA0BC;AAED;;;;;;;;;GASG;AACH,SAAgB,yBAAyB,CACvC,aAA4B,EAC5B,KAA6C,EAC7C,EACE,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,uBAAuB,GAAG,GAAG,EAAE,KAAK;EAYrC;IAED,+BAA+B;IAC/B,MAAM,aAAa,GAAG,CAAC,SAAqB,EAAU,EAAE,CACtD,KAAK,CAAC,QAAQ,CAAC,aAAa,EAAE,SAAS,CAAC,IAAI,aAAa,CAAC,SAAS,CAAE,CAAC;IACxE,OAAO,aAAa,CAAC,GAAG,CACtB,CAAC,KAAK,EAAiD,EAAE;QACvD,mDAAmD;QACnD,MAAM,wBAAwB,GAA6B,EAAE,CAAC;QAC9D,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC;YAC/B,iCAAiC;aAChC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,UAAU,KAAK,gBAAI,CAAC;aAC/C,OAAO,CAAC,CAAC,CAAC,cAAc,EAAE,UAAU,CAAC,EAAE,EAAE;YACxC,gDAAgD;YAChD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,8BAAc,CAAC,CAAC,QAAQ,CAC3D,cAAqB,CACtB;gBACC,CAAC,CAAE,cAAiC;gBACpC,CAAC,CAAC,8BAAc,CAAC,MAAM,CAAC;YAE1B,8CAA8C;YAC9C,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,eAAe,EAAE;gBACnB,MAAM,UAAU,GAAG,wBAAwB,CACzC,eAAe,EACf,cAAc,EACd,uBAAuB,CACxB,CAAC;gBACF,IAAI,UAAU,EAAE;oBACd,aAAa;oBACb,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,EAAE;wBAC7C,wBAAwB,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;qBAC/C;oBAED,qBAAqB;oBACrB,wBAAwB,CAAC,cAAc,CAAE,CAAC,IAAI,CAAC;wBAC7C,KAAK,EAAE,UAAU;wBACjB,IAAI,EAAE,cAAc;qBACrB,CAAC,CAAC;iBACJ;aACF;QACH,CAAC,CAAC,CAAC;QAEL,kDAAkD;QAClD,MAAM,UAAU,GAAqB,EAAE,CAAC;QACxC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC;YAC9B,gCAAgC;aAC/B,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,UAAU,KAAK,gBAAI,CAAC;aAC/C,OAAO,CAAC,CAAC,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,EAAE;;YACvC,8CAA8C;YAC9C,MAAM,oBAAoB,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;YAC/C,IAAI,oBAAoB,EAAE;gBACxB,oBAAoB;gBACpB,MAAM,OAAO,GACX,CAAA,MAAA,oBAAoB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,aAAa,CAAC,0CAC5D,IAAI,MAAK,cAAc,CAAC;gBAC9B,UAAU,CAAC,IAAI,CAAC;oBACd,MAAM,EAAE,OAAO;wBACb,CAAC,CAAC,IAAA,+BAAc,EAAC,oBAAoB,CAAC;wBACtC,CAAC,CAAC,oBAAoB;oBACxB,GAAG,EAAE,aAAa;iBACnB,CAAC,CAAC;aACJ;QACH,CAAC,CAAC,CAAC;QAEL,MAAM,iBAAiB,GAAG,aAAa,CAAC,sBAAU,CAAC,WAAW,CAAC,CAAC;QAChE,MAAM,qBAAqB,GAAG,aAAa,CAAC,sBAAU,CAAC,WAAW,CAAC,CAAC;QACpE,OAAO;YACL,KAAK;YACL;gBACE,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,KAAK,CAAC,CAAC;gBAC7C,wBAAwB;gBACxB,UAAU;gBACV,cAAc,EAAE,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,cAAc,CAAC,CAAC;gBAC/D,WAAW,EACT,iBAAiB,KAAK,sBAAU;oBAC9B,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,4BAA4B,EAAE,iBAAK,CAAC;oBACrD,CAAC,CAAC,KAAK,CAAC,QAAQ,CACZ,4BAA4B,EAC5B,KAAK,CAAC,iBAAiB,CAAC,CACzB;gBACP,WAAW,EACT,qBAAqB,KAAK,sBAAU;oBAClC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,0BAA0B,EAAE,iBAAK,CAAC;oBACnD,CAAC,CAAC,KAAK,CAAC,QAAQ,CACZ,0BAA0B,EAC1B,KAAK,CAAC,qBAAqB,CAAC,CAC7B;gBACP,GAAG,CAAC,aAAa,CAAC,sBAAU,CAAC,MAAM,CAAC,KAAK,gBAAI;oBAC7C,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,MAAM,CAAC,CAAC;oBACrC,CAAC,CAAC;wBACE,MAAM,EAAE,KAAK,CAAC,QAAQ,CACpB,kBAAkB,EAClB,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,MAAM,CAAC,CAAC,CACxC;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,aAAa,CAAC,sBAAU,CAAC,OAAO,CAAC,KAAK,gBAAI;oBAC9C,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,OAAO,CAAC,CAAC;oBACtC,CAAC,CAAC;wBACE,OAAO,EAAE,KAAK,CAAC,QAAQ,CACrB,iBAAiB,EACjB,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,OAAO,CAAC,CAAC,CACvB;qBACpB;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,aAAa,CAAC,sBAAU,CAAC,kBAAkB,CAAC,KAAK,gBAAI;oBACzD,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,kBAAkB,CAAC,CAAC;oBACjD,CAAC,CAAC;wBACE,kBAAkB,EAAE,KAAK,CAAC,QAAQ,CAChC,4BAA4B,EAC5B,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,kBAAkB,CAAC,CAAC,CACvB;qBAC/B;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,aAAa,CAAC,sBAAU,CAAC,aAAa,CAAC,KAAK,gBAAI;oBACpD,KAAK,CAAC,QAAQ,CACZ,uBAAuB,EACvB,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,aAAa,CAAC,CAAC,CAC/C,KAAK,gBAAI;oBACV,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,aAAa,CAAC,CAAC;oBAC5C,CAAC,CAAC;wBACE,MAAM,EAAE,KAAK,CAAC,QAAQ,CACpB,uBAAuB,EACvB,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,aAAa,CAAC,CAAC,CACrB;qBAC5B;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,aAAa,CAAC,sBAAU,CAAC,SAAS,CAAC,KAAK,gBAAI;oBAChD,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,SAAS,CAAC,CAAC;oBACxC,CAAC,CAAC;wBACE,SAAS,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,SAAS,CAAC,CAAC,CAAC;qBAChE;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,aAAa,CAAC,sBAAU,CAAC,WAAW,CAAC,KAAK,gBAAI;oBAClD,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,WAAW,CAAC,CAAC;oBAC1C,CAAC,CAAC;wBACE,WAAW,EAAE,IAAA,+BAAc,EACzB,KAAK,CAAC,aAAa,CAAC,sBAAU,CAAC,WAAW,CAAC,CAAC,CAC7C;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAtKD,8DAsKC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"readCsv.d.ts","sourceRoot":"","sources":["../../src/requests/readCsv.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,CAAC,MAAM,OAAO,CAAC;AAI3B;;;;;;;GAOG;AACH,wBAAgB,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,EACrC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,CAAC,EACR,OAAO,GAAE,OAA2B,GACnC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"readCsv.d.ts","sourceRoot":"","sources":["../../src/requests/readCsv.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,CAAC,MAAM,OAAO,CAAC;AAI3B;;;;;;;GAOG;AACH,wBAAgB,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,EACrC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,CAAC,EACR,OAAO,GAAE,OAA2B,GACnC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAkBf"}
|
|
@@ -36,8 +36,15 @@ const type_utils_1 = require("@transcend-io/type-utils");
|
|
|
36
36
|
* @returns The JSON data
|
|
37
37
|
*/
|
|
38
38
|
function readCsv(pathToFile, codec, options = { columns: true }) {
|
|
39
|
+
// read file contents and parse
|
|
39
40
|
const fileContent = (0, sync_1.default)((0, fs_1.readFileSync)(pathToFile, 'utf-8'), options);
|
|
40
|
-
|
|
41
|
+
// validate codec
|
|
42
|
+
const data = (0, type_utils_1.decodeCodec)(t.array(codec), fileContent);
|
|
43
|
+
// remove any special characters from object keys
|
|
44
|
+
const parsed = data.map((datum) => Object.entries(datum).reduce((acc, [key, value]) => Object.assign(acc, {
|
|
45
|
+
[key.replace(/[^a-z_.\-A-Z ]/g, '')]: value,
|
|
46
|
+
}), {}));
|
|
47
|
+
return parsed;
|
|
41
48
|
}
|
|
42
49
|
exports.readCsv = readCsv;
|
|
43
50
|
//# sourceMappingURL=readCsv.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"readCsv.js","sourceRoot":"","sources":["../../src/requests/readCsv.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AACA,8DAAuC;AACvC,2BAAkC;AAClC,yCAA2B;AAE3B,yDAAuD;AAEvD;;;;;;;GAOG;AACH,SAAgB,OAAO,CACrB,UAAkB,EAClB,KAAQ,EACR,UAAmB,EAAE,OAAO,EAAE,IAAI,EAAE;IAEpC,MAAM,WAAW,GAAG,IAAA,cAAK,EAAC,IAAA,iBAAY,EAAC,UAAU,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"readCsv.js","sourceRoot":"","sources":["../../src/requests/readCsv.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AACA,8DAAuC;AACvC,2BAAkC;AAClC,yCAA2B;AAE3B,yDAAuD;AAEvD;;;;;;;GAOG;AACH,SAAgB,OAAO,CACrB,UAAkB,EAClB,KAAQ,EACR,UAAmB,EAAE,OAAO,EAAE,IAAI,EAAE;IAEpC,+BAA+B;IAC/B,MAAM,WAAW,GAAG,IAAA,cAAK,EAAC,IAAA,iBAAY,EAAC,UAAU,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;IAEtE,iBAAiB;IACjB,MAAM,IAAI,GAAG,IAAA,wBAAW,EAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC;IAEtD,iDAAiD;IACjD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAChC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAC1B,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACpB,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;QACjB,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK;KAC5C,CAAC,EACJ,EAAO,CACR,CACF,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAtBD,0BAsBC"}
|