@jsforce/jsforce-node 3.0.0-next.1 → 3.0.0-next.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api/bulk.d.ts +38 -229
- package/lib/api/bulk.js +26 -720
- package/lib/api/bulk2.d.ts +324 -0
- package/lib/api/bulk2.js +800 -0
- package/lib/connection.d.ts +2 -1
- package/lib/http-api.js +38 -0
- package/lib/index.d.ts +1 -0
- package/lib/index.js +1 -0
- package/lib/soap.js +11 -0
- package/lib/util/get-body-size.d.ts +4 -0
- package/lib/util/get-body-size.js +39 -0
- package/package.json +3 -1
package/lib/api/bulk.js
CHANGED
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.
|
|
6
|
+
exports.Bulk = exports.Batch = exports.Job = void 0;
|
|
7
7
|
/**
|
|
8
8
|
* @file Manages Salesforce Bulk API related operations
|
|
9
9
|
* @author Shinichi Tomita <shinichi.tomita@gmail.com>
|
|
@@ -15,7 +15,7 @@ const record_stream_1 = require("../record-stream");
|
|
|
15
15
|
const http_api_1 = __importDefault(require("../http-api"));
|
|
16
16
|
const jsforce_1 = require("../jsforce");
|
|
17
17
|
const stream_2 = require("../util/stream");
|
|
18
|
-
const
|
|
18
|
+
const is_1 = __importDefault(require("@sindresorhus/is"));
|
|
19
19
|
/**
|
|
20
20
|
* Class for Bulk API Job
|
|
21
21
|
*/
|
|
@@ -264,17 +264,6 @@ class PollingTimeoutError extends Error {
|
|
|
264
264
|
this.batchId = batchId;
|
|
265
265
|
}
|
|
266
266
|
}
|
|
267
|
-
class JobPollingTimeoutError extends Error {
|
|
268
|
-
jobId;
|
|
269
|
-
/**
|
|
270
|
-
*
|
|
271
|
-
*/
|
|
272
|
-
constructor(message, jobId) {
|
|
273
|
-
super(message);
|
|
274
|
-
this.name = 'JobPollingTimeout';
|
|
275
|
-
this.jobId = jobId;
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
267
|
/*--------------------------------------------*/
|
|
279
268
|
/**
|
|
280
269
|
* Batch (extends Writable)
|
|
@@ -387,13 +376,14 @@ class Batch extends stream_1.Writable {
|
|
|
387
376
|
this.once('response', resolve);
|
|
388
377
|
this.once('error', reject);
|
|
389
378
|
});
|
|
390
|
-
if (
|
|
379
|
+
if (is_1.default.nodeStream(input)) {
|
|
391
380
|
// if input has stream.Readable interface
|
|
392
381
|
input.pipe(this._dataStream);
|
|
393
382
|
}
|
|
394
383
|
else {
|
|
395
|
-
|
|
396
|
-
|
|
384
|
+
const recordData = structuredClone(input);
|
|
385
|
+
if (Array.isArray(recordData)) {
|
|
386
|
+
for (const record of recordData) {
|
|
397
387
|
for (const key of Object.keys(record)) {
|
|
398
388
|
if (typeof record[key] === 'boolean') {
|
|
399
389
|
record[key] = String(record[key]);
|
|
@@ -403,8 +393,8 @@ class Batch extends stream_1.Writable {
|
|
|
403
393
|
}
|
|
404
394
|
this.end();
|
|
405
395
|
}
|
|
406
|
-
else if (typeof
|
|
407
|
-
this._dataStream.write(
|
|
396
|
+
else if (typeof recordData === 'string') {
|
|
397
|
+
this._dataStream.write(recordData, 'utf8');
|
|
408
398
|
this._dataStream.end();
|
|
409
399
|
}
|
|
410
400
|
}
|
|
@@ -452,9 +442,13 @@ class Batch extends stream_1.Writable {
|
|
|
452
442
|
throw new Error('Batch not started.');
|
|
453
443
|
}
|
|
454
444
|
const startTime = new Date().getTime();
|
|
445
|
+
const endTime = startTime + timeout;
|
|
446
|
+
if (timeout === 0) {
|
|
447
|
+
throw new PollingTimeoutError(`Skipping polling because of timeout = 0ms. Job Id = ${jobId} | Batch Id = ${batchId}`, jobId, batchId);
|
|
448
|
+
}
|
|
455
449
|
const poll = async () => {
|
|
456
450
|
const now = new Date().getTime();
|
|
457
|
-
if (
|
|
451
|
+
if (endTime < now) {
|
|
458
452
|
const err = new PollingTimeoutError('Polling time out. Job Id = ' + jobId + ' , batch Id = ' + batchId, jobId, batchId);
|
|
459
453
|
this.emit('error', err);
|
|
460
454
|
return;
|
|
@@ -479,7 +473,7 @@ class Batch extends stream_1.Writable {
|
|
|
479
473
|
this.retrieve();
|
|
480
474
|
}
|
|
481
475
|
else {
|
|
482
|
-
this.emit('
|
|
476
|
+
this.emit('inProgress', res);
|
|
483
477
|
setTimeout(poll, interval);
|
|
484
478
|
}
|
|
485
479
|
};
|
|
@@ -526,7 +520,8 @@ class Batch extends stream_1.Writable {
|
|
|
526
520
|
}
|
|
527
521
|
}
|
|
528
522
|
/**
|
|
529
|
-
* Fetch query result as a record stream
|
|
523
|
+
* Fetch query batch result as a record stream
|
|
524
|
+
*
|
|
530
525
|
* @param {String} resultId - Result id
|
|
531
526
|
* @returns {RecordStream} - Record stream, convertible to CSV data stream
|
|
532
527
|
*/
|
|
@@ -575,22 +570,6 @@ class BulkApi extends http_api_1.default {
|
|
|
575
570
|
};
|
|
576
571
|
}
|
|
577
572
|
}
|
|
578
|
-
class BulkApiV2 extends http_api_1.default {
|
|
579
|
-
hasErrorInResponseBody(body) {
|
|
580
|
-
return (Array.isArray(body) &&
|
|
581
|
-
typeof body[0] === 'object' &&
|
|
582
|
-
'errorCode' in body[0]);
|
|
583
|
-
}
|
|
584
|
-
isSessionExpired(response) {
|
|
585
|
-
return (response.statusCode === 401 && /INVALID_SESSION_ID/.test(response.body));
|
|
586
|
-
}
|
|
587
|
-
parseError(body) {
|
|
588
|
-
return {
|
|
589
|
-
errorCode: body[0].errorCode,
|
|
590
|
-
message: body[0].message,
|
|
591
|
-
};
|
|
592
|
-
}
|
|
593
|
-
}
|
|
594
573
|
/*--------------------------------------------*/
|
|
595
574
|
/**
|
|
596
575
|
* Class for Bulk API
|
|
@@ -602,13 +581,16 @@ class Bulk {
|
|
|
602
581
|
_logger;
|
|
603
582
|
/**
|
|
604
583
|
* Polling interval in milliseconds
|
|
584
|
+
*
|
|
585
|
+
* Default: 1000 (1 second)
|
|
605
586
|
*/
|
|
606
587
|
pollInterval = 1000;
|
|
607
588
|
/**
|
|
608
589
|
* Polling timeout in milliseconds
|
|
609
|
-
*
|
|
590
|
+
*
|
|
591
|
+
* Default: 30000 (30 seconds)
|
|
610
592
|
*/
|
|
611
|
-
pollTimeout =
|
|
593
|
+
pollTimeout = 30000;
|
|
612
594
|
/**
|
|
613
595
|
*
|
|
614
596
|
*/
|
|
@@ -633,9 +615,7 @@ class Bulk {
|
|
|
633
615
|
let options = {};
|
|
634
616
|
if (typeof optionsOrInput === 'string' ||
|
|
635
617
|
Array.isArray(optionsOrInput) ||
|
|
636
|
-
|
|
637
|
-
'pipe' in optionsOrInput &&
|
|
638
|
-
typeof optionsOrInput.pipe === 'function')) {
|
|
618
|
+
is_1.default.nodeStream(optionsOrInput)) {
|
|
639
619
|
// when options is not plain hash object, it is omitted
|
|
640
620
|
input = optionsOrInput;
|
|
641
621
|
}
|
|
@@ -660,7 +640,7 @@ class Bulk {
|
|
|
660
640
|
/**
|
|
661
641
|
* Execute bulk query and get record stream
|
|
662
642
|
*/
|
|
663
|
-
query(soql) {
|
|
643
|
+
async query(soql) {
|
|
664
644
|
const m = soql.replace(/\([\s\S]+\)/g, '').match(/FROM\s+(\w+)/i);
|
|
665
645
|
if (!m) {
|
|
666
646
|
throw new Error('No sobject type found in query, maybe caused by invalid SOQL.');
|
|
@@ -668,19 +648,9 @@ class Bulk {
|
|
|
668
648
|
const type = m[1];
|
|
669
649
|
const recordStream = new record_stream_1.Parsable();
|
|
670
650
|
const dataStream = recordStream.stream('csv');
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
const streams = results.map((result) => this.job(result.jobId)
|
|
675
|
-
.batch(result.batchId)
|
|
676
|
-
.result(result.id)
|
|
677
|
-
.stream());
|
|
678
|
-
(0, multistream_1.default)(streams).pipe(dataStream);
|
|
679
|
-
}
|
|
680
|
-
catch (err) {
|
|
681
|
-
recordStream.emit('error', err);
|
|
682
|
-
}
|
|
683
|
-
})();
|
|
651
|
+
const results = await this.load(type, 'query', soql);
|
|
652
|
+
const streams = results.map((result) => this.job(result.jobId).batch(result.batchId).result(result.id).stream());
|
|
653
|
+
(0, multistream_1.default)(streams).pipe(dataStream);
|
|
684
654
|
return recordStream;
|
|
685
655
|
}
|
|
686
656
|
/**
|
|
@@ -700,673 +670,9 @@ class Bulk {
|
|
|
700
670
|
}
|
|
701
671
|
}
|
|
702
672
|
exports.Bulk = Bulk;
|
|
703
|
-
class BulkV2 {
|
|
704
|
-
#connection;
|
|
705
|
-
/**
|
|
706
|
-
* Polling interval in milliseconds
|
|
707
|
-
*/
|
|
708
|
-
pollInterval = 1000;
|
|
709
|
-
/**
|
|
710
|
-
* Polling timeout in milliseconds
|
|
711
|
-
* @type {Number}
|
|
712
|
-
*/
|
|
713
|
-
pollTimeout = 10000;
|
|
714
|
-
constructor(connection) {
|
|
715
|
-
this.#connection = connection;
|
|
716
|
-
}
|
|
717
|
-
/**
|
|
718
|
-
* Create an instance of an ingest job object.
|
|
719
|
-
*
|
|
720
|
-
* @params {NewIngestJobOptions} options object
|
|
721
|
-
* @returns {IngestJobV2} An ingest job instance
|
|
722
|
-
* @example
|
|
723
|
-
* // Upsert records to the Account object.
|
|
724
|
-
*
|
|
725
|
-
* const job = connection.bulk2.createJob({
|
|
726
|
-
* operation: 'insert'
|
|
727
|
-
* object: 'Account',
|
|
728
|
-
* });
|
|
729
|
-
*
|
|
730
|
-
* // create the job in the org
|
|
731
|
-
* await job.open()
|
|
732
|
-
*
|
|
733
|
-
* // upload data
|
|
734
|
-
* await job.uploadData(csvFile)
|
|
735
|
-
*
|
|
736
|
-
* // finished uploading data, mark it as ready for processing
|
|
737
|
-
* await job.close()
|
|
738
|
-
*/
|
|
739
|
-
createJob(options) {
|
|
740
|
-
return new IngestJobV2({
|
|
741
|
-
connection: this.#connection,
|
|
742
|
-
jobInfo: options,
|
|
743
|
-
pollingOptions: this,
|
|
744
|
-
});
|
|
745
|
-
}
|
|
746
|
-
/**
|
|
747
|
-
* Get a ingest job instance specified by a given job ID
|
|
748
|
-
*
|
|
749
|
-
* @param options Options object with a job ID
|
|
750
|
-
* @returns IngestJobV2 An ingest job
|
|
751
|
-
*/
|
|
752
|
-
job(options) {
|
|
753
|
-
return new IngestJobV2({
|
|
754
|
-
connection: this.#connection,
|
|
755
|
-
jobInfo: options,
|
|
756
|
-
pollingOptions: this,
|
|
757
|
-
});
|
|
758
|
-
}
|
|
759
|
-
/**
|
|
760
|
-
* Create, upload, and start bulkload job
|
|
761
|
-
*/
|
|
762
|
-
async loadAndWaitForResults(options) {
|
|
763
|
-
if (!options.pollTimeout)
|
|
764
|
-
options.pollTimeout = this.pollTimeout;
|
|
765
|
-
if (!options.pollInterval)
|
|
766
|
-
options.pollInterval = this.pollInterval;
|
|
767
|
-
const job = this.createJob(options);
|
|
768
|
-
try {
|
|
769
|
-
await job.open();
|
|
770
|
-
await job.uploadData(options.input);
|
|
771
|
-
await job.close();
|
|
772
|
-
await job.poll(options.pollInterval, options.pollTimeout);
|
|
773
|
-
return await job.getAllResults();
|
|
774
|
-
}
|
|
775
|
-
catch (error) {
|
|
776
|
-
const err = error;
|
|
777
|
-
if (err.name !== 'JobPollingTimeoutError') {
|
|
778
|
-
// fires off one last attempt to clean up and ignores the result | error
|
|
779
|
-
job.delete().catch((ignored) => ignored);
|
|
780
|
-
}
|
|
781
|
-
throw err;
|
|
782
|
-
}
|
|
783
|
-
}
|
|
784
|
-
/**
|
|
785
|
-
* Execute bulk query and get records
|
|
786
|
-
*
|
|
787
|
-
* Default timeout: 10000ms
|
|
788
|
-
*
|
|
789
|
-
* @param soql SOQL query
|
|
790
|
-
* @param BulkV2PollingOptions options object
|
|
791
|
-
*
|
|
792
|
-
* @returns Record[]
|
|
793
|
-
*/
|
|
794
|
-
async query(soql, options) {
|
|
795
|
-
const queryJob = new QueryJobV2({
|
|
796
|
-
connection: this.#connection,
|
|
797
|
-
operation: options?.scanAll ? 'queryAll' : 'query',
|
|
798
|
-
query: soql,
|
|
799
|
-
pollingOptions: this,
|
|
800
|
-
});
|
|
801
|
-
try {
|
|
802
|
-
await queryJob.open();
|
|
803
|
-
await queryJob.poll(options?.pollInterval, options?.pollTimeout);
|
|
804
|
-
return await queryJob.getResults();
|
|
805
|
-
}
|
|
806
|
-
catch (error) {
|
|
807
|
-
const err = error;
|
|
808
|
-
if (err.name !== 'JobPollingTimeoutError') {
|
|
809
|
-
// fires off one last attempt to clean up and ignores the result | error
|
|
810
|
-
queryJob.delete().catch((ignored) => ignored);
|
|
811
|
-
}
|
|
812
|
-
throw err;
|
|
813
|
-
}
|
|
814
|
-
}
|
|
815
|
-
}
|
|
816
|
-
exports.BulkV2 = BulkV2;
|
|
817
|
-
class QueryJobV2 extends events_1.EventEmitter {
|
|
818
|
-
#connection;
|
|
819
|
-
#operation;
|
|
820
|
-
#query;
|
|
821
|
-
#pollingOptions;
|
|
822
|
-
#queryResults;
|
|
823
|
-
#error;
|
|
824
|
-
jobInfo;
|
|
825
|
-
locator;
|
|
826
|
-
finished = false;
|
|
827
|
-
constructor(options) {
|
|
828
|
-
super();
|
|
829
|
-
this.#connection = options.connection;
|
|
830
|
-
this.#operation = options.operation;
|
|
831
|
-
this.#query = options.query;
|
|
832
|
-
this.#pollingOptions = options.pollingOptions;
|
|
833
|
-
// default error handler to keep the latest error
|
|
834
|
-
this.on('error', (error) => (this.#error = error));
|
|
835
|
-
}
|
|
836
|
-
/**
|
|
837
|
-
* Creates a query job
|
|
838
|
-
*/
|
|
839
|
-
async open() {
|
|
840
|
-
try {
|
|
841
|
-
this.jobInfo = await this.createQueryRequest({
|
|
842
|
-
method: 'POST',
|
|
843
|
-
path: '',
|
|
844
|
-
body: JSON.stringify({
|
|
845
|
-
operation: this.#operation,
|
|
846
|
-
query: this.#query,
|
|
847
|
-
}),
|
|
848
|
-
headers: {
|
|
849
|
-
'Content-Type': 'application/json; charset=utf-8',
|
|
850
|
-
},
|
|
851
|
-
responseType: 'application/json',
|
|
852
|
-
});
|
|
853
|
-
this.emit('open');
|
|
854
|
-
}
|
|
855
|
-
catch (err) {
|
|
856
|
-
this.emit('error', err);
|
|
857
|
-
throw err;
|
|
858
|
-
}
|
|
859
|
-
}
|
|
860
|
-
/**
|
|
861
|
-
* Set the status to abort
|
|
862
|
-
*/
|
|
863
|
-
async abort() {
|
|
864
|
-
try {
|
|
865
|
-
const state = 'Aborted';
|
|
866
|
-
this.jobInfo = await this.createQueryRequest({
|
|
867
|
-
method: 'PATCH',
|
|
868
|
-
path: `/${this.jobInfo?.id}`,
|
|
869
|
-
body: JSON.stringify({ state }),
|
|
870
|
-
headers: { 'Content-Type': 'application/json; charset=utf-8' },
|
|
871
|
-
responseType: 'application/json',
|
|
872
|
-
});
|
|
873
|
-
this.emit('aborted');
|
|
874
|
-
}
|
|
875
|
-
catch (err) {
|
|
876
|
-
this.emit('error', err);
|
|
877
|
-
throw err;
|
|
878
|
-
}
|
|
879
|
-
}
|
|
880
|
-
/**
|
|
881
|
-
* Poll for the state of the processing for the job.
|
|
882
|
-
*
|
|
883
|
-
* This method will only throw after a timeout. To capture a
|
|
884
|
-
* job failure while polling you must set a listener for the
|
|
885
|
-
* `failed` event before calling it:
|
|
886
|
-
*
|
|
887
|
-
* job.on('failed', (err) => console.error(err))
|
|
888
|
-
* await job.poll()
|
|
889
|
-
*
|
|
890
|
-
* @param interval Polling interval in milliseconds
|
|
891
|
-
* @param timeout Polling timeout in milliseconds
|
|
892
|
-
* @returns {Promise<Record[]>} A promise that resolves to an array of records
|
|
893
|
-
*/
|
|
894
|
-
async poll(interval = this.#pollingOptions.pollInterval, timeout = this.#pollingOptions.pollTimeout) {
|
|
895
|
-
const jobId = getJobIdOrError(this.jobInfo);
|
|
896
|
-
const startTime = Date.now();
|
|
897
|
-
while (startTime + timeout > Date.now()) {
|
|
898
|
-
try {
|
|
899
|
-
const res = await this.check();
|
|
900
|
-
switch (res.state) {
|
|
901
|
-
case 'Open':
|
|
902
|
-
throw new Error('Job has not been started');
|
|
903
|
-
case 'Aborted':
|
|
904
|
-
throw new Error('Job has been aborted');
|
|
905
|
-
case 'UploadComplete':
|
|
906
|
-
case 'InProgress':
|
|
907
|
-
await delay(interval);
|
|
908
|
-
break;
|
|
909
|
-
case 'Failed':
|
|
910
|
-
// unlike ingest jobs, the API doesn't return an error msg:
|
|
911
|
-
// https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/query_get_one_job.htm
|
|
912
|
-
this.emit('failed', new Error('Query job failed to complete.'));
|
|
913
|
-
return;
|
|
914
|
-
case 'JobComplete':
|
|
915
|
-
this.emit('jobcomplete');
|
|
916
|
-
return;
|
|
917
|
-
}
|
|
918
|
-
}
|
|
919
|
-
catch (err) {
|
|
920
|
-
this.emit('error', err);
|
|
921
|
-
throw err;
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
const timeoutError = new JobPollingTimeoutError(`Polling timed out after ${timeout}ms. Job Id = ${jobId}`, jobId);
|
|
925
|
-
this.emit('error', timeoutError);
|
|
926
|
-
throw timeoutError;
|
|
927
|
-
}
|
|
928
|
-
/**
|
|
929
|
-
* Check the latest batch status in server
|
|
930
|
-
*/
|
|
931
|
-
async check() {
|
|
932
|
-
try {
|
|
933
|
-
const jobInfo = await this.createQueryRequest({
|
|
934
|
-
method: 'GET',
|
|
935
|
-
path: `/${getJobIdOrError(this.jobInfo)}`,
|
|
936
|
-
responseType: 'application/json',
|
|
937
|
-
});
|
|
938
|
-
this.jobInfo = jobInfo;
|
|
939
|
-
return jobInfo;
|
|
940
|
-
}
|
|
941
|
-
catch (err) {
|
|
942
|
-
this.emit('error', err);
|
|
943
|
-
throw err;
|
|
944
|
-
}
|
|
945
|
-
}
|
|
946
|
-
request(request, options = {}) {
|
|
947
|
-
// if request is simple string, regard it as url in GET method
|
|
948
|
-
const request_ = typeof request === 'string' ? { method: 'GET', url: request } : request;
|
|
949
|
-
const httpApi = new http_api_1.default(this.#connection, options);
|
|
950
|
-
httpApi.on('response', (response) => {
|
|
951
|
-
this.locator = response.headers['sforce-locator'];
|
|
952
|
-
});
|
|
953
|
-
return httpApi.request(request_);
|
|
954
|
-
}
|
|
955
|
-
getResultsUrl() {
|
|
956
|
-
const url = `${this.#connection.instanceUrl}/services/data/v${this.#connection.version}/jobs/query/${getJobIdOrError(this.jobInfo)}/results`;
|
|
957
|
-
return this.locator ? `${url}?locator=${this.locator}` : url;
|
|
958
|
-
}
|
|
959
|
-
/**
|
|
960
|
-
* Get the results for a query job.
|
|
961
|
-
*
|
|
962
|
-
* @returns {Promise<Record[]>} A promise that resolves to an array of records
|
|
963
|
-
*/
|
|
964
|
-
async getResults() {
|
|
965
|
-
if (this.finished && this.#queryResults) {
|
|
966
|
-
return this.#queryResults;
|
|
967
|
-
}
|
|
968
|
-
this.#queryResults = [];
|
|
969
|
-
while (this.locator !== 'null') {
|
|
970
|
-
const nextResults = await this.request({
|
|
971
|
-
method: 'GET',
|
|
972
|
-
url: this.getResultsUrl(),
|
|
973
|
-
headers: {
|
|
974
|
-
Accept: 'text/csv',
|
|
975
|
-
},
|
|
976
|
-
});
|
|
977
|
-
this.#queryResults = this.#queryResults.concat(nextResults);
|
|
978
|
-
}
|
|
979
|
-
this.finished = true;
|
|
980
|
-
return this.#queryResults;
|
|
981
|
-
}
|
|
982
|
-
/**
|
|
983
|
-
* Deletes a query job.
|
|
984
|
-
*/
|
|
985
|
-
async delete() {
|
|
986
|
-
return this.createQueryRequest({
|
|
987
|
-
method: 'DELETE',
|
|
988
|
-
path: `/${getJobIdOrError(this.jobInfo)}`,
|
|
989
|
-
});
|
|
990
|
-
}
|
|
991
|
-
createQueryRequest(request) {
|
|
992
|
-
const { path, responseType } = request;
|
|
993
|
-
const baseUrl = [
|
|
994
|
-
this.#connection.instanceUrl,
|
|
995
|
-
'services/data',
|
|
996
|
-
`v${this.#connection.version}`,
|
|
997
|
-
'jobs/query',
|
|
998
|
-
].join('/');
|
|
999
|
-
return new BulkApiV2(this.#connection, { responseType }).request({
|
|
1000
|
-
...request,
|
|
1001
|
-
url: baseUrl + path,
|
|
1002
|
-
});
|
|
1003
|
-
}
|
|
1004
|
-
}
|
|
1005
|
-
exports.QueryJobV2 = QueryJobV2;
|
|
1006
|
-
/**
|
|
1007
|
-
* Class for Bulk API V2 Ingest Job
|
|
1008
|
-
*/
|
|
1009
|
-
class IngestJobV2 extends events_1.EventEmitter {
|
|
1010
|
-
#connection;
|
|
1011
|
-
#pollingOptions;
|
|
1012
|
-
#jobData;
|
|
1013
|
-
#bulkJobSuccessfulResults;
|
|
1014
|
-
#bulkJobFailedResults;
|
|
1015
|
-
#bulkJobUnprocessedRecords;
|
|
1016
|
-
#error;
|
|
1017
|
-
jobInfo;
|
|
1018
|
-
/**
|
|
1019
|
-
*
|
|
1020
|
-
*/
|
|
1021
|
-
constructor(options) {
|
|
1022
|
-
super();
|
|
1023
|
-
this.#connection = options.connection;
|
|
1024
|
-
this.#pollingOptions = options.pollingOptions;
|
|
1025
|
-
this.jobInfo = options.jobInfo;
|
|
1026
|
-
this.#jobData = new JobDataV2({
|
|
1027
|
-
createRequest: (request) => this.createIngestRequest(request),
|
|
1028
|
-
job: this,
|
|
1029
|
-
});
|
|
1030
|
-
// default error handler to keep the latest error
|
|
1031
|
-
this.on('error', (error) => (this.#error = error));
|
|
1032
|
-
}
|
|
1033
|
-
get id() {
|
|
1034
|
-
return this.jobInfo.id;
|
|
1035
|
-
}
|
|
1036
|
-
/**
|
|
1037
|
-
* Create a job representing a bulk operation in the org
|
|
1038
|
-
*/
|
|
1039
|
-
async open() {
|
|
1040
|
-
try {
|
|
1041
|
-
this.jobInfo = await this.createIngestRequest({
|
|
1042
|
-
method: 'POST',
|
|
1043
|
-
path: '',
|
|
1044
|
-
body: JSON.stringify({
|
|
1045
|
-
assignmentRuleId: this.jobInfo?.assignmentRuleId,
|
|
1046
|
-
externalIdFieldName: this.jobInfo?.externalIdFieldName,
|
|
1047
|
-
object: this.jobInfo?.object,
|
|
1048
|
-
operation: this.jobInfo?.operation,
|
|
1049
|
-
lineEnding: this.jobInfo?.lineEnding,
|
|
1050
|
-
}),
|
|
1051
|
-
headers: {
|
|
1052
|
-
'Content-Type': 'application/json; charset=utf-8',
|
|
1053
|
-
},
|
|
1054
|
-
responseType: 'application/json',
|
|
1055
|
-
});
|
|
1056
|
-
this.emit('open');
|
|
1057
|
-
}
|
|
1058
|
-
catch (err) {
|
|
1059
|
-
this.emit('error', err);
|
|
1060
|
-
throw err;
|
|
1061
|
-
}
|
|
1062
|
-
}
|
|
1063
|
-
/** Upload data for a job in CSV format
|
|
1064
|
-
*
|
|
1065
|
-
* @param input CSV as a string, or array of records or readable stream
|
|
1066
|
-
*/
|
|
1067
|
-
async uploadData(input) {
|
|
1068
|
-
await this.#jobData.execute(input);
|
|
1069
|
-
}
|
|
1070
|
-
async getAllResults() {
|
|
1071
|
-
const [successfulResults, failedResults, unprocessedRecords,] = await Promise.all([
|
|
1072
|
-
this.getSuccessfulResults(),
|
|
1073
|
-
this.getFailedResults(),
|
|
1074
|
-
this.getUnprocessedRecords(),
|
|
1075
|
-
]);
|
|
1076
|
-
return { successfulResults, failedResults, unprocessedRecords };
|
|
1077
|
-
}
|
|
1078
|
-
/**
|
|
1079
|
-
* Close opened job
|
|
1080
|
-
*/
|
|
1081
|
-
async close() {
|
|
1082
|
-
try {
|
|
1083
|
-
const state = 'UploadComplete';
|
|
1084
|
-
this.jobInfo = await this.createIngestRequest({
|
|
1085
|
-
method: 'PATCH',
|
|
1086
|
-
path: `/${this.jobInfo.id}`,
|
|
1087
|
-
body: JSON.stringify({ state }),
|
|
1088
|
-
headers: { 'Content-Type': 'application/json; charset=utf-8' },
|
|
1089
|
-
responseType: 'application/json',
|
|
1090
|
-
});
|
|
1091
|
-
this.emit('uploadcomplete');
|
|
1092
|
-
}
|
|
1093
|
-
catch (err) {
|
|
1094
|
-
this.emit('error', err);
|
|
1095
|
-
throw err;
|
|
1096
|
-
}
|
|
1097
|
-
}
|
|
1098
|
-
/**
|
|
1099
|
-
* Set the status to abort
|
|
1100
|
-
*/
|
|
1101
|
-
async abort() {
|
|
1102
|
-
try {
|
|
1103
|
-
const state = 'Aborted';
|
|
1104
|
-
this.jobInfo = await this.createIngestRequest({
|
|
1105
|
-
method: 'PATCH',
|
|
1106
|
-
path: `/${this.jobInfo.id}`,
|
|
1107
|
-
body: JSON.stringify({ state }),
|
|
1108
|
-
headers: { 'Content-Type': 'application/json; charset=utf-8' },
|
|
1109
|
-
responseType: 'application/json',
|
|
1110
|
-
});
|
|
1111
|
-
this.emit('aborted');
|
|
1112
|
-
}
|
|
1113
|
-
catch (err) {
|
|
1114
|
-
this.emit('error', err);
|
|
1115
|
-
throw err;
|
|
1116
|
-
}
|
|
1117
|
-
}
|
|
1118
|
-
/**
|
|
1119
|
-
* Poll for the state of the processing for the job.
|
|
1120
|
-
*
|
|
1121
|
-
* This method will only throw after a timeout. To capture a
|
|
1122
|
-
* job failure while polling you must set a listener for the
|
|
1123
|
-
* `failed` event before calling it:
|
|
1124
|
-
*
|
|
1125
|
-
* job.on('failed', (err) => console.error(err))
|
|
1126
|
-
* await job.poll()
|
|
1127
|
-
*
|
|
1128
|
-
* @param interval Polling interval in milliseconds
|
|
1129
|
-
* @param timeout Polling timeout in milliseconds
|
|
1130
|
-
* @returns {Promise<void>} A promise that resolves when the job finishes successfully
|
|
1131
|
-
*/
|
|
1132
|
-
async poll(interval = this.#pollingOptions.pollInterval, timeout = this.#pollingOptions.pollTimeout) {
|
|
1133
|
-
const jobId = getJobIdOrError(this.jobInfo);
|
|
1134
|
-
const startTime = Date.now();
|
|
1135
|
-
while (startTime + timeout > Date.now()) {
|
|
1136
|
-
try {
|
|
1137
|
-
const res = await this.check();
|
|
1138
|
-
switch (res.state) {
|
|
1139
|
-
case 'Open':
|
|
1140
|
-
throw new Error('Job has not been started');
|
|
1141
|
-
case 'Aborted':
|
|
1142
|
-
throw new Error('Job has been aborted');
|
|
1143
|
-
case 'UploadComplete':
|
|
1144
|
-
case 'InProgress':
|
|
1145
|
-
await delay(interval);
|
|
1146
|
-
break;
|
|
1147
|
-
case 'Failed':
|
|
1148
|
-
this.emit('failed', new Error('Ingest job failed to complete.'));
|
|
1149
|
-
return;
|
|
1150
|
-
case 'JobComplete':
|
|
1151
|
-
this.emit('jobcomplete');
|
|
1152
|
-
return;
|
|
1153
|
-
}
|
|
1154
|
-
}
|
|
1155
|
-
catch (err) {
|
|
1156
|
-
this.emit('error', err);
|
|
1157
|
-
throw err;
|
|
1158
|
-
}
|
|
1159
|
-
}
|
|
1160
|
-
const timeoutError = new JobPollingTimeoutError(`Polling timed out after ${timeout}ms. Job Id = ${jobId}`, jobId);
|
|
1161
|
-
this.emit('error', timeoutError);
|
|
1162
|
-
throw timeoutError;
|
|
1163
|
-
}
|
|
1164
|
-
/**
|
|
1165
|
-
* Check the latest batch status in server
|
|
1166
|
-
*/
|
|
1167
|
-
async check() {
|
|
1168
|
-
try {
|
|
1169
|
-
const jobInfo = await this.createIngestRequest({
|
|
1170
|
-
method: 'GET',
|
|
1171
|
-
path: `/${getJobIdOrError(this.jobInfo)}`,
|
|
1172
|
-
responseType: 'application/json',
|
|
1173
|
-
});
|
|
1174
|
-
this.jobInfo = jobInfo;
|
|
1175
|
-
return jobInfo;
|
|
1176
|
-
}
|
|
1177
|
-
catch (err) {
|
|
1178
|
-
this.emit('error', err);
|
|
1179
|
-
throw err;
|
|
1180
|
-
}
|
|
1181
|
-
}
|
|
1182
|
-
async getSuccessfulResults() {
|
|
1183
|
-
if (this.#bulkJobSuccessfulResults) {
|
|
1184
|
-
return this.#bulkJobSuccessfulResults;
|
|
1185
|
-
}
|
|
1186
|
-
const results = await this.createIngestRequest({
|
|
1187
|
-
method: 'GET',
|
|
1188
|
-
path: `/${getJobIdOrError(this.jobInfo)}/successfulResults`,
|
|
1189
|
-
responseType: 'text/csv',
|
|
1190
|
-
});
|
|
1191
|
-
this.#bulkJobSuccessfulResults = results ?? [];
|
|
1192
|
-
return this.#bulkJobSuccessfulResults;
|
|
1193
|
-
}
|
|
1194
|
-
async getFailedResults() {
|
|
1195
|
-
if (this.#bulkJobFailedResults) {
|
|
1196
|
-
return this.#bulkJobFailedResults;
|
|
1197
|
-
}
|
|
1198
|
-
const results = await this.createIngestRequest({
|
|
1199
|
-
method: 'GET',
|
|
1200
|
-
path: `/${getJobIdOrError(this.jobInfo)}/failedResults`,
|
|
1201
|
-
responseType: 'text/csv',
|
|
1202
|
-
});
|
|
1203
|
-
this.#bulkJobFailedResults = results ?? [];
|
|
1204
|
-
return this.#bulkJobFailedResults;
|
|
1205
|
-
}
|
|
1206
|
-
async getUnprocessedRecords() {
|
|
1207
|
-
if (this.#bulkJobUnprocessedRecords) {
|
|
1208
|
-
return this.#bulkJobUnprocessedRecords;
|
|
1209
|
-
}
|
|
1210
|
-
const results = await this.createIngestRequest({
|
|
1211
|
-
method: 'GET',
|
|
1212
|
-
path: `/${getJobIdOrError(this.jobInfo)}/unprocessedrecords`,
|
|
1213
|
-
responseType: 'text/csv',
|
|
1214
|
-
});
|
|
1215
|
-
this.#bulkJobUnprocessedRecords = results ?? [];
|
|
1216
|
-
return this.#bulkJobUnprocessedRecords;
|
|
1217
|
-
}
|
|
1218
|
-
/**
|
|
1219
|
-
* Deletes an ingest job.
|
|
1220
|
-
*/
|
|
1221
|
-
async delete() {
|
|
1222
|
-
return this.createIngestRequest({
|
|
1223
|
-
method: 'DELETE',
|
|
1224
|
-
path: `/${getJobIdOrError(this.jobInfo)}`,
|
|
1225
|
-
});
|
|
1226
|
-
}
|
|
1227
|
-
createIngestRequest(request) {
|
|
1228
|
-
const { path, responseType } = request;
|
|
1229
|
-
const baseUrl = [
|
|
1230
|
-
this.#connection.instanceUrl,
|
|
1231
|
-
'services/data',
|
|
1232
|
-
`v${this.#connection.version}`,
|
|
1233
|
-
'jobs/ingest',
|
|
1234
|
-
].join('/');
|
|
1235
|
-
return new BulkApiV2(this.#connection, { responseType }).request({
|
|
1236
|
-
...request,
|
|
1237
|
-
url: baseUrl + path,
|
|
1238
|
-
});
|
|
1239
|
-
}
|
|
1240
|
-
}
|
|
1241
|
-
exports.IngestJobV2 = IngestJobV2;
|
|
1242
|
-
class JobDataV2 extends stream_1.Writable {
|
|
1243
|
-
#job;
|
|
1244
|
-
#uploadStream;
|
|
1245
|
-
#downloadStream;
|
|
1246
|
-
#dataStream;
|
|
1247
|
-
#result;
|
|
1248
|
-
/**
|
|
1249
|
-
*
|
|
1250
|
-
*/
|
|
1251
|
-
constructor(options) {
|
|
1252
|
-
super({ objectMode: true });
|
|
1253
|
-
const createRequest = options.createRequest;
|
|
1254
|
-
this.#job = options.job;
|
|
1255
|
-
this.#uploadStream = new record_stream_1.Serializable();
|
|
1256
|
-
this.#downloadStream = new record_stream_1.Parsable();
|
|
1257
|
-
const converterOptions = { nullValue: '#N/A' };
|
|
1258
|
-
const uploadDataStream = this.#uploadStream.stream('csv', converterOptions);
|
|
1259
|
-
const downloadDataStream = this.#downloadStream.stream('csv', converterOptions);
|
|
1260
|
-
this.#dataStream = (0, stream_2.concatStreamsAsDuplex)(uploadDataStream, downloadDataStream);
|
|
1261
|
-
this.on('finish', () => this.#uploadStream.end());
|
|
1262
|
-
uploadDataStream.once('readable', () => {
|
|
1263
|
-
try {
|
|
1264
|
-
// pipe upload data to batch API request stream
|
|
1265
|
-
const req = createRequest({
|
|
1266
|
-
method: 'PUT',
|
|
1267
|
-
path: `/${this.#job.jobInfo?.id}/batches`,
|
|
1268
|
-
headers: {
|
|
1269
|
-
'Content-Type': 'text/csv',
|
|
1270
|
-
},
|
|
1271
|
-
responseType: 'application/json',
|
|
1272
|
-
});
|
|
1273
|
-
(async () => {
|
|
1274
|
-
try {
|
|
1275
|
-
const res = await req;
|
|
1276
|
-
this.emit('response', res);
|
|
1277
|
-
}
|
|
1278
|
-
catch (err) {
|
|
1279
|
-
this.emit('error', err);
|
|
1280
|
-
}
|
|
1281
|
-
})();
|
|
1282
|
-
uploadDataStream.pipe(req.stream());
|
|
1283
|
-
}
|
|
1284
|
-
catch (err) {
|
|
1285
|
-
this.emit('error', err);
|
|
1286
|
-
}
|
|
1287
|
-
});
|
|
1288
|
-
}
|
|
1289
|
-
_write(record_, enc, cb) {
|
|
1290
|
-
const { Id, type, attributes, ...rrec } = record_;
|
|
1291
|
-
let record;
|
|
1292
|
-
switch (this.#job.jobInfo.operation) {
|
|
1293
|
-
case 'insert':
|
|
1294
|
-
record = rrec;
|
|
1295
|
-
break;
|
|
1296
|
-
case 'delete':
|
|
1297
|
-
case 'hardDelete':
|
|
1298
|
-
record = { Id };
|
|
1299
|
-
break;
|
|
1300
|
-
default:
|
|
1301
|
-
record = { Id, ...rrec };
|
|
1302
|
-
}
|
|
1303
|
-
this.#uploadStream.write(record, enc, cb);
|
|
1304
|
-
}
|
|
1305
|
-
/**
|
|
1306
|
-
* Returns duplex stream which accepts CSV data input and batch result output
|
|
1307
|
-
*/
|
|
1308
|
-
stream() {
|
|
1309
|
-
return this.#dataStream;
|
|
1310
|
-
}
|
|
1311
|
-
/**
|
|
1312
|
-
* Execute batch operation
|
|
1313
|
-
*/
|
|
1314
|
-
execute(input) {
|
|
1315
|
-
if (this.#result) {
|
|
1316
|
-
throw new Error('Data can only be uploaded to a job once.');
|
|
1317
|
-
}
|
|
1318
|
-
this.#result = new Promise((resolve, reject) => {
|
|
1319
|
-
this.once('response', () => resolve());
|
|
1320
|
-
this.once('error', reject);
|
|
1321
|
-
});
|
|
1322
|
-
if ((0, function_1.isObject)(input) && 'pipe' in input && (0, function_1.isFunction)(input.pipe)) {
|
|
1323
|
-
// if input has stream.Readable interface
|
|
1324
|
-
input.pipe(this.#dataStream);
|
|
1325
|
-
}
|
|
1326
|
-
else {
|
|
1327
|
-
if (Array.isArray(input)) {
|
|
1328
|
-
for (const record of input) {
|
|
1329
|
-
for (const key of Object.keys(record)) {
|
|
1330
|
-
if (typeof record[key] === 'boolean') {
|
|
1331
|
-
record[key] = String(record[key]);
|
|
1332
|
-
}
|
|
1333
|
-
}
|
|
1334
|
-
this.write(record);
|
|
1335
|
-
}
|
|
1336
|
-
this.end();
|
|
1337
|
-
}
|
|
1338
|
-
else if (typeof input === 'string') {
|
|
1339
|
-
this.#dataStream.write(input, 'utf8');
|
|
1340
|
-
this.#dataStream.end();
|
|
1341
|
-
}
|
|
1342
|
-
}
|
|
1343
|
-
return this;
|
|
1344
|
-
}
|
|
1345
|
-
/**
|
|
1346
|
-
* Promise/A+ interface
|
|
1347
|
-
* Delegate to promise, return promise instance for batch result
|
|
1348
|
-
*/
|
|
1349
|
-
then(onResolved, onReject) {
|
|
1350
|
-
if (this.#result === undefined) {
|
|
1351
|
-
this.execute();
|
|
1352
|
-
}
|
|
1353
|
-
return this.#result.then(onResolved, onReject);
|
|
1354
|
-
}
|
|
1355
|
-
}
|
|
1356
|
-
function getJobIdOrError(jobInfo) {
|
|
1357
|
-
const jobId = jobInfo?.id;
|
|
1358
|
-
if (jobId === undefined) {
|
|
1359
|
-
throw new Error('No job id, maybe you need to call `job.open()` first.');
|
|
1360
|
-
}
|
|
1361
|
-
return jobId;
|
|
1362
|
-
}
|
|
1363
|
-
function delay(ms) {
|
|
1364
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1365
|
-
}
|
|
1366
673
|
/*--------------------------------------------*/
|
|
1367
674
|
/*
|
|
1368
675
|
* Register hook in connection instantiation for dynamically adding this API module features
|
|
1369
676
|
*/
|
|
1370
677
|
(0, jsforce_1.registerModule)('bulk', (conn) => new Bulk(conn));
|
|
1371
|
-
(0, jsforce_1.registerModule)('bulk2', (conn) => new BulkV2(conn));
|
|
1372
678
|
exports.default = Bulk;
|