@jsforce/jsforce-node 3.0.0-next.1 → 3.0.0-next.3

package/lib/api/bulk2.js

@@ -0,0 +1,800 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IngestJobV2 = exports.QueryJobV2 = exports.BulkV2 = void 0;
+const events_1 = require("events");
+const stream_1 = require("stream");
+const record_stream_1 = require("../record-stream");
+const http_api_1 = __importDefault(require("../http-api"));
+const jsforce_1 = require("../jsforce");
+const logger_1 = require("../util/logger");
+const stream_2 = require("../util/stream");
+const is_1 = __importDefault(require("@sindresorhus/is"));
+class JobPollingTimeoutError extends Error {
+    jobId;
+    /**
+     *
+     */
+    constructor(message, jobId) {
+        super(message);
+        this.name = 'JobPollingTimeout';
+        this.jobId = jobId;
+    }
+}
+class BulkApiV2 extends http_api_1.default {
+    hasErrorInResponseBody(body) {
+        return (Array.isArray(body) &&
+            typeof body[0] === 'object' &&
+            'errorCode' in body[0]);
+    }
+    isSessionExpired(response) {
+        return (response.statusCode === 401 && /INVALID_SESSION_ID/.test(response.body));
+    }
+    parseError(body) {
+        return {
+            errorCode: body[0].errorCode,
+            message: body[0].message,
+        };
+    }
+}
+class BulkV2 {
+    connection;
+    logger;
+    /**
+     * Polling interval in milliseconds
+     *
+     * Default: 1000 (1 second)
+     */
+    pollInterval = 1000;
+    /**
+     * Polling timeout in milliseconds
+     *
+     * Default: 30000 (30 seconds)
+     */
+    pollTimeout = 30000;
+    constructor(connection) {
+        this.connection = connection;
+        this.logger = this.connection._logLevel
+            ? (0, logger_1.getLogger)('bulk2').createInstance(this.connection._logLevel)
+            : (0, logger_1.getLogger)('bulk2');
+    }
+    /**
+     * Create an instance of an ingest job object.
+     *
+     * @params {NewIngestJobOptions} options object
+     * @returns {IngestJobV2} An ingest job instance
+     * @example
+     * // Upsert records to the Account object.
+     *
+     * const job = connection.bulk2.createJob({
+     *   operation: 'insert'
+     *   object: 'Account',
+     * });
+     *
+     * // create the job in the org
+     * await job.open()
+     *
+     * // upload data
+     * await job.uploadData(csvFile)
+     *
+     * // finished uploading data, mark it as ready for processing
+     * await job.close()
+     */
+    createJob(options) {
+        return new IngestJobV2(this.connection, {
+            bodyParams: options,
+            pollingOptions: this,
+        });
+    }
+    job(type = 'ingest', options) {
+        if (type === 'ingest') {
+            return new IngestJobV2(this.connection, {
+                id: options.id,
+                pollingOptions: this,
+            });
+        }
+        else {
+            return new QueryJobV2(this.connection, {
+                id: options.id,
+                pollingOptions: this,
+            });
+        }
+    }
+    /**
+     * Create, upload, and start bulkload job
+     */
+    async loadAndWaitForResults(options) {
+        if (!options.pollTimeout)
+            options.pollTimeout = this.pollTimeout;
+        if (!options.pollInterval)
+            options.pollInterval = this.pollInterval;
+        const job = this.createJob({
+            object: options.object,
+            operation: options.operation,
+        });
+        try {
+            await job.open();
+            await job.uploadData(options.input);
+            await job.close();
+            await job.poll(options.pollInterval, options.pollTimeout);
+            return await job.getAllResults();
+        }
+        catch (error) {
+            const err = error;
+            this.logger.error(`bulk load failed due to: ${err}`);
+            if (err.name !== 'JobPollingTimeoutError') {
+                // fires off one last attempt to clean up and ignores the result | error
+                job.delete().catch((ignored) => ignored);
+            }
+            throw err;
+        }
+    }
+    /**
+     * Execute bulk query and get a record stream.
+     *
+     * Default timeout: 10000ms
+     *
+     * @param soql SOQL query
+     * @param BulkV2PollingOptions options object
+     *
+     * @returns {RecordStream} - Record stream, convertible to a CSV data stream
+     */
+    async query(soql, options) {
+        const queryJob = new QueryJobV2(this.connection, {
+            bodyParams: {
+                query: soql,
+                operation: options?.scanAll ? 'queryAll' : 'query',
+                columnDelimiter: options?.columnDelimiter,
+                lineEnding: options?.lineEnding,
+            },
+            pollingOptions: this,
+        });
+        const recordStream = new record_stream_1.Parsable();
+        const dataStream = recordStream.stream('csv');
+        try {
+            await queryJob.open();
+            await queryJob.poll(options?.pollInterval, options?.pollTimeout);
+            const queryRecordsStream = await queryJob
+                .result()
+                .then((s) => s.stream());
+            queryRecordsStream.pipe(dataStream);
+        }
+        catch (error) {
+            const err = error;
+            this.logger.error(`bulk query failed due to: ${err}`);
+            if (err.name !== 'JobPollingTimeoutError') {
+                // fires off one last attempt to clean up and ignores the result | error
+                queryJob.delete().catch((ignored) => ignored);
+            }
+            throw err;
+        }
+        return recordStream;
+    }
+}
+exports.BulkV2 = BulkV2;
+class QueryJobV2 extends events_1.EventEmitter {
+    connection;
+    logger;
+    _id;
+    bodyParams;
+    pollingOptions;
+    error;
+    jobInfo;
+    locator;
+    constructor(conn, options) {
+        super();
+        this.connection = conn;
+        this.logger = this.connection._logLevel
+            ? (0, logger_1.getLogger)('bulk2:QueryJobV2').createInstance(this.connection._logLevel)
+            : (0, logger_1.getLogger)('bulk2:QueryJobV2');
+        if ('id' in options) {
+            this._id = options.id;
+        }
+        else {
+            this.bodyParams = options.bodyParams;
+        }
+        this.pollingOptions = options.pollingOptions;
+        // default error handler to keep the latest error
+        this.on('error', (error) => (this.error = error));
+    }
+    /**
+     * Get the query job ID.
+     *
+     * @returns {string} query job Id.
+     */
+    get id() {
+        return this.jobInfo ? this.jobInfo.id : this._id;
+    }
+    /**
+     * Get the query job info.
+     *
+     * @returns {Promise<QueryJobInfoV2>} query job information.
+     */
+    getInfo() {
+        if (this.jobInfo) {
+            return this.jobInfo;
+        }
+        throw new Error('No internal job info. Make sure to call `await job.check`.');
+    }
+    /**
+     * Creates a query job
+     *
+     * @returns {Promise<QueryJobInfoV2>} job information.
+     */
+    async open() {
+        if (!this.bodyParams) {
+            throw new Error('Missing required body params to open a new query job.');
+        }
+        try {
+            this.jobInfo = await this.createQueryRequest({
+                method: 'POST',
+                body: JSON.stringify(this.bodyParams),
+                headers: {
+                    'Content-Type': 'application/json; charset=utf-8',
+                },
+                responseType: 'application/json',
+            });
+            this.logger.debug(`Successfully created job ${this.id}`);
+            this.emit('open', this.jobInfo);
+        }
+        catch (err) {
+            this.emit('error', err);
+            throw err;
+        }
+        return this.jobInfo;
+    }
+    /**
+     * Abort the job
+     *
+     * The 'aborted' event is emitted when the job successfully aborts.
+     * @returns {Promise<QueryJobInfoV2>} job information.
+     */
+    async abort() {
+        try {
+            const state = 'Aborted';
+            this.jobInfo = await this.createQueryRequest({
+                method: 'PATCH',
+                path: `/${this.id}`,
+                body: JSON.stringify({ state }),
+                headers: { 'Content-Type': 'application/json; charset=utf-8' },
+                responseType: 'application/json',
+            });
+            this.logger.debug(`Successfully aborted job ${this.id}`);
+            return this.jobInfo;
+        }
+        catch (err) {
+            this.emit('error', err);
+            throw err;
+        }
+    }
+    /**
+     * Poll for the state of the processing for the job.
+     *
+     * @param interval Polling interval in milliseconds
+     * @param timeout Polling timeout in milliseconds
+     * @returns {Promise<Record[]>} A promise that resolves when the job finished being processed.
+     */
+    async poll(interval = this.pollingOptions.pollInterval, timeout = this.pollingOptions.pollTimeout) {
+        const jobId = this.id;
+        const startTime = Date.now();
+        const endTime = startTime + timeout;
+        this.logger.debug(`Start polling for job status`);
+        this.logger.debug(`Polling options: timeout:${timeout}ms | interval: ${interval}ms.`);
+        if (timeout === 0) {
+            throw new JobPollingTimeoutError(`Skipping polling because of timeout = 0ms. Job Id = ${jobId}`, jobId);
+        }
+        while (endTime > Date.now()) {
+            try {
+                const res = await this.check();
+                switch (res.state) {
+                    case 'Aborted':
+                        throw new Error('Job has been aborted');
+                    case 'UploadComplete':
+                    case 'InProgress':
+                        this.emit('inProgress', res);
+                        await delay(interval);
+                        break;
+                    case 'Failed':
+                        // unlike ingest jobs, the API doesn't return an error msg:
+                        // https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/query_get_one_job.htm
+                        this.logger.debug(res);
+                        throw new Error('Query job failed to complete');
+                    case 'JobComplete':
+                        this.logger.debug(`Job ${this.id} was successfully processed.`);
+                        this.emit('jobComplete');
+                        return;
+                }
+            }
+            catch (err) {
+                this.emit('error', err);
+                throw err;
+            }
+        }
+        const timeoutError = new JobPollingTimeoutError(`Polling timed out after ${timeout}ms. Job Id = ${jobId}`, jobId);
+        this.emit('error', timeoutError);
+        throw timeoutError;
+    }
+    /**
+     * Check the latest job status
+     *
+     * @returns {Promise<QueryJobInfoV2>} job information.
+     */
+    async check() {
+        try {
+            const jobInfo = await this.createQueryRequest({
+                method: 'GET',
+                path: `/${this.id}`,
+                responseType: 'application/json',
+            });
+            this.jobInfo = jobInfo;
+            return jobInfo;
+        }
+        catch (err) {
+            this.emit('error', err);
+            throw err;
+        }
+    }
+    /**
+     * Get the results for a query job as a record stream
+     *
+     * This method assumes the job finished being processed
+     * @returns {RecordStream} - Record stream, convertible to a CSV data stream
+     */
+    async result() {
+        const resultStream = new record_stream_1.Parsable();
+        const resultDataStream = resultStream.stream('csv');
+        const resultsPath = `/${this.id}/results`;
+        while (this.locator !== 'null') {
+            const resPromise = this.createQueryRequest({
+                method: 'GET',
+                path: this.locator
+                    ? `/${resultsPath}?locator=${this.locator}`
+                    : resultsPath,
+                headers: {
+                    Accept: 'text/csv',
+                },
+            });
+            resPromise.stream().pipe(resultDataStream);
+            await resPromise;
+        }
+        return resultStream;
+    }
+    /**
+     * Deletes a query job.
+     */
+    async delete() {
+        return this.createQueryRequest({
+            method: 'DELETE',
+            path: `/${this.id}`,
+        });
+    }
+    createQueryRequest(request) {
+        const { path, responseType } = request;
+        const basePath = `services/data/v${this.connection.version}/jobs/query`;
+        const url = new URL(path ? basePath + path : basePath, this.connection.instanceUrl).toString();
+        const httpApi = new BulkApiV2(this.connection, { responseType });
+        httpApi.on('response', (response) => {
+            this.locator = response.headers['sforce-locator'];
+            this.logger.debug(`sforce-locator: ${this.locator}`);
+        });
+        return httpApi.request({
+            ...request,
+            url,
+        });
+    }
+}
+exports.QueryJobV2 = QueryJobV2;
+/**
+ * Class for Bulk API V2 Ingest Job
+ */
+class IngestJobV2 extends events_1.EventEmitter {
+    connection;
+    logger;
+    _id;
+    bodyParams;
+    jobData;
+    pollingOptions;
+    bulkJobSuccessfulResults;
+    bulkJobFailedResults;
+    bulkJobUnprocessedRecords;
+    error;
+    jobInfo;
+    constructor(conn, options) {
+        super();
+        this.connection = conn;
+        this.logger = this.connection._logLevel
+            ? (0, logger_1.getLogger)('bulk2:IngestJobV2').createInstance(this.connection._logLevel)
+            : (0, logger_1.getLogger)('bulk2:IngestJobV2');
+        this.pollingOptions = options.pollingOptions;
+        if ('id' in options) {
+            this._id = options.id;
+        }
+        else {
+            this.bodyParams = options.bodyParams;
+        }
+        this.jobData = new JobDataV2({
+            createRequest: (request) => this.createIngestRequest(request),
+            job: this,
+        });
+        // default error handler to keep the latest error
+        this.on('error', (error) => (this.error = error));
+    }
+    /**
+     * Get the query job ID.
+     *
+     * @returns {string} query job Id.
+     */
+    get id() {
+        return this.jobInfo ? this.jobInfo.id : this._id;
+    }
+    /**
+     * Get the query job info.
+     *
+     * @returns {Promise<QueryJobInfoV2>} ingest job information.
+     */
+    getInfo() {
+        if (this.jobInfo) {
+            return this.jobInfo;
+        }
+        throw new Error('No internal job info. Make sure to call `await job.check`.');
+    }
+    /**
+     * Create a job representing a bulk operation in the org
+     *
+     * @returns {Promise<QueryJobInfoV2>} job information.
+     */
+    async open() {
+        if (!this.bodyParams) {
+            throw new Error('Missing required body params to open a new ingest job.');
+        }
+        try {
+            this.jobInfo = await this.createIngestRequest({
+                method: 'POST',
+                body: JSON.stringify(this.bodyParams),
+                headers: {
+                    'Content-Type': 'application/json; charset=utf-8',
+                },
+                responseType: 'application/json',
+            });
+            this.logger.debug(`Successfully created job ${this.id}`);
+            this.emit('open');
+        }
+        catch (err) {
+            this.emit('error', err);
+            throw err;
+        }
+        return this.jobInfo;
+    }
+    /** Upload data for a job in CSV format
+     *
+     *  @param input CSV as a string, or array of records or readable stream
+     */
+    async uploadData(input) {
+        await this.jobData.execute(input).result;
+        this.logger.debug(`Successfully uploaded data to job ${this.id}`);
+    }
+    /**
+     * Close opened job
+     *
+     * This method will notify the org  that the upload of job data is complete and is ready for processing.
+     */
+    async close() {
+        try {
+            const state = 'UploadComplete';
+            this.jobInfo = await this.createIngestRequest({
+                method: 'PATCH',
+                path: `/${this.id}`,
+                body: JSON.stringify({ state }),
+                headers: { 'Content-Type': 'application/json; charset=utf-8' },
+                responseType: 'application/json',
+            });
+            this.logger.debug(`Successfully closed job ${this.id}`);
+            this.emit('close');
+        }
+        catch (err) {
+            this.emit('error', err);
+            throw err;
+        }
+    }
+    /**
+     * Set the status to abort
+     */
+    async abort() {
+        try {
+            const state = 'Aborted';
+            this.jobInfo = await this.createIngestRequest({
+                method: 'PATCH',
+                path: `/${this.id}`,
+                body: JSON.stringify({ state }),
+                headers: { 'Content-Type': 'application/json; charset=utf-8' },
+                responseType: 'application/json',
+            });
+            this.logger.debug(`Successfully aborted job ${this.id}`);
+            this.emit('aborted');
+        }
+        catch (err) {
+            this.emit('error', err);
+            throw err;
+        }
+    }
+    /**
+     * Poll for the state of the processing for the job.
+     *
+     * This method will only throw after a timeout. To capture a
+     * job failure while polling you must set a listener for the
+     * `failed` event before calling it:
+     *
+     * job.on('failed', (err) => console.error(err))
+     * await job.poll()
+     *
+     * @param interval Polling interval in milliseconds
+     * @param timeout Polling timeout in milliseconds
+     * @returns {Promise<void>} A promise that resolves when the job finishes successfully
+     */
+    async poll(interval = this.pollingOptions.pollInterval, timeout = this.pollingOptions.pollTimeout) {
+        const jobId = this.id;
+        const startTime = Date.now();
+        const endTime = startTime + timeout;
+        if (timeout === 0) {
+            throw new JobPollingTimeoutError(`Skipping polling because of timeout = 0ms. Job Id = ${jobId}`, jobId);
+        }
+        this.logger.debug(`Start polling for job status`);
+        this.logger.debug(`Polling options: timeout:${timeout}ms | interval: ${interval}ms.`);
+        while (endTime > Date.now()) {
+            try {
+                const res = await this.check();
+                switch (res.state) {
+                    case 'Open':
+                        throw new Error('Job is still open. Make sure close the job by `close` method on the job instance before polling.');
+                    case 'Aborted':
+                        throw new Error('Job has been aborted');
+                    case 'UploadComplete':
+                    case 'InProgress':
+                        this.emit('inProgress', res);
+                        await delay(interval);
+                        break;
+                    case 'Failed':
+                        this.logger.debug(res);
+                        throw new Error(`Ingest job failed to complete due to: ${res.errorMessage}`);
+                    case 'JobComplete':
+                        this.logger.debug(`Job ${this.id} was successfully processed.`);
+                        this.emit('jobComplete');
+                        return;
+                }
+            }
+            catch (err) {
+                this.emit('error', err);
+                throw err;
+            }
+        }
+        const timeoutError = new JobPollingTimeoutError(`Polling timed out after ${timeout}ms. Job Id = ${jobId}`, jobId);
+        this.emit('error', timeoutError);
+        throw timeoutError;
+    }
+    /**
+     * Check the latest batch status in server
+     */
+    async check() {
+        try {
+            const jobInfo = await this.createIngestRequest({
+                method: 'GET',
+                path: `/${this.id}`,
+                responseType: 'application/json',
+            });
+            this.jobInfo = jobInfo;
+            return jobInfo;
+        }
+        catch (err) {
+            this.emit('error', err);
+            throw err;
+        }
+    }
+    /** Return all record results
+     *
+     * This method will return successful, failed and unprocessed records
+     *
+     * @returns Promise<IngestJobV2Results>
+     */
+    async getAllResults() {
+        const [successfulResults, failedResults, unprocessedRecords,] = await Promise.all([
+            this.getSuccessfulResults(),
+            this.getFailedResults(),
+            this.getUnprocessedRecords(),
+        ]);
+        return { successfulResults, failedResults, unprocessedRecords };
+    }
+    /** Return successful results
+     *
+     * The order of records returned is not guaranteed to match the ordering of the uploaded data.
+     *
+     * @returns Promise<IngestJobV2SuccessfulResults>
+     */
+    async getSuccessfulResults() {
+        if (this.bulkJobSuccessfulResults) {
+            return this.bulkJobSuccessfulResults;
+        }
+        const results = await this.createIngestRequest({
+            method: 'GET',
+            path: `/${this.id}/successfulResults`,
+            responseType: 'text/csv',
+        });
+        this.bulkJobSuccessfulResults = results ?? [];
+        return this.bulkJobSuccessfulResults;
+    }
+    /** Return failed results
+     *
+     * The order of records in the response is not guaranteed to match the ordering of records in the original job data.
+     *
+     * @returns Promise<IngestJobV2SuccessfulResults>
+     */
+    async getFailedResults() {
+        if (this.bulkJobFailedResults) {
+            return this.bulkJobFailedResults;
+        }
+        const results = await this.createIngestRequest({
+            method: 'GET',
+            path: `/${this.id}/failedResults`,
+            responseType: 'text/csv',
+        });
+        this.bulkJobFailedResults = results ?? [];
+        return this.bulkJobFailedResults;
+    }
+    /** Return unprocessed results
+     *
+     * The unprocessed records endpoint returns records as a CSV.
+     * If the request helper is able to parse it, you get the records
+     * as an array of objects.
+     * If unable to parse the it (bad CSV), you get the raw response as a string.
+     *
+     * The order of records in the response is not guaranteed to match the ordering of records in the original job data.
+     *
+     * @returns Promise<IngestJobV2UnprocessedRecords>
+     */
+    async getUnprocessedRecords() {
+        if (this.bulkJobUnprocessedRecords) {
+            return this.bulkJobUnprocessedRecords;
+        }
+        const results = await this.createIngestRequest({
+            method: 'GET',
+            path: `/${this.id}/unprocessedrecords`,
+            responseType: 'text/csv',
+        });
+        this.bulkJobUnprocessedRecords = results ?? [];
+        return this.bulkJobUnprocessedRecords;
+    }
+    /**
+     * Deletes an ingest job.
+     */
+    async delete() {
+        return this.createIngestRequest({
+            method: 'DELETE',
+            path: `/${this.id}`,
+        });
+    }
+    createIngestRequest(request) {
+        const { path, responseType } = request;
+        const basePath = `/services/data/v${this.connection.version}/jobs/ingest`;
+        const url = new URL(path ? basePath + path : basePath, this.connection.instanceUrl).toString();
+        return new BulkApiV2(this.connection, { responseType }).request({
+            ...request,
+            url,
+        });
+    }
+}
+exports.IngestJobV2 = IngestJobV2;
+class JobDataV2 extends stream_1.Writable {
+    job;
+    uploadStream;
+    downloadStream;
+    dataStream;
+    result;
+    /**
+     *
+     */
+    constructor(options) {
+        super({ objectMode: true });
+        const createRequest = options.createRequest;
+        this.job = options.job;
+        this.uploadStream = new record_stream_1.Serializable();
+        this.downloadStream = new record_stream_1.Parsable();
+        const converterOptions = { nullValue: '#N/A' };
+        const uploadDataStream = this.uploadStream.stream('csv', converterOptions);
+        const downloadDataStream = this.downloadStream.stream('csv', converterOptions);
+        this.dataStream = (0, stream_2.concatStreamsAsDuplex)(uploadDataStream, downloadDataStream);
+        this.on('finish', () => this.uploadStream.end());
+        uploadDataStream.once('readable', () => {
+            try {
+                // pipe upload data to batch API request stream
+                const req = createRequest({
+                    method: 'PUT',
+                    path: `/${this.job.id}/batches`,
+                    headers: {
+                        'Content-Type': 'text/csv',
+                    },
+                    responseType: 'application/json',
+                });
+                (async () => {
+                    try {
+                        const res = await req;
+                        this.emit('response', res);
+                    }
+                    catch (err) {
+                        this.emit('error', err);
+                    }
+                })();
+                uploadDataStream.pipe(req.stream());
+            }
+            catch (err) {
+                this.emit('error', err);
+            }
+        });
+    }
+    _write(record_, enc, cb) {
+        const { Id, type, attributes, ...rrec } = record_;
+        let record;
+        switch (this.job.getInfo().operation) {
+            case 'insert':
+                record = rrec;
+                break;
+            case 'delete':
+            case 'hardDelete':
+                record = { Id };
+                break;
+            default:
+                record = { Id, ...rrec };
+        }
+        this.uploadStream.write(record, enc, cb);
+    }
+    /**
+     * Returns duplex stream which accepts CSV data input and batch result output
+     */
+    stream() {
+        return this.dataStream;
+    }
+    /**
+     * Execute batch operation
+     */
+    execute(input) {
+        if (this.result) {
+            throw new Error('Data can only be uploaded to a job once.');
+        }
+        this.result = new Promise((resolve, reject) => {
+            this.once('response', () => resolve());
+            this.once('error', reject);
+        });
+        if (is_1.default.nodeStream(input)) {
+            // if input has stream.Readable interface
+            input.pipe(this.dataStream);
+        }
+        else {
+            const recordData = structuredClone(input);
+            if (Array.isArray(recordData)) {
+                for (const record of recordData) {
+                    for (const key of Object.keys(record)) {
+                        if (typeof record[key] === 'boolean') {
+                            record[key] = String(record[key]);
+                        }
+                    }
+                    this.write(record);
+                }
+                this.end();
+            }
+            else if (typeof recordData === 'string') {
+                this.dataStream.write(recordData, 'utf8');
+                this.dataStream.end();
+            }
+        }
+        return this;
+    }
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/*--------------------------------------------*/
+/*
+ * Register hook in connection instantiation for dynamically adding this API module features
+ */
+(0, jsforce_1.registerModule)('bulk2', (conn) => new BulkV2(conn));
+exports.default = BulkV2;