@tetrascience-npm/ts-connectors-sdk 3.1.0

package/dist/src/tdp-client.js

@@ -0,0 +1,823 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TDPClient = exports.Directive = void 0;
+exports.sanitizeConfig = sanitizeConfig;
+const fs = __importStar(require("node:fs"));
+const s3 = __importStar(require("@aws-sdk/client-s3"));
+const sqs = __importStar(require("@aws-sdk/client-sqs"));
+const lib_storage_1 = require("@aws-sdk/lib-storage");
+const node_http_handler_1 = require("@smithy/node-http-handler");
+const axios_1 = __importStar(require("axios"));
+const dotenv = __importStar(require("dotenv"));
+const events_1 = __importDefault(require("events"));
+const fsPromises = __importStar(require("fs/promises"));
+const hpagent_1 = require("hpagent");
+const http = __importStar(require("http"));
+const https = __importStar(require("https"));
+const proxy_agent_1 = require("proxy-agent");
+const proxy_from_env_1 = require("proxy-from-env");
+const tls = __importStar(require("tls"));
+const ts_lib_shared_schema_1 = require("ts-lib-shared-schema");
+const uuid_1 = require("uuid");
+const zlib_1 = __importDefault(require("zlib"));
+const zod_1 = require("zod");
+const api_1 = require("./api");
+const certificates_1 = require("./certificates");
+const auth_1 = require("./auth");
+const checksums_1 = require("./checksums");
+const config_1 = require("./config");
+const https_proxy_agent_1 = require("./https-proxy-agent");
+const logger_1 = require("./logger");
+const schemas_1 = require("./schemas");
+const types_1 = require("./types");
+const utils_1 = require("./utils");
+const manifest_1 = require("./manifest");
+const hubProxyPath = '/etc/hub/proxy.env';
+const HP_AGENT = 'hpagent';
+const AWS_SQS_TIMEOUT = 20 * 1000;
+// Key in connector KV store for connector SDK settings
+// The value is expected to be a JSON object
+const TS_SDK_SETTINGS_KEY = 'TS_SDK';
+const TS_SDK_DISABLE_USER_AGENT_HTTP_HEADER = 'TS_SDK_DISABLE_USER_AGENT_HTTP_HEADER';
+const DestinationIdSchema = zod_1.z
+    .string()
+    .uuid({ message: 'destinationId must be a valid UUID' })
+    .refine((id) => id[14] === '4', 'destinationId must be a UUIDv4');
+var Directive;
+(function (Directive) {
+    Directive[Directive["Append"] = 0] = "Append";
+    Directive[Directive["Replace"] = 1] = "Replace";
+})(Directive || (exports.Directive = Directive = {}));
+const safeAxiosRequestKeys = [
+    'url',
+    'method',
+    'baseURL',
+    'baseUrl',
+    'params',
+    'timeout',
+    'withCredentials',
+    'responseType',
+    'responseEncoding',
+    'xsrfCookieName',
+    'xsrfHeaderName',
+    'maxContentLength',
+    'maxBodyLength',
+    'maxRedirects',
+    'decompress',
+];
+function sanitizeConfig(config) {
+    if (!config)
+        return config;
+    const rval = {};
+    for (const key of safeAxiosRequestKeys) {
+        if (key in config) {
+            rval[key] = config[key];
+        }
+    }
+    return rval;
+}
+class TDPClient {
+    get connector() {
+        return this._connector;
+    }
+    get manifest() {
+        if (!this.isInitialized) {
+            this.logger.error('Client is not initialized - manifest will be empty');
+        }
+        return this._manifest;
+    }
+    constructor(options) {
+        this.commands = new events_1.default();
+        this.isProxyInitialized = false;
+        this.listeningForCommands = false;
+        this.config = new config_1.TDPClientConfig(options);
+        this.logger = this.createLogger();
+        this._certificates = [];
+    }
+    get isInitialized() {
+        var _a;
+        return this.jwt && ((_a = this.api) === null || _a === void 0 ? void 0 : _a.v1); // && this.connector;
+    }
+    get certificates() {
+        if (!this.isInitialized) {
+            this.logger.error('Client is not initialized - certificate array will be empty');
+        }
+        return this._certificates;
+    }
+    setUserAgent(userAgent) {
+        var _a;
+        this.logger.debug('Setting TDPClient user agent, will be used for all API requests and axios clients created from this point on', {
+            userAgent,
+        });
+        this.userAgent = userAgent;
+        (_a = this.api) === null || _a === void 0 ? void 0 : _a.setUserAgent(userAgent);
+    }
+    assertInitialized() {
+        if (!this.isInitialized) {
+            throw new Error('Client is not initialized');
+        }
+        return {
+            api: this.api,
+            s3Client: this.awsClientProvider.getS3Client(),
+            sqsClient: this.awsClientProvider.getSqsClient(),
+            authTokenProvider: this.authTokenProvider,
+        };
+    }
+    assertProxyInitialized() {
+        if (!this.isProxyInitialized) {
+            throw new Error('Proxy settings are not initialized');
+        }
+    }
+    get apiInstance() {
+        return this.assertInitialized().api.v1;
+    }
+    getAdditionalAxiosHeaders() {
+        const headers = new axios_1.AxiosHeaders();
+        if (this.userAgent) {
+            headers['user-agent'] = this.userAgent;
+        }
+        return headers;
+    }
+    /*
+     * Creates an axios instance with extra provided certificates, TDP jwt + orgSlug headers.
+     * @param tdpEndpoint - The TDP endpoint
+     * @param orgSlug - The organization slug
+     * @param jwt - The TDP jwt
+     * @param certificates - The extra certificates to include
+     * @param timeout_ms - The axios http request timeout in milliseconds
+     */
+    createAxiosInstanceWithCertificatesAndHeaders(tdpEndpoint, orgSlug, jwt, certificates, timeout_ms) {
+        const additionalHeaders = this.getAdditionalAxiosHeaders();
+        return axios_1.default.create(Object.assign(Object.assign({}, this.createProxyAgentsForBaseUrl({
+            baseUrl: tdpEndpoint,
+            rejectUnauthorized: shouldRejectUnauthorized(),
+            includeAdditionalCertificates: certificates,
+        })), { proxy: false, timeout: timeout_ms !== undefined ? timeout_ms : this.config.httpRequestTimeout, baseURL: tdpEndpoint, headers: Object.assign({ [ts_lib_shared_schema_1.HTTPAuthKeys.scopeToOrgHeaderKey]: orgSlug, [ts_lib_shared_schema_1.HTTPAuthKeys.jwtAuthTokenHeaderKey]: jwt }, additionalHeaders) }));
+    }
+    /**
+     * Creates an axios instance for the given base url, with the given config and rejectUnauthorized, and using
+     * the appropriate proxies configured for the connector. TDPClient.init must be called first. baseUrl is required
+     * to select the correct proxy, but should otherwise match config.baseUrl.
+     * @param options.baseUrl base url for proxy selection when this connector is hosted in a Hub
+     * @param options.config axios config
+     * @param options.rejectUnauthorized rejectUnauthorized, passed to the underlying http(s) Agent
+     * @param options.includeAdditionalCertificates whether to include organization certificates in the http agent
+     * @param timeout_ms - The axios http request timeout in milliseconds
+     * @returns an Axios instance configured with these settings and with proxying http(s) Agents
+     */
+    createAxiosInstance({ baseUrl, config, timeout_ms, rejectUnauthorized = true, includeAdditionalCertificates = true, }) {
+        this.assertInitialized();
+        this.logger.info('Creating axios instance', { baseUrl, config: sanitizeConfig(config), rejectUnauthorized });
+        const headers = this.getAdditionalAxiosHeaders();
+        return axios_1.default.create(Object.assign(Object.assign(Object.assign({ baseURL: baseUrl, timeout: timeout_ms !== undefined ? timeout_ms : this.config.httpRequestTimeout }, this.createProxyAgentsForBaseUrl({ baseUrl, rejectUnauthorized, includeAdditionalCertificates })), { proxy: false, headers }), config));
+    }
+    /**
+     * Creates httpAgent and httpsAgent appropriate for proxied connections to the given baseUrl.
+     * Reads proxy env vars to determine which proxy, if any, to use for a given url. npm proxy-agent
+     * and npm https-proxy-agent do not support rejectUnauthorized, which must be set on the agent level.
+     * Until we can fork npm proxy-agent to add this support, we cannot easily support per-request proxy selection
+     * like proxy-agent does, so we need the baseUrl up front to select a proxy.
+     * If using this with axios, use createAxiosInstance instead, or be sure to specify proxy: false to circumvent
+     * a related axios bug.
+     * @param options.baseURL The URL that determines which proxy env var (http_proxy, https_proxy, no_proxy) should apply
+     * @param options.rejectUnauthorized Whether to inject unauthorized requests in the httpsAgent
+     * @param options.includeAdditionalCertificates Whether to include organization certificates in the httpsAgent.
+     * If omitted or set falsy: only Node's built-in certificates will be included
+     * If set to true, built-in plus org certificates will be included.
+     * If set to an array, built-in plus this array of certificates will be included.
+     * @returns httpAgent and httpsAgent with proxy, rejectUnauthorized, and includeAdditionalCertificates settings
+     */
+    createProxyAgentsForBaseUrl({ baseUrl, rejectUnauthorized = true, includeAdditionalCertificates = true, }) {
+        this.assertProxyInitialized();
+        this.logger.info('Checking proxy for url', { baseUrl });
+        const proxyUrl = (0, proxy_from_env_1.getProxyForUrl)(baseUrl);
+        const builtInCertStrings = tls.rootCertificates;
+        const orgCertStrings = includeAdditionalCertificates === true ? this.certificates.map((c) => c.content) : [];
+        const additionalCertStrings = Array.isArray(includeAdditionalCertificates) ? includeAdditionalCertificates : [];
+        const ca = [...builtInCertStrings, ...orgCertStrings, ...additionalCertStrings];
+        if (orgCertStrings.length > 0) {
+            this.logger.info(`Using ${orgCertStrings.length} organization certificates`, {
+                certificates: this.certificates.map((c) => ({ id: c.id, name: c.name })),
+            });
+        }
+        if (additionalCertStrings.length > 0) {
+            this.logger.info(`Using ${additionalCertStrings.length} additional certificates`);
+        }
+        if (proxyUrl) {
+            this.logger.info('Creating HttpProxyAgent and HttpsProxyAgent with proxyUrl', {
+                rejectUnauthorized,
+                includeAdditionalCertificates,
+            });
+            const httpAgent = new hpagent_1.HttpProxyAgent({ proxy: proxyUrl });
+            let httpsAgent;
+            if (shouldUseHpAgent()) {
+                this.logger.info('Using hpagent for https agent');
+                httpsAgent = new hpagent_1.HttpsProxyAgent({
+                    proxy: proxyUrl,
+                    proxyRequestOptions: { ca },
+                    rejectUnauthorized,
+                });
+            }
+            else {
+                this.logger.info('Using default https agent');
+                httpsAgent = new https_proxy_agent_1.PatchedHttpsProxyAgent(proxyUrl, {
+                    ca,
+                    rejectUnauthorized,
+                });
+            }
+            return {
+                httpAgent,
+                httpsAgent,
+            };
+        }
+        if (rejectUnauthorized !== undefined) {
+            this.logger.info('Creating http.Agent and https.Agent with rejectUnauthorized', { rejectUnauthorized });
+            return {
+                httpAgent: new http.Agent(),
+                httpsAgent: new https.Agent({ rejectUnauthorized, ca }),
+            };
+        }
+        this.logger.info('No proxy settings detected, returning undefined');
+        return undefined;
+    }
+    /**
+     * Creates a NodeHttpHandler using npm proxy-agent, which will select a correct proxy per request.
+     * Only works for connections where rejectUnauthorized or additional certificates are not needed,
+     * such as to AWS or trusted servers; if you need to connect to an untrusted server or use organization certificates,
+     * use createProxyAgentsForBaseUrl and specify a baseUrl and (optionally) rejectUnauthorized: false.
+     * @returns A NodeHttpHandler with agents that will choose a correct proxy for each request.
+     */
+    createProxyNodeHttpHandler() {
+        this.assertProxyInitialized();
+        this.logger.info('Creating NodeHttpHandler');
+        return new node_http_handler_1.NodeHttpHandler({
+            httpAgent: new proxy_agent_1.ProxyAgent(),
+            httpsAgent: new proxy_agent_1.ProxyAgent(),
+        });
+    }
+    createLogger() {
+        return new logger_1.Logger({
+            writeToCloudwatch: !!process.env.CONNECTOR_TOKEN && process.env.SKIP_CLOUDWATCH !== 'true',
+            orgSlug: this.config.orgSlug,
+            connectorId: this.config.connectorId,
+            metadata: {
+                hubId: this.config.hubId,
+            },
+        });
+    }
+    init() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.loadProxySettingsToEnv();
+            if (process.env.CONNECTOR_TOKEN) {
+                const localTdpCertificates = yield (0, certificates_1.loadCertificatesFromLocalVolume)(this.config, this.logger);
+                const axiosClientWithLocalCertificates = this.createAxiosInstanceWithCertificatesAndHeaders(this.config.tdpEndpoint, this.config.orgSlug, process.env.CONNECTOR_TOKEN, localTdpCertificates);
+                this.awsClientProvider = new auth_1.AwsRefreshClientProvider(this.createProxyNodeHttpHandler(), axiosClientWithLocalCertificates, this.config.awsRegion, this.config.connectorId, this.logger);
+                this.authTokenProvider = new auth_1.UserSuppliedAuthTokenProvider(process.env.CONNECTOR_TOKEN);
+                if (!process.env.SKIP_CLOUDWATCH) {
+                    yield this.logger.startCloudwatch(this.awsClientProvider.getCloudwatchLogsClient());
+                }
+            }
+            else {
+                this.awsClientProvider = new auth_1.AwsEnvClientProvider(this.config.awsRegion, this.createProxyNodeHttpHandler());
+                this.authTokenProvider =
+                    this.config.authTokenProvider ||
+                        new auth_1.AwsSecretAuthTokenProvider(this.awsClientProvider.getSsmClient(), this.config.jwtTokenParameter);
+            }
+            this.logger.info(`Retrieving auth token`, {
+                jwtTokenParameter: this.config.jwtTokenParameter,
+                hasEnvToken: !!process.env.CONNECTOR_TOKEN,
+            });
+            const authToken = yield this.authTokenProvider.getAuthToken();
+            this.logger.info('Retrieved auth token');
+            this.jwt = authToken.value;
+            const tdpCertificates = yield (0, certificates_1.loadTdpCertificates)(this.awsClientProvider.getS3Client(), this.logger, this.config);
+            this.api = new api_1.Api(this.config, this.createAxiosInstanceWithCertificatesAndHeaders(this.config.tdpEndpoint, this.config.orgSlug, this.jwt, tdpCertificates), this.logger);
+            const requestId = (0, uuid_1.v4)();
+            const options = { requestId };
+            try {
+                this._certificates = yield this.getCertificates(options);
+                this.logger.info(`Loaded ${this._certificates.length} organization certificates`, {
+                    certificates: this._certificates.map((c) => ({ id: c.id, name: c.name })),
+                    requestId,
+                });
+            }
+            catch (ex) {
+                this.logger.error('Error loading organization certificates. Initialization will continue, but untrusted SSL connections will fail.', ex, requestId);
+            }
+            yield this.loadConnectorManifest();
+            yield this.reloadConnector(options);
+            this.logger.info('TDPClient.init complete');
+        });
+    }
+    loadConnectorManifest() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                this._manifest = yield (0, manifest_1.getConnectorManifest)(this.config.manifestFile, this.logger);
+            }
+            catch (err) {
+                this.logger.error('Error loading connector manifest', err);
+                this._manifest = null;
+            }
+        });
+    }
+    loadProxySettingsToEnv() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.logger.info('Loading proxy settings from Hub');
+            try {
+                // dotenv will silently no-op if the file doesn't exist or isn't readable, so we check here for better logging
+                yield fsPromises.access(hubProxyPath, fs.constants.R_OK);
+                // Axios only supports no_proxy via environment, not via explicit configuration.
+                // So we just use environment for everything.
+                dotenv.config({ path: hubProxyPath });
+                // Log whether settings exist, but don't log them, they may contain auth information
+                this.logger.info('Loaded proxy settings from Hub', {
+                    isNoProxySet: !!process.env.no_proxy,
+                    isHttpProxySet: !!process.env.http_proxy,
+                    isHttpsProxySet: !!process.env.https_proxy,
+                });
+                this.isProxyInitialized = true;
+            }
+            catch (ex) {
+                if (ex.code === 'ENOENT') {
+                    this.logger.info('No Hub env proxy file found', { hubProxyPath });
+                    this.isProxyInitialized = true;
+                    if (process.env.NO_PROXY && !process.env.no_proxy) {
+                        this.logger.info('Setting no_proxy from NO_PROXY');
+                        process.env.no_proxy = process.env.NO_PROXY;
+                    }
+                    if (process.env.HTTP_PROXY && !process.env.http_proxy) {
+                        this.logger.info('Setting http_proxy from HTTP_PROXY');
+                        process.env.http_proxy = process.env.HTTP_PROXY;
+                    }
+                    if (process.env.HTTPS_PROXY && !process.env.https_proxy) {
+                        this.logger.info('Setting https_proxy from HTTPS_PROXY');
+                        process.env.https_proxy = process.env.HTTPS_PROXY;
+                    }
+                    this.logger.info('Environment proxy settings', {
+                        isNoProxySet: !!process.env.no_proxy,
+                        isHttpProxySet: !!process.env.http_proxy,
+                        isHttpsProxySet: !!process.env.https_proxy,
+                    });
+                    return;
+                }
+                this.logger.error('Error reading proxy env file', ex);
+                throw ex;
+            }
+        });
+    }
+    startCommandsListener() {
+        this.logger.info('Starting commands listener');
+        this.listeningForCommands = true;
+        setImmediate(() => __awaiter(this, void 0, void 0, function* () {
+            while (this.listeningForCommands) {
+                try {
+                    const message = yield this.waitForCommandMessage();
+                    if (message) {
+                        yield this.handleCommandSqsMessages(message);
+                    }
+                }
+                catch (err) {
+                    this.logger.error('Error handling command', err);
+                }
+            }
+        }));
+    }
+    stopCommandsListener() {
+        this.logger.info('Stopping commands listener');
+        this.listeningForCommands = false;
+    }
+    waitForCommandMessage() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const waitTimeSeconds = 20;
+            const { Messages = [] } = yield this.assertInitialized().sqsClient.send(new sqs.ReceiveMessageCommand({
+                QueueUrl: this._connector.commandQueue,
+                MaxNumberOfMessages: 1,
+                WaitTimeSeconds: waitTimeSeconds,
+            }), {
+                abortSignal: AbortSignal.timeout(waitTimeSeconds + AWS_SQS_TIMEOUT),
+            });
+            return Messages.length > 0 ? Messages[0] : null;
+        });
+    }
+    handleCommandSqsMessages(message) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.logger.info('Handling command message', { messageId: message.MessageId });
+            const { Body, ReceiptHandle } = message;
+            const messageBody = JSON.parse(Body);
+            const expiresAt = new Date(messageBody.expiresAt);
+            if (expiresAt < new Date()) {
+                this.logger.info('Command expiry date passed, rejecting', {
+                    messageId: message.MessageId,
+                    action: messageBody.action,
+                    expiresAt,
+                });
+                yield this.sendCommandResponse(Object.assign(Object.assign({}, messageBody), { status: types_1.CommandStatus.REJECTED, body: {
+                        reason: `Command expiry date [${expiresAt.toISOString()}] is passed`,
+                    } }));
+            }
+            else {
+                this.logger.info('Marking command processing', {
+                    messageId: message.MessageId,
+                    action: messageBody.action,
+                });
+                yield this.sendCommandResponse(Object.assign(Object.assign({}, messageBody), { status: types_1.CommandStatus.PROCESSING }));
+                if (messageBody.bodyDelivery === types_1.PayloadDelivery.REFERENCED && messageBody.bodyUrl) {
+                    this.logger.info('Command body is referenced, downloading', { messageId: message.MessageId });
+                    messageBody.body = yield this.downloadContent(messageBody.bodyUrl);
+                }
+                this.logger.info('Emitting command events', {
+                    messageId: message.MessageId,
+                    action: messageBody.action,
+                });
+                this.commands.emit('command', messageBody);
+                this.commands.emit(messageBody.action, messageBody);
+            }
+            this.logger.info('Deleting SQS message', { messageId: message.MessageId });
+            yield this.assertInitialized().sqsClient.send(new sqs.DeleteMessageCommand({
+                QueueUrl: this._connector.commandQueue,
+                ReceiptHandle,
+            }), { abortSignal: AbortSignal.timeout(AWS_SQS_TIMEOUT) });
+            this.logger.info('Deleted SQS message', { messageId: message.MessageId });
+        });
+    }
+    downloadContent(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { baseUrl, remainingPart } = (0, utils_1.splitUrl)(url);
+            const axios = this.createAxiosInstance({ baseUrl });
+            const { data } = yield axios.get(remainingPart);
+            return data;
+        });
+    }
+    /**
+     * Send a command response to the TDP
+     * @param command - The command response to send
+     * @param command.commandId - The command ID
+     * @param command.targetId - The target ID (connector ID)
+     * @param command.status - The status of the command
+     * @param command.body - The body of the command
+     * @param command.responseBodyUploadUrl - The URL to upload the response body to.
+     *  Should be passed over from the command request
+     */
+    sendCommandResponse(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.logger.info('Sending command response', {
+                commandId: command.commandId,
+            });
+            const { commandId, targetId, status, body } = command;
+            const messageBody = {
+                commandId,
+                targetId,
+                status,
+                body,
+                responseBodyDelivery: types_1.PayloadDelivery.EMBEDDED,
+                createdAt: new Date().toISOString(),
+            };
+            if (this.shouldDeliverCommandPayloadViaUpload(command)) {
+                this.logger.info('Uploading command response body', { commandId: command.commandId });
+                yield this.uploadCommandResponseBody(command.responseBodyUploadUrl, command.body);
+                messageBody.responseBodyDelivery = types_1.PayloadDelivery.REFERENCED;
+                messageBody.body = undefined;
+            }
+            this.assertInitialized().sqsClient.send(new sqs.SendMessageCommand({
+                QueueUrl: this.config.outboundCommandQueue,
+                MessageBody: JSON.stringify(messageBody),
+            }), { abortSignal: AbortSignal.timeout(AWS_SQS_TIMEOUT) });
+            this.logger.info('Sent command response', { commandId: command.commandId });
+        });
+    }
+    shouldDeliverCommandPayloadViaUpload(command) {
+        if (!command.responseBodyUploadUrl || !command.body) {
+            return false;
+        }
+        const { commandId, targetId, status, body } = command;
+        const messageBody = {
+            commandId,
+            targetId,
+            status,
+            body,
+            responseBodyDelivery: types_1.PayloadDelivery.EMBEDDED,
+            createdAt: new Date().toISOString(),
+        };
+        const messageSize = Buffer.byteLength(JSON.stringify(messageBody), 'utf8');
+        const maxEmbeddedMessageSize = 256 * 1024;
+        return messageSize > maxEmbeddedMessageSize;
+    }
+    uploadCommandResponseBody(uploadUrl, responseBody) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { baseUrl, remainingPart } = (0, utils_1.splitUrl)(uploadUrl);
+            const axios = this.createAxiosInstance({ baseUrl });
+            const bodyString = JSON.stringify(responseBody);
+            const compressedBodyString = zlib_1.default.gzipSync(Buffer.from(bodyString));
+            yield axios.put(remainingPart, compressedBodyString, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Encoding': 'gzip',
+                },
+            });
+        });
+    }
+    getCertificates(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.apiInstance.getCertificates(options);
+        });
+    }
+    getConfig(_a, options_1) {
+        return __awaiter(this, arguments, void 0, function* ({ version }, options) {
+            return this.apiInstance.getConnectorConfig(this.config.connectorId, { version }, options);
+        });
+    }
+    getConnector() {
+        return __awaiter(this, arguments, void 0, function* ({ resolveSecrets, include } = {}, options) {
+            return this.apiInstance.getConnectorById(this.config.connectorId, { resolveSecrets, include }, options);
+        });
+    }
+    reloadConnector(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this._connector = yield this.getConnector({
+                resolveSecrets: true,
+                include: ['config', 'artifact'],
+            }, options);
+            yield this.configureUserAgent();
+        });
+    }
+    hasUserAgentSetToDisabled() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let value = undefined;
+            try {
+                value = yield this.getValue(TS_SDK_SETTINGS_KEY);
+            }
+            catch (err) {
+                this.logger.error('Error getting connector value for user agent', err);
+            }
+            if (value === undefined) {
+                return false;
+            }
+            const disableUserAgent = value[TS_SDK_DISABLE_USER_AGENT_HTTP_HEADER];
+            return disableUserAgent === true;
+        });
+    }
+    configureUserAgent() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const disabledUserAgent = yield this.hasUserAgentSetToDisabled();
+            if (disabledUserAgent === true) {
+                this.logger.info('User-Agent header is disabled');
+                return;
+            }
+            let userAgent = 'PluggableConnector';
+            const manifest = this._manifest;
+            if (manifest) {
+                this.logger.debug('Configuring user agent using artifact details from manifest');
+                const version = manifest.version.startsWith('v') ? manifest.version : `v${manifest.version}`;
+                userAgent += ` ${manifest.slug} ${version}`;
+            }
+            else {
+                this.logger.debug('Manifest not found, using default user agent');
+                userAgent += ` ${this.constructor.name}`;
+            }
+            this.setUserAgent(userAgent);
+        });
+    }
+    getValue(key, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const [value] = yield this.getValues([key], options);
+            return value;
+        });
+    }
+    getValues(...args) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let options;
+            let keys;
+            if (Array.isArray(args[0])) {
+                keys = args[0];
+                options = args[1];
+            }
+            else {
+                options = { requestId: undefined };
+                keys = args;
+            }
+            const { values } = yield this.apiInstance.getConnectorData(this.config.connectorId, { keys }, options);
+            return values.map(({ value }) => value);
+        });
+    }
+    saveValue(key, value, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const _a = options || {}, { requestId } = _a, requestOptions = __rest(_a, ["requestId"]);
+            const [{ value: savedValue }] = yield this.saveValues([Object.assign({ key, value }, requestOptions)], { requestId });
+            return savedValue;
+        });
+    }
+    saveValues(values, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { values: savedValues } = yield this.apiInstance.saveConnectorData(this.config.connectorId, { values }, options);
+            return savedValues;
+        });
+    }
+    getFiles(query, options) {
+        return this.apiInstance.getConnectorFiles(this.config.connectorId, query, options);
+    }
+    saveFile(file) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const { files, failures } = yield this.saveFiles([file]);
+            if (failures.length > 0) {
+                const failure = failures === null || failures === void 0 ? void 0 : failures[0];
+                this.logger.error('File save failed', {
+                    fileId: file.id,
+                    uniqueExternalId: file.uniqueExternalId,
+                    failure,
+                });
+                throw new Error((_a = failure === null || failure === void 0 ? void 0 : failure.message) !== null && _a !== void 0 ? _a : 'File save failed');
+            }
+            return files === null || files === void 0 ? void 0 : files[0];
+        });
+    }
+    saveFiles(files, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.apiInstance.updateConnectorFiles(this.config.connectorId, { files }, options);
+        });
+    }
+    reportMetrics(metrics, options) {
+        return this.apiInstance.reportMetrics(this.config.connectorId, { metrics }, options);
+    }
+    reportHealthStatus(request, options) {
+        return this.apiInstance.health(this.config.connectorId, request, options);
+    }
+    heartbeat(options) {
+        return this.apiInstance.heartbeat(this.config.connectorId, options);
+    }
+    /**
+     * Determine PartSize for Upload based on object size
+     *
+     * @param objectSize size in bytes of object to upload
+     */
+    calculatePartSize(objectSize) {
+        const DEFAULT_PART_SIZE = 1024 * 1024 * 5; // AWS lower limit
+        const AWS_MAX_PARTS = 10000;
+        if (!objectSize) {
+            return DEFAULT_PART_SIZE;
+        }
+        else {
+            // inflate object size by a byte to avoid possible annoying boundary
+            // issues
+            return Math.max(DEFAULT_PART_SIZE, Math.ceil((objectSize + 1) / AWS_MAX_PARTS));
+        }
+    }
+    /**
+     * Uploads a file to the TDP datalake
+  
+     * @param {UploadFileRequest} request - The upload file request
+     * @param {boolean} [strictMtlValidation=false] - Whether to throw an error if the request MTL is invalid.
+     * If false, will just log a warning.
+     * @returns {Promise<UploadFileResponse>} - Uploaded file bucket, key, version, and id
+     * @throws {Error} - If the client is not initialized
+     * @throws {Error} - If the request has invalid MTL and strictMtlValidation is true
+     */
+    uploadFile(request_1) {
+        return __awaiter(this, arguments, void 0, function* (request, strictMtlValidation = false) {
+            var _a, _b, _c, _d, _e;
+            if (!this.isInitialized) {
+                throw new Error('Client is not initialized');
+            }
+            try {
+                request = (0, schemas_1.validateUploadFileRequestAndThrowFriendlyError)(request);
+            }
+            catch (ex) {
+                if (strictMtlValidation) {
+                    this.logger.error('Invalid MTL on upload file request', ex);
+                    throw ex;
+                }
+                else {
+                    this.logger.warn('Invalid MTL on upload file request', ex);
+                }
+            }
+            if (request.destinationId != null) {
+                // will throw with error message if destinationId invalid
+                DestinationIdSchema.parse(request.destinationId);
+            }
+            const { orgSlug, connectorId, datalakeBucket } = this.config;
+            const idForKey = (_a = request.destinationId) !== null && _a !== void 0 ? _a : connectorId;
+            const fileId = request.fileId || (0, uuid_1.v4)();
+            const traceId = request.traceId || fileId;
+            const key = this.buildS3Key(orgSlug, idForKey, ts_lib_shared_schema_1.FileCategories.RAW, request.filepath);
+            this.logger.debug('Uploading file', {
+                fileId,
+                traceId,
+                key,
+            });
+            const customMetadata = request.metadataDirective === Directive.Replace
+                ? request.metadata || {}
+                : Object.assign(Object.assign({}, this._connector.metadata), request.metadata);
+            const customTags = request.tagsDirective === Directive.Replace
+                ? request.tags || []
+                : [...(this._connector.tags || []), ...(request.tags || [])];
+            const parameterChecksums = (_b = request.checksums) !== null && _b !== void 0 ? _b : {};
+            const metadataChecksums = (0, checksums_1.mapChecksumData)(parameterChecksums);
+            const awsChecksums = {
+                ChecksumAlgorithm: parameterChecksums.s3AdditionalChecksum || s3.ChecksumAlgorithm.SHA256,
+            };
+            const artifact = this._connector.artifact;
+            const connectorSlug = artifact.slug;
+            const hostType = (_c = this._connector) === null || _c === void 0 ? void 0 : _c.hostType;
+            const isHubConnector = hostType === api_1.ConnectorHostType.HUB;
+            const integrationType = isHubConnector ? ts_lib_shared_schema_1.IntegrationTypes.HUB : ts_lib_shared_schema_1.IntegrationTypes.API;
+            const integrationId = isHubConnector ? (_d = this._connector) === null || _d === void 0 ? void 0 : _d.hub.id : ts_lib_shared_schema_1.Constants.API_UPLOAD_V1_INTEGRATION_ID;
+            const trace = request.trace || {};
+            const traceMetadata = this.transformTraceMetadata(trace);
+            if (request.labels && request.labels.length) {
+                const labelKey = this.buildS3Key(orgSlug, idForKey, ts_lib_shared_schema_1.FileCategories.TMP, `${request.filepath}/${fileId}.labels`);
+                const labelFile = JSON.stringify(request.labels);
+                yield this.assertInitialized().s3Client.send(new s3.PutObjectCommand(Object.assign({ Bucket: datalakeBucket, Key: labelKey, Body: labelFile, ServerSideEncryption: 'aws:kms', SSEKMSKeyId: this.config.kmsKeyId }, awsChecksums)));
+            }
+            const upload = new lib_storage_1.Upload({
+                client: this.assertInitialized().s3Client,
+                params: Object.assign(Object.assign(Object.assign(Object.assign({ Bucket: datalakeBucket, Key: key, Body: request.content, ServerSideEncryption: 'aws:kms', SSEKMSKeyId: this.config.kmsKeyId }, awsChecksums), (request.contentType ? { ContentType: request.contentType } : {})), (request.contentEncoding ? { ContentEncoding: request.contentEncoding } : {})), { Metadata: Object.assign(Object.assign(Object.assign(Object.assign({}, metadataChecksums), traceMetadata), { [ts_lib_shared_schema_1.S3MetadataFields.FILE_ID]: fileId, [ts_lib_shared_schema_1.S3MetadataFields.TRACE_ID]: traceId, [ts_lib_shared_schema_1.S3MetadataFields.INTEGRATION_TYPE]: integrationType, [ts_lib_shared_schema_1.S3MetadataFields.INTEGRATION_ID]: integrationId, [ts_lib_shared_schema_1.S3MetadataFields.CONNECTOR_NAMESPACE]: artifact.namespace, [ts_lib_shared_schema_1.S3MetadataFields.CONNECTOR_SLUG]: connectorSlug, [ts_lib_shared_schema_1.S3MetadataFields.CONNECTOR_VERSION]: artifact.version, [ts_lib_shared_schema_1.S3MetadataFields.CUSTOM_METADATA]: new URLSearchParams(customMetadata).toString(), [ts_lib_shared_schema_1.S3MetadataFields.CUSTOM_TAGS]: (0, utils_1.unique)(customTags).join(','), [ts_lib_shared_schema_1.S3MetadataFields.SOURCE_ID]: connectorId, [ts_lib_shared_schema_1.S3MetadataFields.SOURCE_NAME]: (_e = this._connector) === null || _e === void 0 ? void 0 : _e.name, [ts_lib_shared_schema_1.S3MetadataFields.SOURCE_TYPE]: request.sourceType || artifact.sourceType || ts_lib_shared_schema_1.SourceTypes.UNKNOWN, [ts_lib_shared_schema_1.S3MetadataFields.DO_NOT_INHERIT_LABELS]: request.labelsDirective === Directive.Replace ? utils_1.Bool.True : utils_1.Bool.False }), (request.destinationId ? { [ts_lib_shared_schema_1.S3MetadataFields.DESTINATION_ID]: request.destinationId } : {})) }),
+                partSize: this.calculatePartSize(request.objectSize),
+                abortController: request.abortController,
+            });
+            const { VersionId: versionId } = yield upload.done();
+            this.logger.debug('Uploaded file', {
+                fileId,
+                traceId,
+                key,
+                versionId,
+            });
+            return {
+                fileId,
+                bucket: datalakeBucket,
+                key,
+                versionId: versionId,
+            };
+        });
+    }
+    /**
+     * Transforms the metadata fields of a trace to include the "ts_" prefix
+     * @example
+     * // returns { ts_field1: 'value1', ts_field2: 'value2' }
+     * transformTraceMetadata({ field1: 'value1', field2: 'value2' })
+     *
+     * @param {Trace} trace - The trace object to be transformed
+     * @returns {object} - An object with transformed trace metadata.
+     */
+    transformTraceMetadata(trace) {
+        return Object.keys(trace).reduce((meta, field) => (Object.assign(Object.assign({}, meta), { [`ts_${field}`]: trace[field] })), {});
+    }
+    buildS3Key(orgSlug, idForKey, category, filepath) {
+        const prefix = `${orgSlug}/${idForKey}/${category}`;
+        return filepath.startsWith('/') ? prefix + filepath : `${prefix}/${filepath}`;
+    }
+}
+exports.TDPClient = TDPClient;
+const shouldUseHpAgent = () => process.env.HTTPS_AGENT === HP_AGENT;
+const shouldRejectUnauthorized = () => {
+    if (process.env.NODE_TLS_REJECT_UNAUTHORIZED === undefined) {
+        return true;
+    }
+    return !['false', '0'].includes(process.env.NODE_TLS_REJECT_UNAUTHORIZED);
+};
+//# sourceMappingURL=tdp-client.js.map