@jupiterone/integration-sdk-http-client 12.1.0 → 12.2.0

package/dist/src/errors.js

@@ -1,15 +1,87 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.APIError = void 0;
-class APIError extends Error {
-    /**
-     * Code associated with the error.
-     */
-    code;
-    constructor(config) {
-        super(config.message);
-        this.code = config.code;
-    }
-}
-exports.APIError = APIError;
+exports.isRetryableRequest = exports.fatalRequestError = exports.retryableRequestError = exports.HTTPResponseError = exports.RateLimitError = exports.RetryableIntegrationProviderApiError = void 0;
+const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
+class RetryableIntegrationProviderApiError extends integration_sdk_core_1.IntegrationProviderAPIError {
+    retryable = true;
+}
+exports.RetryableIntegrationProviderApiError = RetryableIntegrationProviderApiError;
+class RateLimitError extends RetryableIntegrationProviderApiError {
+    constructor(options) {
+        super(options);
+        this.retryAfter = options.retryAfter;
+    }
+    retryAfter;
+}
+exports.RateLimitError = RateLimitError;
+class HTTPResponseError extends Error {
+    response;
+    constructor(response) {
+        super(`HTTP Error Response: ${response.status} ${response.statusText}`);
+        this.response = response;
+    }
+}
+exports.HTTPResponseError = HTTPResponseError;
+async function handleLogErrorBody(response, logger, logErrorBody) {
+    if (!logErrorBody) {
+        return;
+    }
+    let errorBody;
+    try {
+        errorBody = await response.json();
+        logger.error({ errBody: JSON.stringify(errorBody) }, 'Encountered error from API');
+    }
+    catch (e) {
+        // pass
+    }
+}
+async function retryableRequestError({ endpoint, response, logger, logErrorBody, }) {
+    await handleLogErrorBody(response, logger, logErrorBody);
+    if (response.status === 429) {
+        return new RateLimitError({
+            cause: new HTTPResponseError(response),
+            status: response.status,
+            statusText: response.statusText,
+            endpoint,
+            retryAfter: Number(response.headers.get('retry-after')),
+        });
+    }
+    return new RetryableIntegrationProviderApiError({
+        cause: new HTTPResponseError(response),
+        endpoint,
+        status: response.status,
+        statusText: response.statusText,
+    });
+}
+exports.retryableRequestError = retryableRequestError;
+async function fatalRequestError({ endpoint, response, logger, logErrorBody, }) {
+    await handleLogErrorBody(response, logger, logErrorBody);
+    const apiErrorOptions = {
+        cause: new HTTPResponseError(response),
+        endpoint,
+        status: response.status,
+        statusText: response.statusText,
+    };
+    if (response.status === 401) {
+        return new integration_sdk_core_1.IntegrationProviderAuthenticationError(apiErrorOptions);
+    }
+    else if (response.status === 403) {
+        return new integration_sdk_core_1.IntegrationProviderAuthorizationError(apiErrorOptions);
+    }
+    else {
+        return new integration_sdk_core_1.IntegrationProviderAPIError(apiErrorOptions);
+    }
+}
+exports.fatalRequestError = fatalRequestError;
+/**
+ * Function for determining if a request is retryable
+ * based on the returned status.
+ */
+function isRetryableRequest({ status }) {
+    return (
+    // 5xx error from provider (their fault, might be retryable)
+    // 429 === too many requests, we got rate limited so safe to try again
+    status >= 500 || status === 429);
+}
+exports.isRetryableRequest = isRetryableRequest;
 //# sourceMappingURL=errors.js.map

package/dist/src/errors.js.map

@@ -1 +1 @@
-{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":";;;AAMA,MAAa,QAAS,SAAQ,KAAK;IACjC;;OAEG;IACM,IAAI,CAAS;IAEtB,YAAY,MAAsB;QAChC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IAC1B,CAAC;CACF;AAVD,4BAUC"}
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":";;;AAAA,2EAK0C;AAgB1C,MAAa,oCAAqC,SAAQ,kDAA2B;IACnF,SAAS,GAAG,IAAI,CAAC;CAClB;AAFD,oFAEC;AAED,MAAa,cAAe,SAAQ,oCAAoC;IACtE,YAAY,OAA6B;QACvC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACvC,CAAC;IACD,UAAU,CAAS;CACpB;AAND,wCAMC;AAED,MAAa,iBAAkB,SAAQ,KAAK;IAC1C,QAAQ,CAAW;IACnB,YAAY,QAAkB;QAC5B,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;CACF;AAND,8CAMC;AAED,KAAK,UAAU,kBAAkB,CAC/B,QAAkB,EAClB,MAAyB,EACzB,YAAqB;IAErB,IAAI,CAAC,YAAY,EAAE;QACjB,OAAO;KACR;IACD,IAAI,SAAc,CAAC;IACnB,IAAI;QACF,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CACV,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,EACtC,4BAA4B,CAC7B,CAAC;KACH;IAAC,OAAO,CAAC,EAAE;QACV,OAAO;KACR;AACH,CAAC;AAEM,KAAK,UAAU,qBAAqB,CAAC,EAC1C,QAAQ,EACR,QAAQ,EACR,MAAM,EACN,YAAY,GACO;IACnB,MAAM,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAEzD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;QAC3B,OAAO,IAAI,cAAc,CAAC;YACxB,KAAK,EAAE,IAAI,iBAAiB,CAAC,QAAQ,CAAC;YACtC,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,QAAQ;YACR,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;SACxD,CAAC,CAAC;KACJ;IAED,OAAO,IAAI,oCAAoC,CAAC;QAC9C,KAAK,EAAE,IAAI,iBAAiB,CAAC,QAAQ,CAAC;QACtC,QAAQ;QACR,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;KAChC,CAAC,CAAC;AACL,CAAC;AAxBD,sDAwBC;AAEM,KAAK,UAAU,iBAAiB,CAAC,EACtC,QAAQ,EACR,QAAQ,EACR,MAAM,EACN,YAAY,GACO;IACnB,MAAM,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAEzD,MAAM,eAAe,GAAG;QACtB,KAAK,EAAE,IAAI,iBAAiB,CAAC,QAAQ,CAAC;QACtC,QAAQ;QACR,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;KAChC,CAAC;IACF,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;QAC3B,OAAO,IAAI,6DAAsC,CAAC,eAAe,CAAC,CAAC;KACpE;SAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;QAClC,OAAO,IAAI,4DAAqC,CAAC,eAAe,CAAC,CAAC;KACnE;SAAM;QACL,OAAO,IAAI,kDAA2B,CAAC,eAAe,CAAC,CAAC;KACzD;AACH,CAAC;AArBD,8CAqBC;AAED;;;GAGG;AACH,SAAgB,kBAAkB,CAAC,EAAE,MAAM,EAAY;IACrD,OAAO;IACL,4DAA4D;IAC5D,sEAAsE;IACtE,MAAM,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG,CAChC,CAAC;AACJ,CAAC;AAND,gDAMC"}

package/dist/src/index.d.ts

@@ -1,14 +1,3 @@
-import { RateLimitConfig, APIResponse, APIRequest } from './types';
-export declare const DEFAULT_RATE_LIMIT_CONFIG: RateLimitConfig;
-export type APIResourceIterationCallback<T> = (resources: T[]) => boolean | void | Promise<boolean | void>;
-export declare class APIClient {
-    private rateLimitConfig;
-    private rateLimitState;
-    executeAPIRequest(request: APIRequest): Promise<APIResponse>;
-    /**
-     * Get rate limit info, sleep, and then loop to retry
-     *
-     * @param headers
-     */
-    private handleRateLimitError;
-}
+export * from './client';
+export * from './errors';
+export * from './types';

package/dist/src/index.js

@@ -1,78 +1,20 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.APIClient = exports.DEFAULT_RATE_LIMIT_CONFIG = void 0;
-const node_fetch_1 = __importDefault(require("node-fetch"));
-const util_1 = require("./util");
-const errors_1 = require("./errors");
-exports.DEFAULT_RATE_LIMIT_CONFIG = {
-    maxAttempts: 5,
-    reserveLimit: 30,
-    cooldownPeriod: 1000,
-    sleepAdditionalSeconds: 0,
-};
-class APIClient {
-    // TODO (adam-in-ict) make rateLimitConfig configurable and only use the default values when none are provided
-    rateLimitConfig = exports.DEFAULT_RATE_LIMIT_CONFIG;
-    rateLimitState;
-    async executeAPIRequest(request) {
-        let attempts = 0;
-        do {
-            const response = await (0, node_fetch_1.default)(request.url, request);
-            if ((0, util_1.isNonRetryableError)(response.status)) {
-                const err = new errors_1.APIError({
-                    message: `API request error for ${request.url}: ${response.status} ${response.statusText}`,
-                    code: response.status,
-                });
-                throw err;
-            }
-            // TODO (adam-in-ict) make rate limit error configurable for instances where an API isn't using the default of 429
-            if (response.status === 429) {
-                await this.handleRateLimitError(response.headers);
-            }
-            else {
-                // Successful response, so return
-                return {
-                    data: await response.json(),
-                    headers: response.headers,
-                    status: response.status,
-                    statusText: response.statusText,
-                };
-            }
-            attempts += 1;
-        } while (attempts < this.rateLimitConfig.maxAttempts);
-        throw new errors_1.APIError({
-            message: `Could not complete request within ${attempts} attempts!`,
-            code: 429,
-        });
-    }
-    //TODO (adam-in-ict) abstract out which Header values we're grabbing for rate limiting.
-    /**
-     * Get rate limit info, sleep, and then loop to retry
-     *
-     * @param headers
-     */
-    async handleRateLimitError(headers) {
-        this.rateLimitState = {
-            limitRemaining: Number(headers.get('X-RateLimit-Remaining')),
-            perMinuteLimit: Number(headers.get('X-RateLimit-Limit')),
-            retryAfter: headers.get('X-RateLimit-RetryAfter') &&
-                Number(headers.get('X-RateLimit-RetryAfter')),
-        };
-        const unixTimeNow = (0, util_1.getUnixTimeNow)();
-        const timeToSleepInSeconds = this.rateLimitState.retryAfter
-            ? Math.max(this.rateLimitState.retryAfter +
-                this.rateLimitConfig.sleepAdditionalSeconds -
-                unixTimeNow, 0)
-            : 0;
-        await (0, util_1.sleep)(timeToSleepInSeconds * 1000);
-        if (this.rateLimitState.limitRemaining &&
-            this.rateLimitState.limitRemaining <= this.rateLimitConfig.reserveLimit) {
-            await (0, util_1.sleep)(this.rateLimitConfig.cooldownPeriod);
-        }
-    }
-}
-exports.APIClient = APIClient;
+__exportStar(require("./client"), exports);
+__exportStar(require("./errors"), exports);
+__exportStar(require("./types"), exports);
 //# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;AAAA,4DAA+B;AAC/B,iCAAoE;AAOpE,qCAAoC;AAEvB,QAAA,yBAAyB,GAAoB;IACxD,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,EAAE;IAChB,cAAc,EAAE,IAAI;IACpB,sBAAsB,EAAE,CAAC;CAC1B,CAAC;AAMF,MAAa,SAAS;IACpB,8GAA8G;IACtG,eAAe,GAAoB,iCAAyB,CAAC;IAC7D,cAAc,CAAiB;IAEvC,KAAK,CAAC,iBAAiB,CAAC,OAAmB;QACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,GAAG;YACD,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAK,EAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAEnD,IAAI,IAAA,0BAAmB,EAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACxC,MAAM,GAAG,GAAG,IAAI,iBAAQ,CAAC;oBACvB,OAAO,EAAE,yBAAyB,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;oBAC1F,IAAI,EAAE,QAAQ,CAAC,MAAM;iBACtB,CAAC,CAAC;gBACH,MAAM,GAAG,CAAC;aACX;YAED,kHAAkH;YAClH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;gBAC3B,MAAM,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aACnD;iBAAM;gBACL,iCAAiC;gBACjC,OAAO;oBACL,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE;oBAC3B,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;iBAChC,CAAC;aACH;YAED,QAAQ,IAAI,CAAC,CAAC;SACf,QAAQ,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE;QAEtD,MAAM,IAAI,iBAAQ,CAAC;YACjB,OAAO,EAAE,qCAAqC,QAAQ,YAAY;YAClE,IAAI,EAAE,GAAG;SACV,CAAC,CAAC;IACL,CAAC;IAED,uFAAuF;IACvF;;;;OAIG;IACK,KAAK,CAAC,oBAAoB,CAAC,OAAO;QACxC,IAAI,CAAC,cAAc,GAAG;YACpB,cAAc,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;YAC5D,cAAc,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACxD,UAAU,EACR,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;gBACrC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;SAChD,CAAC;QAEF,MAAM,WAAW,GAAG,IAAA,qBAAc,GAAE,CAAC;QACrC,MAAM,oBAAoB,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU;YACzD,CAAC,CAAC,IAAI,CAAC,GAAG,CACN,IAAI,CAAC,cAAc,CAAC,UAAU;gBAC5B,IAAI,CAAC,eAAe,CAAC,sBAAsB;gBAC3C,WAAW,EACb,CAAC,CACF;YACH,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,IAAA,YAAK,EAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;QACzC,IACE,IAAI,CAAC,cAAc,CAAC,cAAc;YAClC,IAAI,CAAC,cAAc,CAAC,cAAc,IAAI,IAAI,CAAC,eAAe,CAAC,YAAY,EACvE;YACA,MAAM,IAAA,YAAK,EAAC,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;SAClD;IACH,CAAC;CACF;AAzED,8BAyEC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2CAAyB;AACzB,2CAAyB;AACzB,0CAAwB"}

package/dist/src/types.d.ts

@@ -1,132 +1,42 @@
-export type APIRequest = {
-    url: string;
-    method: string;
-    headers: Object;
-};
-export type APIResponse = {
-    data: any;
-    headers: any;
-    status: number;
-    statusText: string;
-};
-type PaginationState = {
-    /**
-     * Fetch limit, will be URL encoded as value of `limit` GET parameter.
-     */
-    limit?: number;
-    /**
-     * Total number of objects reported in API response. The number is affected by
-     * the filter, if any.
-     */
-    total?: number;
-    /**
-     * Fetch offset, a number to start from or a cursor, depending on API.
-     */
-    offset?: number | string;
-    /**
-     * Offset cursor expiration time, only present for cursor based API.
-     */
-    expiresAt?: number;
-    /**
-     * Similar to "offset", a value to start from aka a cursor.
-     */
-    after?: number | string;
-    /**
-     * Number of resources returned through pagination to the point of offset.
-     */
-    seen: number;
-    /**
-     * Number of pages returned through pagination to the point of offset.
-     */
-    pages: number;
-    /**
-     * Pagination has completed according to provided pagination parameters.
-     */
-    finished: boolean;
-};
-export type PaginationParams = Partial<PaginationState>;
-export type QueryParams = {
-    filter?: string;
-    [name: string]: string | undefined;
-};
-export type RateLimitConfig = {
-    /**
-     * The limit remaining value at which the client should slow down. This
-     * prevents the client from consuming all available requests.
-     */
-    reserveLimit: number;
-    /**
-     * A recommended period of time in milliseconds to wait between requests when
-     * the `reserveLimit` is reached.
-     *
-     * This can be a value representing the refill rate of a "leaky bucket" or
-     * just a guess about how soon another request can be made. Ideally there will
-     * be enough information in the response headers to calculate a better value.
-     */
-    cooldownPeriod: number;
-    /**
-     * Maximum number of times to retry a request that continues to receive 429
-     * responses.
-     *
-     * The client will respect `x-ratelimit-retryafter`, but should it end up in a
-     * battle to get the next allowed request, it will give up after this many
-     * tries.
-     */
+import { IntegrationLogger } from '@jupiterone/integration-sdk-core';
+import { AttemptContext } from '@lifeomic/attempt';
+export type OptionalPromise<T> = T | Promise<T>;
+export interface RequestOptions {
+    method?: 'GET' | 'POST';
+    body?: Record<string, unknown>;
+    headers?: Record<string, string>;
+    authorize?: boolean;
+    bucketTokens?: number;
+}
+export interface RetryOptions {
     maxAttempts: number;
-    /**
-     * Optional number of additional seconds to sleep in the event of a rate limit
-     * event.
-     */
-    sleepAdditionalSeconds: number;
-};
-/**
- * The last seen values from rate limit response headers.
- */
-export type RateLimitState = {
-    /**
-     * Maximum number of requests per minute that can be made by all API clients
-     * in a customer account. Initial value assumes the published default of 100.
-     */
-    perMinuteLimit: number;
-    /**
-     * Number of requests that remain in the account's rate limiting pool. The
-     * total number available is not known.
-     */
-    limitRemaining: number;
-    /**
-     * The next time when an account's rate limit pool will have at least one
-     * request available.
-     */
-    retryAfter?: number;
-};
-/**
- * Metadata in API responses indicating the pagination state.
- */
-export type PaginationMeta = {
-    limit: number;
-    total: number;
-    offset: number | string;
-    expires_at?: number;
-};
-type ResponseMeta = {
-    trace_id: string;
-    pagination?: PaginationMeta;
-};
-type ResponseError = {
-    code: number;
-    message: string;
-};
-export type ResourcesResponse<T> = {
-    meta: ResponseMeta;
-    errors: ResponseError[];
-    resources: T[];
-};
-export type APIClientRateLimitConfig = {
-    limitHeader: string;
-    limitRemainingHeader: string;
-    retryAfterHeader: string;
-};
-export type APIClientConfiguration = {
-    rateLimitConfig: APIClientRateLimitConfig;
-};
-export {};
+    delay: number;
+    factor: number;
+    timeout: number;
+    handleError: (err: any, context: AttemptContext, logger: IntegrationLogger) => OptionalPromise<void>;
+}
+export interface RateLimitHeaders {
+    limit: string;
+    remaining: string;
+    reset: string;
+}
+export interface RateLimitThrottlingOptions {
+    threshold: number;
+    rateLimitHeaders?: RateLimitHeaders;
+}
+export interface TokenBucketOptions {
+    maximumCapacity: number;
+    refillRate: number;
+}
+export interface ClientConfig {
+    baseUrl: string;
+    logger: IntegrationLogger;
+    retryOptions?: Partial<RetryOptions>;
+    logErrorBody?: boolean;
+    rateLimitThrottling?: RateLimitThrottlingOptions;
+    tokenBucket?: TokenBucketOptions;
+}
+export interface IterateCallbackResult {
+    nextUrl?: string;
+    nextRequestOptions?: RequestOptions;
+}