@gitbeaker/requester-utils 39.17.0 → 39.19.0

package/README.md

@@ -73,7 +73,7 @@
 
 - [Usage](#usage)
 - [Testing](../../docs/TESTING.md)
-- [FAQ](../../FAQ.md)
+- [FAQ](../../docs/FAQ.md)
 - [Contributors](#contributors)
 - [Changelog](./CHANGELOG.md)
 
@@ -230,6 +230,7 @@ import { RequesterUtils, BaseResource } from '@gitbeaker/requester-utils';
       <td align="center" valign="top" width="3.84%"><a href="https://github.com/demedos"><img src="https://images.weserv.nl/?url=https://avatars.githubusercontent.com/u/16702156?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="Alessandro Diez"/></td>
       <td align="center" valign="top" width="3.84%"><a href="https://github.com/artlist-scottambrose"><img src="https://images.weserv.nl/?url=https://avatars.githubusercontent.com/u/124692101?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="artlist-scottambrose"/></td>
       <td align="center" valign="top" width="3.84%"><a href="https://github.com/mercutiodesign"><img src="https://images.weserv.nl/?url=https://avatars.githubusercontent.com/u/1114120?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="Martin Dreher"/></td>
+      <td align="center" valign="top" width="3.84%"><a href="https://github.com/glensc"><img src="https://images.weserv.nl/?url=https://avatars.githubusercontent.com/u/199095?v=4&h=25&w=25&fit=cover&mask=circle&maxage=7d" alt="Elan Ruusamäe"/></td>
     </tr>
 </p>
 

package/dist/index.d.mts

@@ -1,3 +1,13 @@
+import { RateLimit } from 'async-sema';
+
+type RateLimiters = Record<string, ReturnType<typeof RateLimit> | {
+    method: string;
+    limit: ReturnType<typeof RateLimit>;
+}>;
+type RateLimitOptions = Record<string, number | {
+    method: string;
+    limit: number;
+}>;
 type ResponseBodyTypes = Record<string, unknown> | Record<string, unknown>[] | ReadableStream | Blob | string | string[] | number | void | null;
 interface FormattedResponse<T extends ResponseBodyTypes = ResponseBodyTypes> {
     body: T;
@@ -16,6 +26,7 @@ type ResourceOptions = {
     };
     url: string;
     rejectUnauthorized: boolean;
+    rateLimits?: RateLimitOptions;
 };
 type DefaultRequestOptions = {
     body?: FormData | Record<string, unknown>;
@@ -34,6 +45,7 @@ type RequestOptions = {
     body?: string | FormData;
     asStream?: boolean;
     signal?: AbortSignal;
+    rateLimiters?: Record<string, ReturnType<typeof RateLimit>>;
 };
 interface RequesterType {
     get<T extends ResponseBodyTypes>(endpoint: string, options?: DefaultRequestOptions): Promise<FormattedResponse<T>>;
@@ -42,12 +54,14 @@ interface RequesterType {
     patch<T extends ResponseBodyTypes>(endpoint: string, options?: DefaultRequestOptions): Promise<FormattedResponse<T>>;
     delete<T extends ResponseBodyTypes>(endpoint: string, options?: DefaultRequestOptions): Promise<FormattedResponse<T>>;
 }
+type RequestHandlerFn<T extends ResponseBodyTypes = ResponseBodyTypes> = (endpoint: string, options?: Record<string, unknown>) => Promise<FormattedResponse<T>>;
 declare function formatQuery(params?: Record<string, unknown>): string;
 type OptionsHandlerFn = (serviceOptions: ResourceOptions, requestOptions: RequestOptions) => Promise<RequestOptions>;
 declare function defaultOptionsHandler(resourceOptions: ResourceOptions, { body, searchParams, sudo, signal, asStream, method, }?: DefaultRequestOptions): Promise<RequestOptions>;
-type RequestHandlerFn<T extends ResponseBodyTypes = ResponseBodyTypes> = (endpoint: string, options?: Record<string, unknown>) => Promise<FormattedResponse<T>>;
+declare function createRateLimiters(rateLimitOptions?: RateLimitOptions): RateLimiters;
 declare function createRequesterFn(optionsHandler: OptionsHandlerFn, requestHandler: RequestHandlerFn): (serviceOptions: ResourceOptions) => RequesterType;
 declare function presetResourceArguments<T extends Record<string, Constructable>>(resources: T, customConfig?: Record<string, unknown>): T;
+declare function getMatchingRateLimiter(endpoint: string, rateLimiters?: RateLimiters, method?: string): () => Promise<void>;
 
 interface RootResourceOptions<C> {
     requesterFn?: (resourceOptions: ResourceOptions) => RequesterType;
@@ -59,6 +73,7 @@ interface RootResourceOptions<C> {
     sudo?: string | number;
     profileToken?: string;
     profileMode?: 'execution' | 'memory';
+    rateLimits?: RateLimitOptions;
 }
 type GitlabToken = string | (() => Promise<string>);
 interface BaseRequestOptionsWithOAuthToken<C> extends RootResourceOptions<C> {
@@ -83,7 +98,7 @@ declare class BaseResource<C extends boolean = false> {
     };
     readonly camelize: C | undefined;
     readonly rejectUnauthorized: boolean;
-    constructor({ sudo, profileToken, camelize, requesterFn, profileMode, host, prefixUrl, rejectUnauthorized, queryTimeout, ...tokens }: BaseResourceOptions<C>);
+    constructor({ sudo, profileToken, camelize, requesterFn, profileMode, host, prefixUrl, rejectUnauthorized, queryTimeout, rateLimits, ...tokens }: BaseResourceOptions<C>);
 }
 
-export { BaseRequestOptionsWithAccessToken, BaseRequestOptionsWithJobToken, BaseRequestOptionsWithOAuthToken, BaseResource, BaseResourceOptions, Constructable, DefaultRequestOptions, FormattedResponse, GitlabToken, OptionsHandlerFn, RequestHandlerFn, RequestOptions, RequesterType, ResourceOptions, ResponseBodyTypes, RootResourceOptions, createRequesterFn, defaultOptionsHandler, formatQuery, presetResourceArguments };
+export { BaseRequestOptionsWithAccessToken, BaseRequestOptionsWithJobToken, BaseRequestOptionsWithOAuthToken, BaseResource, BaseResourceOptions, Constructable, DefaultRequestOptions, FormattedResponse, GitlabToken, OptionsHandlerFn, RateLimitOptions, RateLimiters, RequestHandlerFn, RequestOptions, RequesterType, ResourceOptions, ResponseBodyTypes, RootResourceOptions, createRateLimiters, createRequesterFn, defaultOptionsHandler, formatQuery, getMatchingRateLimiter, presetResourceArguments };

package/dist/index.d.ts

@@ -1,3 +1,13 @@
+import { RateLimit } from 'async-sema';
+
+type RateLimiters = Record<string, ReturnType<typeof RateLimit> | {
+    method: string;
+    limit: ReturnType<typeof RateLimit>;
+}>;
+type RateLimitOptions = Record<string, number | {
+    method: string;
+    limit: number;
+}>;
 type ResponseBodyTypes = Record<string, unknown> | Record<string, unknown>[] | ReadableStream | Blob | string | string[] | number | void | null;
 interface FormattedResponse<T extends ResponseBodyTypes = ResponseBodyTypes> {
     body: T;
@@ -16,6 +26,7 @@ type ResourceOptions = {
     };
     url: string;
     rejectUnauthorized: boolean;
+    rateLimits?: RateLimitOptions;
 };
 type DefaultRequestOptions = {
     body?: FormData | Record<string, unknown>;
@@ -34,6 +45,7 @@ type RequestOptions = {
     body?: string | FormData;
     asStream?: boolean;
     signal?: AbortSignal;
+    rateLimiters?: Record<string, ReturnType<typeof RateLimit>>;
 };
 interface RequesterType {
     get<T extends ResponseBodyTypes>(endpoint: string, options?: DefaultRequestOptions): Promise<FormattedResponse<T>>;
@@ -42,12 +54,14 @@ interface RequesterType {
     patch<T extends ResponseBodyTypes>(endpoint: string, options?: DefaultRequestOptions): Promise<FormattedResponse<T>>;
     delete<T extends ResponseBodyTypes>(endpoint: string, options?: DefaultRequestOptions): Promise<FormattedResponse<T>>;
 }
+type RequestHandlerFn<T extends ResponseBodyTypes = ResponseBodyTypes> = (endpoint: string, options?: Record<string, unknown>) => Promise<FormattedResponse<T>>;
 declare function formatQuery(params?: Record<string, unknown>): string;
 type OptionsHandlerFn = (serviceOptions: ResourceOptions, requestOptions: RequestOptions) => Promise<RequestOptions>;
 declare function defaultOptionsHandler(resourceOptions: ResourceOptions, { body, searchParams, sudo, signal, asStream, method, }?: DefaultRequestOptions): Promise<RequestOptions>;
-type RequestHandlerFn<T extends ResponseBodyTypes = ResponseBodyTypes> = (endpoint: string, options?: Record<string, unknown>) => Promise<FormattedResponse<T>>;
+declare function createRateLimiters(rateLimitOptions?: RateLimitOptions): RateLimiters;
 declare function createRequesterFn(optionsHandler: OptionsHandlerFn, requestHandler: RequestHandlerFn): (serviceOptions: ResourceOptions) => RequesterType;
 declare function presetResourceArguments<T extends Record<string, Constructable>>(resources: T, customConfig?: Record<string, unknown>): T;
+declare function getMatchingRateLimiter(endpoint: string, rateLimiters?: RateLimiters, method?: string): () => Promise<void>;
 
 interface RootResourceOptions<C> {
     requesterFn?: (resourceOptions: ResourceOptions) => RequesterType;
@@ -59,6 +73,7 @@ interface RootResourceOptions<C> {
     sudo?: string | number;
     profileToken?: string;
     profileMode?: 'execution' | 'memory';
+    rateLimits?: RateLimitOptions;
 }
 type GitlabToken = string | (() => Promise<string>);
 interface BaseRequestOptionsWithOAuthToken<C> extends RootResourceOptions<C> {
@@ -83,7 +98,7 @@ declare class BaseResource<C extends boolean = false> {
     };
     readonly camelize: C | undefined;
     readonly rejectUnauthorized: boolean;
-    constructor({ sudo, profileToken, camelize, requesterFn, profileMode, host, prefixUrl, rejectUnauthorized, queryTimeout, ...tokens }: BaseResourceOptions<C>);
+    constructor({ sudo, profileToken, camelize, requesterFn, profileMode, host, prefixUrl, rejectUnauthorized, queryTimeout, rateLimits, ...tokens }: BaseResourceOptions<C>);
 }
 
-export { BaseRequestOptionsWithAccessToken, BaseRequestOptionsWithJobToken, BaseRequestOptionsWithOAuthToken, BaseResource, BaseResourceOptions, Constructable, DefaultRequestOptions, FormattedResponse, GitlabToken, OptionsHandlerFn, RequestHandlerFn, RequestOptions, RequesterType, ResourceOptions, ResponseBodyTypes, RootResourceOptions, createRequesterFn, defaultOptionsHandler, formatQuery, presetResourceArguments };
+export { BaseRequestOptionsWithAccessToken, BaseRequestOptionsWithJobToken, BaseRequestOptionsWithOAuthToken, BaseResource, BaseResourceOptions, Constructable, DefaultRequestOptions, FormattedResponse, GitlabToken, OptionsHandlerFn, RateLimitOptions, RateLimiters, RequestHandlerFn, RequestOptions, RequesterType, ResourceOptions, ResponseBodyTypes, RootResourceOptions, createRateLimiters, createRequesterFn, defaultOptionsHandler, formatQuery, getMatchingRateLimiter, presetResourceArguments };

package/dist/index.js

@@ -2,6 +2,12 @@
 
 var qs = require('qs');
 var xcase = require('xcase');
+var asyncSema = require('async-sema');
+var micromatch = require('micromatch');
+
+function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
+
+var micromatch__default = /*#__PURE__*/_interopDefault(micromatch);
 
 // src/RequesterUtils.ts
 function formatQuery(params = {}) {
@@ -45,10 +51,24 @@ async function defaultOptionsHandler(resourceOptions, {
     defaultOptions.searchParams = q;
   return Promise.resolve(defaultOptions);
 }
+function createRateLimiters(rateLimitOptions = {}) {
+  const rateLimiters = {};
+  Object.entries(rateLimitOptions).forEach(([key, config]) => {
+    if (typeof config === "number")
+      rateLimiters[key] = asyncSema.RateLimit(config, { timeUnit: 6e4 });
+    else
+      rateLimiters[key] = {
+        method: config.method.toUpperCase(),
+        limit: asyncSema.RateLimit(config.limit, { timeUnit: 6e4 })
+      };
+  });
+  return rateLimiters;
+}
 function createRequesterFn(optionsHandler, requestHandler) {
   const methods = ["get", "post", "put", "patch", "delete"];
   return (serviceOptions) => {
     const requester = {};
+    const rateLimiters = createRateLimiters(serviceOptions.rateLimits);
     methods.forEach((m) => {
       requester[m] = async (endpoint, options) => {
         const defaultRequestOptions = await defaultOptionsHandler(serviceOptions, {
@@ -56,13 +76,13 @@ function createRequesterFn(optionsHandler, requestHandler) {
           method: m.toUpperCase()
         });
         const requestOptions = await optionsHandler(serviceOptions, defaultRequestOptions);
-        return requestHandler(endpoint, requestOptions);
+        return requestHandler(endpoint, { ...requestOptions, rateLimiters });
       };
     });
     return requester;
   };
 }
-function extendClass(Base, customConfig = {}) {
+function extendClass(Base, customConfig) {
   return class extends Base {
     constructor(...options) {
       const [config, ...opts] = options;
@@ -77,11 +97,58 @@ function presetResourceArguments(resources, customConfig = {}) {
   });
   return updated;
 }
+function getMatchingRateLimiter(endpoint, rateLimiters = {}, method = "GET") {
+  const sortedEndpoints = Object.keys(rateLimiters).sort().reverse();
+  const match = sortedEndpoints.find((ep) => micromatch__default.default.isMatch(endpoint, ep));
+  const rateLimitConfig = match && rateLimiters[match];
+  if (rateLimitConfig && typeof rateLimitConfig !== "object") {
+    return rateLimitConfig;
+  }
+  if (rateLimitConfig && rateLimitConfig.method.toUpperCase() === method.toUpperCase()) {
+    return rateLimitConfig.limit;
+  }
+  return asyncSema.RateLimit(3e3, { timeUnit: 6e4 });
+}
 
 // src/BaseResource.ts
 function getDynamicToken(tokenArgument) {
   return tokenArgument instanceof Function ? tokenArgument() : Promise.resolve(tokenArgument);
 }
+var DEFAULT_RATE_LIMITS = Object.freeze({
+  // Default rate limit
+  "**": 3e3,
+  // Import/Export
+  "projects/import": 6,
+  "projects/*/export": 6,
+  "projects/*/download": 1,
+  "groups/import": 6,
+  "groups/*/export": 6,
+  "groups/*/download": 1,
+  // Note creation
+  "projects/*/issues/*/notes": {
+    method: "post",
+    limit: 300
+  },
+  "projects/*/snippets/*/notes": {
+    method: "post",
+    limit: 300
+  },
+  "projects/*/merge_requests/*/notes": {
+    method: "post",
+    limit: 300
+  },
+  "groups/*/epics/*/notes": {
+    method: "post",
+    limit: 300
+  },
+  // Repositories - get file archive
+  "projects/*/repository/archive*": 5,
+  // Project Jobs
+  "projects/*/jobs": 600,
+  // Member deletion
+  "projects/*/members": 60,
+  "groups/*/members": 60
+});
 var BaseResource = class {
   url;
   requester;
@@ -100,6 +167,7 @@ var BaseResource = class {
     prefixUrl = "",
     rejectUnauthorized = true,
     queryTimeout = 3e5,
+    rateLimits = DEFAULT_RATE_LIMITS,
     ...tokens
   }) {
     if (!requesterFn)
@@ -128,12 +196,14 @@ var BaseResource = class {
     }
     if (sudo)
       this.headers.Sudo = `${sudo}`;
-    this.requester = requesterFn({ ...this });
+    this.requester = requesterFn({ ...this, rateLimits });
   }
 };
 
 exports.BaseResource = BaseResource;
+exports.createRateLimiters = createRateLimiters;
 exports.createRequesterFn = createRequesterFn;
 exports.defaultOptionsHandler = defaultOptionsHandler;
 exports.formatQuery = formatQuery;
+exports.getMatchingRateLimiter = getMatchingRateLimiter;
 exports.presetResourceArguments = presetResourceArguments;

package/dist/index.mjs

@@ -1,5 +1,7 @@
 import { stringify } from 'qs';
 import { decamelizeKeys } from 'xcase';
+import { RateLimit } from 'async-sema';
+import micromatch from 'micromatch';
 
 // src/RequesterUtils.ts
 function formatQuery(params = {}) {
@@ -43,10 +45,24 @@ async function defaultOptionsHandler(resourceOptions, {
     defaultOptions.searchParams = q;
   return Promise.resolve(defaultOptions);
 }
+function createRateLimiters(rateLimitOptions = {}) {
+  const rateLimiters = {};
+  Object.entries(rateLimitOptions).forEach(([key, config]) => {
+    if (typeof config === "number")
+      rateLimiters[key] = RateLimit(config, { timeUnit: 6e4 });
+    else
+      rateLimiters[key] = {
+        method: config.method.toUpperCase(),
+        limit: RateLimit(config.limit, { timeUnit: 6e4 })
+      };
+  });
+  return rateLimiters;
+}
 function createRequesterFn(optionsHandler, requestHandler) {
   const methods = ["get", "post", "put", "patch", "delete"];
   return (serviceOptions) => {
     const requester = {};
+    const rateLimiters = createRateLimiters(serviceOptions.rateLimits);
     methods.forEach((m) => {
       requester[m] = async (endpoint, options) => {
         const defaultRequestOptions = await defaultOptionsHandler(serviceOptions, {
@@ -54,13 +70,13 @@ function createRequesterFn(optionsHandler, requestHandler) {
           method: m.toUpperCase()
         });
         const requestOptions = await optionsHandler(serviceOptions, defaultRequestOptions);
-        return requestHandler(endpoint, requestOptions);
+        return requestHandler(endpoint, { ...requestOptions, rateLimiters });
       };
     });
     return requester;
   };
 }
-function extendClass(Base, customConfig = {}) {
+function extendClass(Base, customConfig) {
   return class extends Base {
     constructor(...options) {
       const [config, ...opts] = options;
@@ -75,11 +91,58 @@ function presetResourceArguments(resources, customConfig = {}) {
   });
   return updated;
 }
+function getMatchingRateLimiter(endpoint, rateLimiters = {}, method = "GET") {
+  const sortedEndpoints = Object.keys(rateLimiters).sort().reverse();
+  const match = sortedEndpoints.find((ep) => micromatch.isMatch(endpoint, ep));
+  const rateLimitConfig = match && rateLimiters[match];
+  if (rateLimitConfig && typeof rateLimitConfig !== "object") {
+    return rateLimitConfig;
+  }
+  if (rateLimitConfig && rateLimitConfig.method.toUpperCase() === method.toUpperCase()) {
+    return rateLimitConfig.limit;
+  }
+  return RateLimit(3e3, { timeUnit: 6e4 });
+}
 
 // src/BaseResource.ts
 function getDynamicToken(tokenArgument) {
   return tokenArgument instanceof Function ? tokenArgument() : Promise.resolve(tokenArgument);
 }
+var DEFAULT_RATE_LIMITS = Object.freeze({
+  // Default rate limit
+  "**": 3e3,
+  // Import/Export
+  "projects/import": 6,
+  "projects/*/export": 6,
+  "projects/*/download": 1,
+  "groups/import": 6,
+  "groups/*/export": 6,
+  "groups/*/download": 1,
+  // Note creation
+  "projects/*/issues/*/notes": {
+    method: "post",
+    limit: 300
+  },
+  "projects/*/snippets/*/notes": {
+    method: "post",
+    limit: 300
+  },
+  "projects/*/merge_requests/*/notes": {
+    method: "post",
+    limit: 300
+  },
+  "groups/*/epics/*/notes": {
+    method: "post",
+    limit: 300
+  },
+  // Repositories - get file archive
+  "projects/*/repository/archive*": 5,
+  // Project Jobs
+  "projects/*/jobs": 600,
+  // Member deletion
+  "projects/*/members": 60,
+  "groups/*/members": 60
+});
 var BaseResource = class {
   url;
   requester;
@@ -98,6 +161,7 @@ var BaseResource = class {
     prefixUrl = "",
     rejectUnauthorized = true,
     queryTimeout = 3e5,
+    rateLimits = DEFAULT_RATE_LIMITS,
     ...tokens
   }) {
     if (!requesterFn)
@@ -126,8 +190,8 @@ var BaseResource = class {
     }
     if (sudo)
       this.headers.Sudo = `${sudo}`;
-    this.requester = requesterFn({ ...this });
+    this.requester = requesterFn({ ...this, rateLimits });
   }
 };
 
-export { BaseResource, createRequesterFn, defaultOptionsHandler, formatQuery, presetResourceArguments };
+export { BaseResource, createRateLimiters, createRequesterFn, defaultOptionsHandler, formatQuery, getMatchingRateLimiter, presetResourceArguments };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gitbeaker/requester-utils",
-  "version": "39.17.0",
+  "version": "39.19.0",
   "description": "Utility functions for requester implementatons used in @gitbeaker",
   "license": "MIT",
   "engines": {
@@ -51,6 +51,8 @@
     "release": "auto shipit"
   },
   "dependencies": {
+    "async-sema": "^3.1.1",
+    "micromatch": "^4.0.5",
     "qs": "^6.11.2",
     "xcase": "^2.0.1"
   },
@@ -59,5 +61,5 @@
     "tsup": "^7.1.0",
     "typescript": "^5.1.6"
   },
-  "gitHead": "604085015c962cdcbe542b96d717d64d2051ca3c"
+  "gitHead": "b6ac7ead7ee50ced0f99f66ea0c212d2afcba3c2"
 }