faros-airbyte-common 0.11.4 → 0.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/common/bucketing.d.ts +19 -0
- package/lib/common/bucketing.js +51 -0
- package/lib/common/bucketing.js.map +1 -0
- package/lib/common/index.d.ts +1 -9
- package/lib/common/index.js +6 -24
- package/lib/common/index.js.map +1 -1
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/lib/vanta/index.d.ts +1 -0
- package/lib/vanta/index.js +18 -0
- package/lib/vanta/index.js.map +1 -0
- package/lib/vanta/types.d.ts +73 -0
- package/lib/vanta/types.js +3 -0
- package/lib/vanta/types.js.map +1 -0
- package/package.json +5 -1
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface BucketExecutionState {
|
|
2
|
+
__bucket_execution_state?: {
|
|
3
|
+
last_executed_bucket_id: number;
|
|
4
|
+
};
|
|
5
|
+
[key: string]: any;
|
|
6
|
+
}
|
|
7
|
+
export interface RoundRobinConfig {
|
|
8
|
+
round_robin_bucket_execution?: boolean;
|
|
9
|
+
bucket_id?: number;
|
|
10
|
+
bucket_total?: number;
|
|
11
|
+
[key: string]: any;
|
|
12
|
+
}
|
|
13
|
+
export declare function bucket(key: string, data: string, bucketTotal: number): number;
|
|
14
|
+
export declare function validateBucketingConfig(bucketId?: number, bucketTotal?: number): void;
|
|
15
|
+
export declare function nextBucketId(config: RoundRobinConfig, state?: BucketExecutionState): number;
|
|
16
|
+
export declare function applyRoundRobinBucketing(config: RoundRobinConfig, state?: BucketExecutionState, logger?: (message: string) => void): {
|
|
17
|
+
config: RoundRobinConfig;
|
|
18
|
+
state: BucketExecutionState;
|
|
19
|
+
};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.bucket = bucket;
|
|
7
|
+
exports.validateBucketingConfig = validateBucketingConfig;
|
|
8
|
+
exports.nextBucketId = nextBucketId;
|
|
9
|
+
exports.applyRoundRobinBucketing = applyRoundRobinBucketing;
|
|
10
|
+
const crypto_1 = require("crypto");
|
|
11
|
+
const verror_1 = __importDefault(require("verror"));
|
|
12
|
+
function bucket(key, data, bucketTotal) {
|
|
13
|
+
const md5 = (0, crypto_1.createHmac)('md5', key);
|
|
14
|
+
md5.update(data);
|
|
15
|
+
const hex = md5.digest('hex').substring(0, 8);
|
|
16
|
+
return (parseInt(hex, 16) % bucketTotal) + 1; // 1-index for readability
|
|
17
|
+
}
|
|
18
|
+
function validateBucketingConfig(bucketId = 1, bucketTotal = 1) {
|
|
19
|
+
if (bucketTotal < 1) {
|
|
20
|
+
throw new verror_1.default('bucket_total must be a positive integer');
|
|
21
|
+
}
|
|
22
|
+
if (bucketId < 1 || bucketId > bucketTotal) {
|
|
23
|
+
throw new verror_1.default(`bucket_id must be between 1 and ${bucketTotal}`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function nextBucketId(config, state) {
|
|
27
|
+
var _a, _b, _c;
|
|
28
|
+
const bucketTotal = (_a = config.bucket_total) !== null && _a !== void 0 ? _a : 1;
|
|
29
|
+
const lastExecutedBucketId = (_c = (_b = state === null || state === void 0 ? void 0 : state.__bucket_execution_state) === null || _b === void 0 ? void 0 : _b.last_executed_bucket_id) !== null && _c !== void 0 ? _c : bucketTotal;
|
|
30
|
+
return (lastExecutedBucketId % bucketTotal) + 1;
|
|
31
|
+
}
|
|
32
|
+
function applyRoundRobinBucketing(config, state, logger) {
|
|
33
|
+
if (!config.round_robin_bucket_execution) {
|
|
34
|
+
return { config, state };
|
|
35
|
+
}
|
|
36
|
+
const next = nextBucketId(config, state);
|
|
37
|
+
logger === null || logger === void 0 ? void 0 : logger(`Using round robin bucket execution. Bucket id: ${next}`);
|
|
38
|
+
return {
|
|
39
|
+
config: {
|
|
40
|
+
...config,
|
|
41
|
+
bucket_id: next,
|
|
42
|
+
},
|
|
43
|
+
state: {
|
|
44
|
+
...state,
|
|
45
|
+
__bucket_execution_state: {
|
|
46
|
+
last_executed_bucket_id: next,
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=bucketing.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bucketing.js","sourceRoot":"","sources":["../../src/common/bucketing.ts"],"names":[],"mappings":";;;;;AAiBA,wBAKC;AAED,0DAUC;AAED,oCASC;AAED,4DAwBC;AAvED,mCAAkC;AAClC,oDAA4B;AAgB5B,SAAgB,MAAM,CAAC,GAAW,EAAE,IAAY,EAAE,WAAmB;IACnE,MAAM,GAAG,GAAG,IAAA,mBAAU,EAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACnC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,0BAA0B;AAC1E,CAAC;AAED,SAAgB,uBAAuB,CACrC,WAAmB,CAAC,EACpB,cAAsB,CAAC;IAEvB,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,gBAAM,CAAC,yCAAyC,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,WAAW,EAAE,CAAC;QAC3C,MAAM,IAAI,gBAAM,CAAC,mCAAmC,WAAW,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,SAAgB,YAAY,CAC1B,MAAwB,EACxB,KAA4B;;IAE5B,MAAM,WAAW,GAAG,MAAA,MAAM,CAAC,YAAY,mCAAI,CAAC,CAAC;IAC7C,MAAM,oBAAoB,GACxB,MAAA,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,wBAAwB,0CAAE,uBAAuB,mCAAI,WAAW,CAAC;IAE1E,OAAO,CAAC,oBAAoB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;AAClD,CAAC;AAED,SAAgB,wBAAwB,CACtC,MAAwB,EACxB,KAA4B,EAC5B,MAAkC;IAElC,IAAI,CAAC,MAAM,CAAC,4BAA4B,EAAE,CAAC;QACzC,OAAO,EAAC,MAAM,EAAE,KAAK,EAAC,CAAC;IACzB,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACzC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,kDAAkD,IAAI,EAAE,CAAC,CAAC;IAEnE,OAAO;QACL,MAAM,EAAE;YACN,GAAG,MAAM;YACT,SAAS,EAAE,IAAI;SAChB;QACD,KAAK,EAAE;YACL,GAAG,KAAK;YACR,wBAAwB,EAAE;gBACxB,uBAAuB,EAAE,IAAI;aAC9B;SACF;KACF,CAAC;AACJ,CAAC"}
|
package/lib/common/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { FarosClient } from 'faros-js-client';
|
|
2
|
+
export { bucket, validateBucketingConfig, nextBucketId, applyRoundRobinBucketing, } from './bucketing';
|
|
2
3
|
export interface FileDiff {
|
|
3
4
|
deletions: number;
|
|
4
5
|
additions: number;
|
|
@@ -7,15 +8,6 @@ export interface FileDiff {
|
|
|
7
8
|
deleted?: boolean;
|
|
8
9
|
new?: boolean;
|
|
9
10
|
}
|
|
10
|
-
export declare function bucket(key: string, data: string, bucketTotal: number): number;
|
|
11
|
-
export declare function validateBucketingConfig(bucketId?: number, bucketTotal?: number): void;
|
|
12
|
-
export declare function nextBucketId(config: {
|
|
13
|
-
bucket_total?: number;
|
|
14
|
-
}, state?: {
|
|
15
|
-
__bucket_execution_state?: {
|
|
16
|
-
last_executed_bucket_id?: number;
|
|
17
|
-
};
|
|
18
|
-
}): number;
|
|
19
11
|
export declare function normalizeString(str: string): string;
|
|
20
12
|
export declare function calculateDateRange(options: {
|
|
21
13
|
start_date?: string;
|
package/lib/common/index.js
CHANGED
|
@@ -3,40 +3,22 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.bucket =
|
|
7
|
-
exports.validateBucketingConfig = validateBucketingConfig;
|
|
8
|
-
exports.nextBucketId = nextBucketId;
|
|
6
|
+
exports.applyRoundRobinBucketing = exports.nextBucketId = exports.validateBucketingConfig = exports.bucket = void 0;
|
|
9
7
|
exports.normalizeString = normalizeString;
|
|
10
8
|
exports.calculateDateRange = calculateDateRange;
|
|
11
9
|
exports.collectReposByOrg = collectReposByOrg;
|
|
12
10
|
exports.getFarosOptions = getFarosOptions;
|
|
13
|
-
const crypto_1 = require("crypto");
|
|
14
11
|
const faros_js_client_1 = require("faros-js-client");
|
|
15
12
|
const fs_1 = __importDefault(require("fs"));
|
|
16
13
|
const lodash_1 = require("lodash");
|
|
17
14
|
const luxon_1 = require("luxon");
|
|
18
15
|
const path_1 = __importDefault(require("path"));
|
|
19
16
|
const verror_1 = require("verror");
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
}
|
|
26
|
-
function validateBucketingConfig(bucketId = 1, bucketTotal = 1) {
|
|
27
|
-
if (bucketTotal < 1) {
|
|
28
|
-
throw new verror_1.VError('bucket_total must be a positive integer');
|
|
29
|
-
}
|
|
30
|
-
if (bucketId < 1 || bucketId > bucketTotal) {
|
|
31
|
-
throw new verror_1.VError(`bucket_id must be between 1 and ${bucketTotal}`);
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
function nextBucketId(config, state) {
|
|
35
|
-
var _a, _b, _c;
|
|
36
|
-
const bucketTotal = (_a = config.bucket_total) !== null && _a !== void 0 ? _a : 1;
|
|
37
|
-
const lastExecutedBucketId = (_c = (_b = state === null || state === void 0 ? void 0 : state.__bucket_execution_state) === null || _b === void 0 ? void 0 : _b.last_executed_bucket_id) !== null && _c !== void 0 ? _c : bucketTotal;
|
|
38
|
-
return (lastExecutedBucketId % bucketTotal) + 1;
|
|
39
|
-
}
|
|
17
|
+
var bucketing_1 = require("./bucketing");
|
|
18
|
+
Object.defineProperty(exports, "bucket", { enumerable: true, get: function () { return bucketing_1.bucket; } });
|
|
19
|
+
Object.defineProperty(exports, "validateBucketingConfig", { enumerable: true, get: function () { return bucketing_1.validateBucketingConfig; } });
|
|
20
|
+
Object.defineProperty(exports, "nextBucketId", { enumerable: true, get: function () { return bucketing_1.nextBucketId; } });
|
|
21
|
+
Object.defineProperty(exports, "applyRoundRobinBucketing", { enumerable: true, get: function () { return bucketing_1.applyRoundRobinBucketing; } });
|
|
40
22
|
function normalizeString(str) {
|
|
41
23
|
return str.replace(/\s/g, '').toLowerCase();
|
|
42
24
|
}
|
package/lib/common/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/common/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/common/index.ts"],"names":[],"mappings":";;;;;;AAwBA,0CAEC;AAED,gDA0CC;AAED,8CAkBC;AAwBD,0CA2CC;AA7JD,qDAA8D;AAC9D,4CAAoB;AACpB,mCAA+B;AAC/B,iCAA+B;AAC/B,gDAAwB;AACxB,mCAA8B;AAE9B,yCAKqB;AAJnB,mGAAA,MAAM,OAAA;AACN,oHAAA,uBAAuB,OAAA;AACvB,yGAAA,YAAY,OAAA;AACZ,qHAAA,wBAAwB,OAAA;AAa1B,SAAgB,eAAe,CAAC,GAAW;IACzC,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AAC9C,CAAC;AAED,SAAgB,kBAAkB,CAAC,OAKlC;IACC,MAAM,EAAC,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAC,GAAG,OAAO,CAAC;IAE5D,IAAI,SAAe,CAAC;IACpB,IAAI,OAAa,CAAC;IAElB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,CAAC,2CAA2C,CAAC,CAAC;QACpD,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,UAAU,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,CAAC,kEAAkE,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,SAAS,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,CAAC,4DAA4D,CAAC,CAAC;YACrE,SAAS,GAAG,gBAAQ,CAAC,UAAU,CAAC,OAAO,CAAC;iBACrC,KAAK,CAAC,EAAC,IAAI,EAAE,WAAW,EAAC,CAAC;iBAC1B,QAAQ,EAAE,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,eAAe,SAAS,uBAAuB,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,CAAC,0BAA0B,SAAS,OAAO,OAAO,EAAE,CAAC,CAAC;IAE5D,OAAO,EAAC,SAAS,EAAE,OAAO,EAAC,CAAC;AAC9B,CAAC;AAED,SAAgB,iBAAiB,CAC/B,UAAoC,EACpC,KAA4B;IAE5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,eAAM,CACd,4BAA4B,IAAI,gDAAgD,CACjF,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,IAAA,gBAAO,EAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,aAAa,GAAG,IAAA,gBAAO,EAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACtC,CAAC;QACD,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED,MAAM,qBAAqB,GAAG,CAAC,QAAgB,EAAE,EAAE,CACjD,YAAE,CAAC,YAAY,CACb,cAAI,CAAC,IAAI,CACP,SAAS,EACT,IAAI,EACJ,IAAI,EACJ,WAAW,EACX,QAAQ,EACR,SAAS,EACT,QAAQ,CACT,EACD,MAAM,CACP,CAAC;AAEJ,MAAM,4BAA4B,GAAG,qBAAqB,CACxD,kCAAkC,CACnC,CAAC;AAEF,MAAM,4BAA4B,GAAG,qBAAqB,CACxD,kCAAkC,CACnC,CAAC;AAEK,KAAK,UAAU,eAAe,CACnC,WAAmC,EACnC,MAAc,EACd,WAAwB,EACxB,KAAa;IAKb,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,IAAI,KAAa,CAAC;IAClB,QAAQ,WAAW,EAAE,CAAC;QACpB,KAAK,OAAO;YACV,KAAK,GAAG,4BAA4B,CAAC;YACrC,MAAM;QACR,KAAK,YAAY;YACf,KAAK,GAAG,4BAA4B,CAAC;YACrC,MAAM;QACR;YACE,MAAM,IAAI,KAAK,CAAC,+BAA+B,WAAW,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,CAAC,YAAY,CACnC,KAAK,EACL,KAAK,EACL,IAAI,EACJ,kCAAgB,EAChB,IAAI,GAAG,CACL,MAAM,CAAC,OAAO,CAAC;QACb,MAAM;KACP,CAAC,CACH,CACF,CAAC;IACF,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,sBAAsB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,OAAO,CAAC,iBAAiB,KAAK,UAAU,EAAE,CAAC;YAC7C,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;aAAM,IAAI,OAAO,CAAC,iBAAiB,KAAK,UAAU,EAAE,CAAC;YACpD,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,EAAC,QAAQ,EAAE,QAAQ,EAAC,CAAC;AAC9B,CAAC;AAED,MAAM,sBAAsB,GAAG,CAC7B,WAAmC,EACnC,OAAY,EACJ,EAAE;;IACV,QAAQ,WAAW,EAAE,CAAC;QACpB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,KAAK,GAAG,MAAA,OAAO,CAAC,KAAK,0CAAE,GAAG,CAAC;YACjC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,IAAI,GAAG,MAAA,OAAO,CAAC,UAAU,0CAAE,IAAI,CAAC;YACtC,MAAM,GAAG,GAAG,MAAA,MAAA,OAAO,CAAC,UAAU,0CAAE,YAAY,0CAAE,GAAG,CAAC;YAClD,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;YACD,OAAO,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC,CAAC"}
|