airbyte-faros-destination 0.23.39 → 0.23.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/converters/claude/configured_spend_limits.d.ts +8 -0
- package/lib/converters/claude/configured_spend_limits.js +87 -0
- package/lib/converters/claude/configured_spend_limits.js.map +1 -0
- package/lib/converters/claude/spend_limits.d.ts +1 -0
- package/lib/converters/claude/spend_limits.js +1 -0
- package/lib/converters/claude/spend_limits.js.map +1 -1
- package/lib/converters/codex/usage_limits.d.ts +12 -0
- package/lib/converters/codex/usage_limits.js +175 -0
- package/lib/converters/codex/usage_limits.js.map +1 -0
- package/lib/converters/converter.d.ts +3 -0
- package/lib/converters/converter.js +10 -0
- package/lib/converters/converter.js.map +1 -1
- package/lib/converters/faros-feeds/faros_feed.js +2 -1
- package/lib/converters/faros-feeds/faros_feed.js.map +1 -1
- package/lib/destination.d.ts +9 -0
- package/lib/destination.js +52 -7
- package/lib/destination.js.map +1 -1
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/package.json +3 -3
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { AirbyteRecord } from 'faros-airbyte-cdk';
|
|
2
|
+
import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
|
|
3
|
+
import { ClaudePlatformConverter } from './platform_common';
|
|
4
|
+
export declare class ConfiguredSpendLimits extends ClaudePlatformConverter {
|
|
5
|
+
readonly destinationModels: ReadonlyArray<DestinationModel>;
|
|
6
|
+
id(record: AirbyteRecord): any;
|
|
7
|
+
convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ConfiguredSpendLimits = void 0;
|
|
4
|
+
const luxon_1 = require("luxon");
|
|
5
|
+
const ai_1 = require("../common/ai");
|
|
6
|
+
const platform_common_1 = require("./platform_common");
|
|
7
|
+
const spend_limits_1 = require("./spend_limits");
|
|
8
|
+
function scopeMapping(scope) {
|
|
9
|
+
switch (scope?.type) {
|
|
10
|
+
case 'organization':
|
|
11
|
+
return { category: 'Account', detail: null, pooled: true };
|
|
12
|
+
case 'seat_tier':
|
|
13
|
+
return { category: 'Plan', detail: scope.seat_tier ?? null, pooled: false };
|
|
14
|
+
case 'rbac_group':
|
|
15
|
+
return { category: 'Group', detail: null, pooled: false };
|
|
16
|
+
case 'organization_service':
|
|
17
|
+
return { category: 'Custom', detail: scope.service ?? null, pooled: true };
|
|
18
|
+
default:
|
|
19
|
+
return { category: 'Custom', detail: scope?.type ?? null, pooled: null };
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
class ConfiguredSpendLimits extends platform_common_1.ClaudePlatformConverter {
|
|
23
|
+
constructor() {
|
|
24
|
+
super(...arguments);
|
|
25
|
+
this.destinationModels = [
|
|
26
|
+
ai_1.AI_ACCOUNT_MODEL,
|
|
27
|
+
ai_1.AI_GROUP_MODEL,
|
|
28
|
+
ai_1.AI_LIMIT_MODEL,
|
|
29
|
+
];
|
|
30
|
+
}
|
|
31
|
+
id(record) {
|
|
32
|
+
return record?.record?.data?.id;
|
|
33
|
+
}
|
|
34
|
+
async convert(record, ctx) {
|
|
35
|
+
const item = record.record.data;
|
|
36
|
+
if (!item?.id || item.scope?.type === 'user')
|
|
37
|
+
return [];
|
|
38
|
+
const account = this.account(platform_common_1.ENTERPRISE_ACCOUNT_UID);
|
|
39
|
+
const result = [];
|
|
40
|
+
const accountRecord = this.destinationDimensionRecord(ai_1.AI_ACCOUNT_MODEL, account.uid, account);
|
|
41
|
+
if (accountRecord)
|
|
42
|
+
result.push(accountRecord);
|
|
43
|
+
let group = null;
|
|
44
|
+
if (item.scope.type === 'rbac_group') {
|
|
45
|
+
group = this.accountEntity(item.scope.rbac_group_id, account);
|
|
46
|
+
if (group) {
|
|
47
|
+
// Key fields only, so a null name never clears one set elsewhere.
|
|
48
|
+
const groupRecord = this.destinationDimensionRecord(ai_1.AI_GROUP_MODEL, group.uid, { ...group });
|
|
49
|
+
if (groupRecord)
|
|
50
|
+
result.push(groupRecord);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
const periodStart = item.period === 'monthly'
|
|
54
|
+
? luxon_1.DateTime.fromISO(item.observed_at, { zone: 'utc' }).startOf('month')
|
|
55
|
+
: null;
|
|
56
|
+
const { category, detail, pooled } = scopeMapping(item.scope);
|
|
57
|
+
result.push(this.destinationRecord(ai_1.AI_LIMIT_MODEL, {
|
|
58
|
+
uid: [
|
|
59
|
+
'configured_spend_limit',
|
|
60
|
+
item.id,
|
|
61
|
+
item.period,
|
|
62
|
+
periodStart?.toISODate() ?? '',
|
|
63
|
+
].join(':'),
|
|
64
|
+
source: this.source,
|
|
65
|
+
type: { category: 'Spend', detail: null },
|
|
66
|
+
scope: { category, detail },
|
|
67
|
+
inheritedFrom: null,
|
|
68
|
+
model: null,
|
|
69
|
+
amount: (0, spend_limits_1.majorUnits)(item.amount, item.currency),
|
|
70
|
+
unit: item.currency ?? null,
|
|
71
|
+
consumed: (0, spend_limits_1.majorUnits)(item.period_to_date_spend, item.currency),
|
|
72
|
+
period: item.period ?? null,
|
|
73
|
+
periodStartedAt: periodStart?.toJSDate() ?? null,
|
|
74
|
+
periodEndedAt: periodStart?.plus({ months: 1 }).toJSDate() ?? null,
|
|
75
|
+
pooled,
|
|
76
|
+
enforced: item.is_enabled ?? null,
|
|
77
|
+
account,
|
|
78
|
+
project: null,
|
|
79
|
+
group,
|
|
80
|
+
principal: null,
|
|
81
|
+
credential: null,
|
|
82
|
+
}));
|
|
83
|
+
return result;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
exports.ConfiguredSpendLimits = ConfiguredSpendLimits;
|
|
87
|
+
//# sourceMappingURL=configured_spend_limits.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"configured_spend_limits.js","sourceRoot":"","sources":["../../../src/converters/claude/configured_spend_limits.ts"],"names":[],"mappings":";;;AAEA,iCAA+B;AAE/B,qCAKsB;AAEtB,uDAG2B;AAC3B,iDAA0C;AAU1C,SAAS,YAAY,CACnB,KAA0C;IAE1C,QAAQ,KAAK,EAAE,IAAI,EAAE,CAAC;QACpB,KAAK,cAAc;YACjB,OAAO,EAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAC,CAAC;QAC3D,KAAK,WAAW;YACd,OAAO,EAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI,EAAE,MAAM,EAAE,KAAK,EAAC,CAAC;QAC5E,KAAK,YAAY;YACf,OAAO,EAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAC,CAAC;QAC1D,KAAK,sBAAsB;YACzB,OAAO,EAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,EAAC,CAAC;QAC3E;YACE,OAAO,EAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,EAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,MAAa,qBAAsB,SAAQ,yCAAuB;IAAlE;;QACW,sBAAiB,GAAoC;YAC5D,qBAAgB;YAChB,mBAAc;YACd,mBAAc;SACf,CAAC;IAwEJ,CAAC;IAtEC,EAAE,CAAC,MAAqB;QACtB,OAAQ,MAAM,EAAE,MAAM,EAAE,IAAmC,EAAE,EAAE,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAkC,CAAC;QAC9D,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,IAAI,CAAC,KAAK,EAAE,IAAI,KAAK,MAAM;YAAE,OAAO,EAAE,CAAC;QAExD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,wCAAsB,CAAC,CAAC;QACrD,MAAM,MAAM,GAAwB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,0BAA0B,CACnD,qBAAgB,EAChB,OAAO,CAAC,GAAG,EACX,OAAO,CACR,CAAC;QACF,IAAI,aAAa;YAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE9C,IAAI,KAAK,GAA8B,IAAI,CAAC;QAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACrC,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;YAC9D,IAAI,KAAK,EAAE,CAAC;gBACV,kEAAkE;gBAClE,MAAM,WAAW,GAAG,IAAI,CAAC,0BAA0B,CACjD,mBAAc,EACd,KAAK,CAAC,GAAG,EACT,EAAC,GAAG,KAAK,EAAC,CACX,CAAC;gBACF,IAAI,WAAW;oBAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GACf,IAAI,CAAC,MAAM,KAAK,SAAS;YACvB,CAAC,CAAC,gBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,EAAC,IAAI,EAAE,KAAK,EAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;YACpE,CAAC,CAAC,IAAI,CAAC;QACX,MAAM,EAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAC,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE5D,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,mBAAc,EAAE;YACrC,GAAG,EAAE;gBACH,wBAAwB;gBACxB,IAAI,CAAC,EAAE;gBACP,IAAI,CAAC,MAAM;gBACX,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE;aAC/B,CAAC,IAAI,CAAC,GAAG,CAAC;YACX,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,EAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAC;YACvC,KAAK,EAAE,EAAC,QAAQ,EAAE,MAAM,EAAC;YACzB,aAAa,EAAE,IAAI;YACnB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,IAAA,yBAAU,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC;YAC9C,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;YAC3B,QAAQ,EAAE,IAAA,yBAAU,EAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,QAAQ,CAAC;YAC9D,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;YAC3B,eAAe,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,IAAI;YAChD,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,EAAC,MAAM,EAAE,CAAC,EAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,IAAI;YAChE,MAAM;YACN,QAAQ,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;YACjC,OAAO;YACP,OAAO,EAAE,IAAI;YACb,KAAK;YACL,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,IAAI;SACjB,CAAC,CACH,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AA7ED,sDA6EC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { AirbyteRecord } from 'faros-airbyte-cdk';
|
|
2
2
|
import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
|
|
3
3
|
import { ClaudePlatformConverter } from './platform_common';
|
|
4
|
+
export declare function majorUnits(minorUnits: string | null | undefined, currency: string): number | null;
|
|
4
5
|
export declare class SpendLimits extends ClaudePlatformConverter {
|
|
5
6
|
readonly destinationModels: ReadonlyArray<DestinationModel>;
|
|
6
7
|
id(record: AirbyteRecord): any;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.SpendLimits = void 0;
|
|
4
|
+
exports.majorUnits = majorUnits;
|
|
4
5
|
const luxon_1 = require("luxon");
|
|
5
6
|
const ai_1 = require("../common/ai");
|
|
6
7
|
const platform_common_1 = require("./platform_common");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"spend_limits.js","sourceRoot":"","sources":["../../../src/converters/claude/spend_limits.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"spend_limits.js","sourceRoot":"","sources":["../../../src/converters/claude/spend_limits.ts"],"names":[],"mappings":";;;AAmCA,gCAUC;AA3CD,iCAA+B;AAE/B,qCAKsB;AAEtB,uDAG2B;AAE3B,MAAM,uBAAuB,GAAgC,IAAI,GAAG,CAAC;IACnE,CAAC,WAAW,EAAE,MAAM,CAAC;IACrB,CAAC,YAAY,EAAE,OAAO,CAAC;IACvB,CAAC,cAAc,EAAE,SAAS,CAAC;CAC5B,CAAC,CAAC;AAEH,SAAS,sBAAsB,CAAC,QAAgB;IAC9C,IAAI,CAAC;QACH,OAAO,CACL,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1B,KAAK,EAAE,UAAU;YACjB,QAAQ;SACT,CAAC,CAAC,eAAe,EAAE,CAAC,qBAAqB,IAAI,CAAC,CAChD,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED,SAAgB,UAAU,CACxB,UAAqC,EACrC,QAAgB;IAEhB,IAAI,UAAU,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,sEAAsE;IACtE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,EAAE,IAAI,sBAAsB,CAAC,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QAC1E,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,MAAa,WAAY,SAAQ,yCAAuB;IAAxD;;QACW,sBAAiB,GAAoC;YAC5D,qBAAgB;YAChB,uBAAkB;YAClB,mBAAc;YACd,mBAAc;SACf,CAAC;IA0FJ,CAAC;IAxFC,EAAE,CAAC,MAAqB;QACtB,MAAM,IAAI,GAAG,MAAM,EAAE,MAAM,EAAE,IAAwB,CAAC;QACtD,OAAO,IAAI,EAAE,OAAO,IAAI,IAAI,EAAE,MAAM;YAClC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE;YAClC,CAAC,CAAC,SAAS,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAwB,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,wCAAsB,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5D,IAAI,CAAC,SAAS;YAAE,OAAO,EAAE,CAAC;QAE1B,MAAM,MAAM,GAAwB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,0BAA0B,CACnD,qBAAgB,EAChB,OAAO,CAAC,GAAG,EACX,OAAO,CACR,CAAC;QACF,IAAI,aAAa;YAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC9C,MAAM,eAAe,GAAG,IAAI,CAAC,0BAA0B,CACrD,uBAAkB,EAClB,SAAS,CAAC,GAAG,EACb,EAAC,GAAG,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI,EAAC,CACzD,CAAC;QACF,IAAI,eAAe;YAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAElD,MAAM,WAAW,GACf,IAAI,CAAC,MAAM,KAAK,SAAS;YACvB,CAAC,CAAC,gBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,EAAC,IAAI,EAAE,KAAK,EAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;YACpE,CAAC,CAAC,IAAI,CAAC;QACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;QACrC,MAAM,KAAK,GACT,UAAU,KAAK,YAAY;YACzB,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC;YACzD,CAAC,CAAC,IAAI,CAAC;QACX,IAAI,KAAK,EAAE,CAAC;YACV,kEAAkE;YAClE,MAAM,WAAW,GAAG,IAAI,CAAC,0BAA0B,CACjD,mBAAc,EACd,KAAK,CAAC,GAAG,EACT,EAAC,GAAG,KAAK,EAAC,CACX,CAAC;YACF,IAAI,WAAW;gBAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,mBAAc,EAAE;YACrC,GAAG,EAAE;gBACH,aAAa;gBACb,IAAI,CAAC,OAAO;gBACZ,IAAI,CAAC,MAAM;gBACX,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE;aAC/B,CAAC,IAAI,CAAC,GAAG,CAAC;YACX,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,EAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAC;YACvC,KAAK,EAAE,EAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,IAAI,IAAI,EAAC;YAChE,aAAa,EACX,UAAU,IAAI,UAAU,KAAK,MAAM;gBACjC,CAAC,CAAC;oBACE,QAAQ,EAAE,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,QAAQ;oBAC7D,MAAM,EACJ,IAAI,CAAC,MAAM,EAAE,SAAS;wBACtB,IAAI,CAAC,MAAM,EAAE,aAAa;wBAC1B,UAAU;iBACb;gBACH,CAAC,CAAC,IAAI;YACV,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC;YAC9C,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;YAC3B,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,QAAQ,CAAC;YAC9D,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;YAC3B,eAAe,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,IAAI;YAChD,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,EAAC,MAAM,EAAE,CAAC,EAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,IAAI;YAChE,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,IAAI;YACd,OAAO;YACP,OAAO,EAAE,IAAI;YACb,KAAK;YACL,SAAS;YACT,UAAU,EAAE,IAAI;SACjB,CAAC,CACH,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAhGD,kCAgGC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { AirbyteRecord } from 'faros-airbyte-cdk';
|
|
2
|
+
import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
|
|
3
|
+
import { CodexConverter } from './common';
|
|
4
|
+
export declare class UsageLimits extends CodexConverter {
|
|
5
|
+
readonly destinationModels: ReadonlyArray<DestinationModel>;
|
|
6
|
+
private readonly seenDimensions;
|
|
7
|
+
id(record: AirbyteRecord): any;
|
|
8
|
+
convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
|
|
9
|
+
private firstSetting;
|
|
10
|
+
private limitRecord;
|
|
11
|
+
private pushDimension;
|
|
12
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UsageLimits = void 0;
|
|
4
|
+
const luxon_1 = require("luxon");
|
|
5
|
+
const ai_1 = require("../common/ai");
|
|
6
|
+
const common_1 = require("./common");
|
|
7
|
+
const SCOPE_CATEGORIES = {
|
|
8
|
+
workspace: 'Account',
|
|
9
|
+
group: 'Group',
|
|
10
|
+
user: 'Principal',
|
|
11
|
+
};
|
|
12
|
+
const UNITS = { usd: 'USD', credit: 'credit' };
|
|
13
|
+
class UsageLimits extends common_1.CodexConverter {
|
|
14
|
+
constructor() {
|
|
15
|
+
super(...arguments);
|
|
16
|
+
this.destinationModels = [
|
|
17
|
+
ai_1.AI_ACCOUNT_MODEL,
|
|
18
|
+
ai_1.AI_PRINCIPAL_MODEL,
|
|
19
|
+
ai_1.AI_GROUP_MODEL,
|
|
20
|
+
ai_1.AI_LIMIT_MODEL,
|
|
21
|
+
];
|
|
22
|
+
this.seenDimensions = new Set();
|
|
23
|
+
}
|
|
24
|
+
id(record) {
|
|
25
|
+
const item = record?.record?.data;
|
|
26
|
+
return item?.level && item?.id ? `${item.level}:${item.id}` : undefined;
|
|
27
|
+
}
|
|
28
|
+
async convert(record, ctx) {
|
|
29
|
+
const item = record.record.data;
|
|
30
|
+
if (!item.workspace_id || !item.id)
|
|
31
|
+
return [];
|
|
32
|
+
const account = { uid: item.workspace_id, source: this.source };
|
|
33
|
+
const result = [];
|
|
34
|
+
this.pushDimension(result, ai_1.AI_ACCOUNT_MODEL, account.uid, account);
|
|
35
|
+
const periodStart = luxon_1.DateTime.fromISO(item.observed_at, {
|
|
36
|
+
zone: 'utc',
|
|
37
|
+
}).startOf('month');
|
|
38
|
+
const limitOf = (setting, fields) => this.limitRecord(item, setting, periodStart, account, fields);
|
|
39
|
+
if (item.level === 'workspace') {
|
|
40
|
+
const setting = this.firstSetting(item, ctx);
|
|
41
|
+
if (setting)
|
|
42
|
+
result.push(limitOf(setting, { pooled: false }));
|
|
43
|
+
return result;
|
|
44
|
+
}
|
|
45
|
+
if (item.level === 'group') {
|
|
46
|
+
const group = { uid: item.id, account };
|
|
47
|
+
this.pushDimension(result, ai_1.AI_GROUP_MODEL, group.uid, {
|
|
48
|
+
...group,
|
|
49
|
+
name: item.group_name ?? null,
|
|
50
|
+
});
|
|
51
|
+
const setting = this.firstSetting(item, ctx);
|
|
52
|
+
if (setting)
|
|
53
|
+
result.push(limitOf(setting, { pooled: false, group }));
|
|
54
|
+
return result;
|
|
55
|
+
}
|
|
56
|
+
const principal = { uid: item.id, account };
|
|
57
|
+
this.pushDimension(result, ai_1.AI_PRINCIPAL_MODEL, principal.uid, {
|
|
58
|
+
...principal,
|
|
59
|
+
email: item.email ?? null,
|
|
60
|
+
});
|
|
61
|
+
const effective = item.effective_monthly_usage_limit;
|
|
62
|
+
if (!effective?.limit)
|
|
63
|
+
return result;
|
|
64
|
+
const source = effective.source ?? { kind: 'unknown' };
|
|
65
|
+
let group = null;
|
|
66
|
+
if (source.kind === 'group_default' && source.group_id) {
|
|
67
|
+
group = { uid: source.group_id, account };
|
|
68
|
+
this.pushDimension(result, ai_1.AI_GROUP_MODEL, group.uid, {
|
|
69
|
+
...group,
|
|
70
|
+
name: source.group_name ?? null,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
result.push(limitOf(effective.limit, {
|
|
74
|
+
inheritedFrom: inheritedFrom(source),
|
|
75
|
+
consumed: consumed(item, limitUnit(effective.limit)),
|
|
76
|
+
pooled: null,
|
|
77
|
+
group,
|
|
78
|
+
principal,
|
|
79
|
+
}));
|
|
80
|
+
return result;
|
|
81
|
+
}
|
|
82
|
+
firstSetting(item, ctx) {
|
|
83
|
+
const settings = item.default_user_monthly_usage_limit ?? [];
|
|
84
|
+
if (settings.length > 1) {
|
|
85
|
+
ctx.logger.warn(`Codex ${item.level} ${item.id} has ${settings.length} default usage limits; only the first is synced`);
|
|
86
|
+
}
|
|
87
|
+
return settings[0];
|
|
88
|
+
}
|
|
89
|
+
limitRecord(item, setting, periodStart, account, fields) {
|
|
90
|
+
const unit = limitUnit(setting);
|
|
91
|
+
return {
|
|
92
|
+
model: ai_1.AI_LIMIT_MODEL,
|
|
93
|
+
record: {
|
|
94
|
+
uid: [
|
|
95
|
+
'usage_limit',
|
|
96
|
+
item.workspace_id,
|
|
97
|
+
item.level,
|
|
98
|
+
item.id,
|
|
99
|
+
'monthly',
|
|
100
|
+
periodStart.toISODate(),
|
|
101
|
+
].join(':'),
|
|
102
|
+
source: this.source,
|
|
103
|
+
type: unit
|
|
104
|
+
? { category: unit === 'usd' ? 'Spend' : 'Credits', detail: null }
|
|
105
|
+
: null,
|
|
106
|
+
scope: { category: SCOPE_CATEGORIES[item.level], detail: null },
|
|
107
|
+
inheritedFrom: null,
|
|
108
|
+
model: null,
|
|
109
|
+
amount: limitAmount(setting),
|
|
110
|
+
unit: unit ? UNITS[unit] : null,
|
|
111
|
+
consumed: null,
|
|
112
|
+
period: 'monthly',
|
|
113
|
+
periodStartedAt: periodStart.toJSDate(),
|
|
114
|
+
periodEndedAt: periodStart.plus({ months: 1 }).toJSDate(),
|
|
115
|
+
pooled: null,
|
|
116
|
+
enforced: true,
|
|
117
|
+
account,
|
|
118
|
+
project: null,
|
|
119
|
+
group: null,
|
|
120
|
+
principal: null,
|
|
121
|
+
credential: null,
|
|
122
|
+
...fields,
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
pushDimension(result, model, uid, record) {
|
|
127
|
+
const key = `${model}:${uid}`;
|
|
128
|
+
if (this.seenDimensions.has(key))
|
|
129
|
+
return;
|
|
130
|
+
this.seenDimensions.add(key);
|
|
131
|
+
result.push({ model, record });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
exports.UsageLimits = UsageLimits;
|
|
135
|
+
// Plans without unit-aware limits report only the legacy credit limit
|
|
136
|
+
function limitUnit(setting) {
|
|
137
|
+
if (setting.limit_amount?.unit)
|
|
138
|
+
return setting.limit_amount.unit;
|
|
139
|
+
return setting.limit != null ? 'credit' : null;
|
|
140
|
+
}
|
|
141
|
+
function limitAmount(setting) {
|
|
142
|
+
if (setting.type === 'unlimited')
|
|
143
|
+
return null;
|
|
144
|
+
const amount = setting.limit_amount
|
|
145
|
+
? Number(setting.limit_amount.amount)
|
|
146
|
+
: setting.limit;
|
|
147
|
+
return Number.isFinite(amount) ? amount : null;
|
|
148
|
+
}
|
|
149
|
+
// Usage is only comparable to the limit when both are in the same unit
|
|
150
|
+
function consumed(item, unit) {
|
|
151
|
+
if (!unit)
|
|
152
|
+
return null;
|
|
153
|
+
if (item.current_month_usage_amount?.unit === unit) {
|
|
154
|
+
return item.current_month_usage_amount.amount ?? null;
|
|
155
|
+
}
|
|
156
|
+
if (item.current_month_usage_unit === unit) {
|
|
157
|
+
return item.current_month_usage ?? null;
|
|
158
|
+
}
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
function inheritedFrom(source) {
|
|
162
|
+
switch (source.kind) {
|
|
163
|
+
case 'individual_override':
|
|
164
|
+
return null;
|
|
165
|
+
case 'group_default':
|
|
166
|
+
return { category: 'Group', detail: source.group_id ?? null };
|
|
167
|
+
case 'workspace_default':
|
|
168
|
+
return { category: 'Account', detail: source.seat_type ?? null };
|
|
169
|
+
case 'role_based_personal_budget':
|
|
170
|
+
return { category: 'Custom', detail: source.role_id ?? source.kind };
|
|
171
|
+
default:
|
|
172
|
+
return { category: 'Custom', detail: source.kind };
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=usage_limits.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"usage_limits.js","sourceRoot":"","sources":["../../../src/converters/codex/usage_limits.ts"],"names":[],"mappings":";;;AACA,iCAA+B;AAE/B,qCAOsB;AAEtB,qCAAwC;AAmCxC,MAAM,gBAAgB,GAAG;IACvB,SAAS,EAAE,SAAS;IACpB,KAAK,EAAE,OAAO;IACd,IAAI,EAAE,WAAW;CACT,CAAC;AAEX,MAAM,KAAK,GAAyB,EAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAC,CAAC;AAEnE,MAAa,WAAY,SAAQ,uBAAc;IAA/C;;QACW,sBAAiB,GAAoC;YAC5D,qBAAgB;YAChB,uBAAkB;YAClB,mBAAc;YACd,mBAAc;SACf,CAAC;QAEe,mBAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IA+ItD,CAAC;IA7IC,EAAE,CAAC,MAAqB;QACtB,MAAM,IAAI,GAAG,MAAM,EAAE,MAAM,EAAE,IAAsB,CAAC;QACpD,OAAO,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAsB,CAAC;QAClD,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QAE9C,MAAM,OAAO,GAAiB,EAAC,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAC,CAAC;QAC5E,MAAM,MAAM,GAAwB,EAAE,CAAC;QACvC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,qBAAgB,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAEnE,MAAM,WAAW,GAAG,gBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE;YACrD,IAAI,EAAE,KAAK;SACZ,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACpB,MAAM,OAAO,GAAG,CACd,OAA0B,EAC1B,MAAc,EACK,EAAE,CACrB,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAEhE,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAC,MAAM,EAAE,KAAK,EAAC,CAAC,CAAC,CAAC;YAC5D,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAuB,EAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAC,CAAC;YAC1D,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,mBAAc,EAAE,KAAK,CAAC,GAAG,EAAE;gBACpD,GAAG,KAAK;gBACR,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;aAC9B,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAC,CAAC,CAAC,CAAC;YACnE,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,SAAS,GAAuB,EAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAC,CAAC;QAC9D,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,uBAAkB,EAAE,SAAS,CAAC,GAAG,EAAE;YAC5D,GAAG,SAAS;YACZ,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;SAC1B,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,CAAC,6BAA6B,CAAC;QACrD,IAAI,CAAC,SAAS,EAAE,KAAK;YAAE,OAAO,MAAM,CAAC;QAErC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,EAAC,IAAI,EAAE,SAAS,EAAC,CAAC;QACrD,IAAI,KAAK,GAA8B,IAAI,CAAC;QAC5C,IAAI,MAAM,CAAC,IAAI,KAAK,eAAe,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACvD,KAAK,GAAG,EAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAC,CAAC;YACxC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,mBAAc,EAAE,KAAK,CAAC,GAAG,EAAE;gBACpD,GAAG,KAAK;gBACR,IAAI,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;aAChC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CACT,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE;YACvB,aAAa,EAAE,aAAa,CAAC,MAAM,CAAC;YACpC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACpD,MAAM,EAAE,IAAI;YACZ,KAAK;YACL,SAAS;SACV,CAAC,CACH,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,YAAY,CAClB,IAAoB,EACpB,GAAkB;QAElB,MAAM,QAAQ,GAAG,IAAI,CAAC,gCAAgC,IAAI,EAAE,CAAC;QAC7D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,GAAG,CAAC,MAAM,CAAC,IAAI,CACb,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,QAAQ,QAAQ,CAAC,MAAM,iDAAiD,CACvG,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IAEO,WAAW,CACjB,IAAoB,EACpB,OAA0B,EAC1B,WAAqB,EACrB,OAAqB,EACrB,MAAc;QAEd,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,mBAAc;YACrB,MAAM,EAAE;gBACN,GAAG,EAAE;oBACH,aAAa;oBACb,IAAI,CAAC,YAAY;oBACjB,IAAI,CAAC,KAAK;oBACV,IAAI,CAAC,EAAE;oBACP,SAAS;oBACT,WAAW,CAAC,SAAS,EAAE;iBACxB,CAAC,IAAI,CAAC,GAAG,CAAC;gBACX,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,IAAI,EAAE,IAAI;oBACR,CAAC,CAAC,EAAC,QAAQ,EAAE,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAC;oBAChE,CAAC,CAAC,IAAI;gBACR,KAAK,EAAE,EAAC,QAAQ,EAAE,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAC;gBAC7D,aAAa,EAAE,IAAI;gBACnB,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC;gBAC5B,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;gBAC/B,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,SAAS;gBACjB,eAAe,EAAE,WAAW,CAAC,QAAQ,EAAE;gBACvC,aAAa,EAAE,WAAW,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,CAAC,EAAC,CAAC,CAAC,QAAQ,EAAE;gBACvD,MAAM,EAAE,IAAI;gBACZ,QAAQ,EAAE,IAAI;gBACd,OAAO;gBACP,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,IAAI;gBACf,UAAU,EAAE,IAAI;gBAChB,GAAG,MAAM;aACV;SAC8B,CAAC;IACpC,CAAC;IAEO,aAAa,CACnB,MAA2B,EAC3B,KAAuB,EACvB,GAAW,EACX,MAAc;QAEd,MAAM,GAAG,GAAG,GAAG,KAAK,IAAI,GAAG,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QACzC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,MAAM,EAAiC,CAAC,CAAC;IAC/D,CAAC;CACF;AAvJD,kCAuJC;AAED,sEAAsE;AACtE,SAAS,SAAS,CAAC,OAA0B;IAC3C,IAAI,OAAO,CAAC,YAAY,EAAE,IAAI;QAAE,OAAO,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC;IACjE,OAAO,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED,SAAS,WAAW,CAAC,OAA0B;IAC7C,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY;QACjC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC;QACrC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClB,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED,uEAAuE;AACvE,SAAS,QAAQ,CAAC,IAAoB,EAAE,IAAiB;IACvD,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,IAAI,IAAI,CAAC,0BAA0B,EAAE,IAAI,KAAK,IAAI,EAAE,CAAC;QACnD,OAAO,IAAI,CAAC,0BAA0B,CAAC,MAAM,IAAI,IAAI,CAAC;IACxD,CAAC;IACD,IAAI,IAAI,CAAC,wBAAwB,KAAK,IAAI,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CACpB,MAAkC;IAElC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,qBAAqB;YACxB,OAAO,IAAI,CAAC;QACd,KAAK,eAAe;YAClB,OAAO,EAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI,EAAC,CAAC;QAC9D,KAAK,mBAAmB;YACtB,OAAO,EAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI,EAAC,CAAC;QACjE,KAAK,4BAA4B;YAC/B,OAAO,EAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,EAAC,CAAC;QACrE;YACE,OAAO,EAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAC,CAAC;IACrD,CAAC;AACH,CAAC"}
|
|
@@ -44,6 +44,7 @@ export declare class StreamContext implements OriginProvider {
|
|
|
44
44
|
private readonly streamsByResetModel;
|
|
45
45
|
private readonly resetModels;
|
|
46
46
|
private readonly skipResetModels;
|
|
47
|
+
private readonly fullResetModels;
|
|
47
48
|
private sourceConfig;
|
|
48
49
|
private originWithBucketSuffix?;
|
|
49
50
|
constructor(logger: AirbyteLogger, config: DestinationConfig, streamsSyncMode: Dictionary<DestinationSyncMode>, graph?: string, origin?: string, farosClient?: FarosClient);
|
|
@@ -56,6 +57,8 @@ export declare class StreamContext implements OriginProvider {
|
|
|
56
57
|
set(streamName: string, id: string, record: AirbyteRecord): void;
|
|
57
58
|
stats(includeIds?: boolean): string;
|
|
58
59
|
registerStreamResetModels(streamName: string, models: Set<string>): void;
|
|
60
|
+
registerStreamFullResetModels(streamName: string, models: Set<string>): void;
|
|
61
|
+
getFullResetModels(): ReadonlySet<string>;
|
|
59
62
|
markStreamForReset(streamName: string): void;
|
|
60
63
|
disableResetForModel(model: string): void;
|
|
61
64
|
markAllStreamsForReset(): void;
|
|
@@ -77,6 +77,9 @@ class StreamContext {
|
|
|
77
77
|
this.streamsByResetModel = {};
|
|
78
78
|
this.resetModels = new Set();
|
|
79
79
|
this.skipResetModels = new Set();
|
|
80
|
+
// Models that the source explicitly requested to delete in full, which may
|
|
81
|
+
// be reset even when the sync does not write any records
|
|
82
|
+
this.fullResetModels = new Set();
|
|
80
83
|
this.sourceConfig = {};
|
|
81
84
|
this.recordsByStreamName = Object.create(null);
|
|
82
85
|
}
|
|
@@ -154,6 +157,13 @@ class StreamContext {
|
|
|
154
157
|
}
|
|
155
158
|
});
|
|
156
159
|
}
|
|
160
|
+
registerStreamFullResetModels(streamName, models) {
|
|
161
|
+
this.registerStreamResetModels(streamName, models);
|
|
162
|
+
models.forEach((model) => this.fullResetModels.add(model));
|
|
163
|
+
}
|
|
164
|
+
getFullResetModels() {
|
|
165
|
+
return this.fullResetModels;
|
|
166
|
+
}
|
|
157
167
|
markStreamForReset(streamName) {
|
|
158
168
|
const models = this.resetModelsByStream[streamName];
|
|
159
169
|
models?.forEach((model) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"converter.js","sourceRoot":"","sources":["../../src/converters/converter.ts"],"names":[],"mappings":";;;;;;AA6EA,8CAaC;
|
|
1
|
+
{"version":3,"file":"converter.js","sourceRoot":"","sources":["../../src/converters/converter.ts"],"names":[],"mappings":";;;;;;AA6EA,8CAaC;AA8LD,wCASC;AA1RD,qDAIyB;AACzB,mCAA6C;AAC7C,kEAAmC;AAEnC,mCAA8B;AAI9B,wCAAwC;AACxC,MAAsB,cAAc;IAMlC,0CAA0C;IAC1C,IAAI,UAAU;QACZ,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QACpC,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,UAAU,CACjC,GAAG,IAAI,CAAC,MAAM,GAAG,qCAAmB,GAAG,IAAA,kBAAS,EAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAC1E,CAAC;QACF,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;+DAE2D;IAC3D,IAAI,YAAY;QACd,OAAO,EAAE,CAAC;IACZ,CAAC;IAKD;;4CAEwC;IACxC,sBAAsB,CACpB,MAAqB,EACrB,GAAkB;QAElB,OAAO,MAAM,CAAC;IAChB,CAAC;IAWD;;2FAEuF;IACvF,KAAK,CAAC,oBAAoB,CACxB,GAAkB,CAAC,wDAAwD;;QAE3E,OAAO,EAAE,CAAC;IACZ,CAAC;CACF;AApDD,wCAoDC;AACD,MAAsB,SAAU,SAAQ,cAA+B;CAAG;AAA1E,8BAA0E;AAE1E,8DAA8D;AAC9D,4EAA4E;AAC5E,SAAgB,iBAAiB,CAAI,GAAQ,EAAE,IAAY;IACzD,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxC,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,eAAM,CACd,oCAAoC,IAAI,IAAI,GAAG,YAAY,CAAC,EAAE,CAC/D,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,IAAI,eAAM,CAAC,GAAG,IAAI,mDAAmD,CAAC,CAAC;AAC/E,CAAC;AAED,kEAAkE;AAClE,MAAa,aAAa;IAiBxB,YACW,MAAqB,EACrB,MAAyB,EACzB,eAAgD,EAChD,KAAc,EACN,MAAe,EACvB,WAAyB;QALzB,WAAM,GAAN,MAAM,CAAe;QACrB,WAAM,GAAN,MAAM,CAAmB;QACzB,oBAAe,GAAf,eAAe,CAAiC;QAChD,UAAK,GAAL,KAAK,CAAS;QACN,WAAM,GAAN,MAAM,CAAS;QACvB,gBAAW,GAAX,WAAW,CAAc;QAtBpC,6EAA6E;QAC7E,YAAY;QACK,wBAAmB,GAA4B,EAAE,CAAC;QAEnE,8EAA8E;QAC9E,sEAAsE;QACrD,wBAAmB,GAA4B,EAAE,CAAC;QAElD,gBAAW,GAAgB,IAAI,GAAG,EAAE,CAAC;QACrC,oBAAe,GAAgB,IAAI,GAAG,EAAE,CAAC;QAC1D,2EAA2E;QAC3E,yDAAyD;QACxC,oBAAe,GAAgB,IAAI,GAAG,EAAE,CAAC;QAClD,iBAAY,GAAkB,EAAE,CAAC;QAoCxB,wBAAmB,GAClC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IA3BnB,CAAC;IAEJ,SAAS;QACP,IAAI,IAAI,CAAC,sBAAsB;YAAE,OAAO,IAAI,CAAC,sBAAsB,CAAC;QACpE,IACE,IAAI,CAAC,YAAY,EAAE,4BAA4B;YAC/C,IAAI,CAAC,YAAY,EAAE,SAAS,EAC5B,CAAC;YACD,IAAI,CAAC,sBAAsB,GAAG;gBAC5B,IAAI,CAAC,MAAM;gBACX,QAAQ;gBACR,IAAI,CAAC,YAAY,CAAC,SAAS;aAC5B,CAAC,IAAI,CAAC,qCAAmB,CAAC,CAAC;YAC5B,OAAO,IAAI,CAAC,sBAAsB,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,eAAe,CAAC,cAA6B;QAC3C,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;IACrC,CAAC;IAKD,MAAM,CAAC,UAAkB;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,GAAG,CAAC,UAAkB,EAAE,EAAU;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;YACrB,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC;QACtB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,GAAG,CAAC,UAAkB,EAAE,EAAU,EAAE,MAAqB;QACvD,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,CAAC,IAAI;YAAE,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACtE,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;IACpD,CAAC;IACD,KAAK,CAAC,UAAU,GAAG,KAAK;QACtB,MAAM,WAAW,GAAG,IAAA,uBAAM,EAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,EAAC,WAAW,EAAC,CAAC;QAC1B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACtD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,IAAI,UAAU,EAAE,CAAC;gBACf,GAAG,CAAC,CAAC,CAAC,GAAG;oBACP,KAAK,EAAE,GAAG,CAAC,MAAM;oBACjB,GAAG;iBACJ,CAAC;YACJ,CAAC;;gBAAM,GAAG,CAAC,CAAC,CAAC,GAAG,EAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,yBAAyB,CAAC,UAAkB,EAAE,MAAmB;QAC/D,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACvB,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAChD,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACvB,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,6BAA6B,CAAC,UAAkB,EAAE,MAAmB;QACnE,IAAI,CAAC,yBAAyB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACnD,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,kBAAkB;QAChB,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;IAED,kBAAkB,CAAC,UAAkB;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACxB,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAChD,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC3B,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBACvB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,oBAAoB,CAAC,KAAa;QAChC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,qDAAqD;IACrD,sBAAsB;QACpB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE;YAC3D,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,iBAAiB;QACf,OAAO,IAAI,GAAG,CACZ,IAAA,mBAAU,EAAC,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,CAC7D,CAAC;IACJ,CAAC;CACF;AAhJD,sCAgJC;AAED;;;;;;GAMG;AACH,MAAa,UAAU;IACrB,YACW,MAAc,EACd,IAAY;QADZ,WAAM,GAAN,MAAM,CAAQ;QACd,SAAI,GAAJ,IAAI,CAAQ;IACpB,CAAC;IAIJ,IAAI,QAAQ;QACV,IAAI,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC;QAC9B,IAAI,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,qCAAmB,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5E,OAAO,IAAI,CAAC,GAAG,CAAC;IAClB,CAAC;IAED,MAAM,CAAC,UAAU,CAAC,CAAS;QACzB,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,IAAI,eAAM,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,qCAAmB,CAAC,CAAC;QACvC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,eAAM,CACd,uBAAuB,CAAC,uCAAuC,qCAAmB,IAAI,CACvF,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,GAAG,GAAG,cAAc,CAAC,CAAC,EAAE,qCAAmB,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;CACF;AA7BD,gCA6BC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,0BAA0B;AAC1B,SAAgB,cAAc,CAAC,GAAW,EAAE,SAAiB,EAAE,KAAa;IAC1E,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACnC,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IAExC,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACzD,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,YAAY,CAAC;AACtB,CAAC"}
|
|
@@ -48,11 +48,12 @@ class FarosFeed extends converter_1.Converter {
|
|
|
48
48
|
// These are issued by the feed and are only applicable to the V1 API
|
|
49
49
|
// Transform them into resets for the v2 API
|
|
50
50
|
// We accumulate the model names in 'resetModels' to reset them in topological order
|
|
51
|
+
// The deletion is explicit, so the models are reset even if no records are written
|
|
51
52
|
if (model.endsWith('__Deletion') && Object.entries(rec).length == 1) {
|
|
52
53
|
const [key, value] = Object.entries(rec).pop();
|
|
53
54
|
if (key === 'where' && typeof value == 'string') {
|
|
54
55
|
const [baseModel] = model.split('__', 1);
|
|
55
|
-
ctx.
|
|
56
|
+
ctx.registerStreamFullResetModels('faros_feed', new Set([baseModel]));
|
|
56
57
|
return [];
|
|
57
58
|
}
|
|
58
59
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"faros_feed.js","sourceRoot":"","sources":["../../../src/converters/faros-feeds/faros_feed.ts"],"names":[],"mappings":";;;AACA,qDAAiD;AAGjD,4CAKsB;AACtB,+CAA8C;AAY9C,SAAS,kBAAkB,CAAC,IAAqB;IAC/C,OAAO,CACL,IAAI,IAAI,IAAI;QACZ,IAAI,CAAC,MAAM;QACX,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ;QAC/B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CACtC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,IAAqB;IAC3C,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;QACvD,OAAO,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC;IACtB,CAAC;SAAM,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1C,+DAA+D;QAC/D,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;QAChD,OAAO,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC;IACtB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAa,SAAU,SAAQ,qBAAS;IAAxC;;QACE,WAAM,GAAG,aAAa,CAAC;QAEd,sBAAiB,GAAoC,6BAAe,CAAC;QAEtE,WAAM,GAAqB,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"faros_feed.js","sourceRoot":"","sources":["../../../src/converters/faros-feeds/faros_feed.ts"],"names":[],"mappings":";;;AACA,qDAAiD;AAGjD,4CAKsB;AACtB,+CAA8C;AAY9C,SAAS,kBAAkB,CAAC,IAAqB;IAC/C,OAAO,CACL,IAAI,IAAI,IAAI;QACZ,IAAI,CAAC,MAAM;QACX,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ;QAC/B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CACtC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,IAAqB;IAC3C,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;QACvD,OAAO,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC;IACtB,CAAC;SAAM,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1C,+DAA+D;QAC/D,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;QAChD,OAAO,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC;IACtB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAa,SAAU,SAAQ,qBAAS;IAAxC;;QACE,WAAM,GAAG,aAAa,CAAC;QAEd,sBAAiB,GAAoC,6BAAe,CAAC;QAEtE,WAAM,GAAqB,SAAS,CAAC;IA6C/C,CAAC;IA3CC,EAAE;QACA,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACjD,IAAI,CAAC,MAAM,GAAG,IAAI,kCAAgB,CAChC,MAAM,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAC5C,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QAEhC,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAE1B,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAC5B,MAAM,EAAC,KAAK,EAAE,GAAG,EAAC,GAAG,WAAW,CAAC;QAEjC,+BAA+B;QAC/B,+DAA+D;QAC/D,qEAAqE;QACrE,4CAA4C;QAC5C,oFAAoF;QACpF,mFAAmF;QACnF,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpE,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAC/C,IAAI,GAAG,KAAK,OAAO,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAChD,MAAM,CAAC,SAAS,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBACzC,GAAG,CAAC,6BAA6B,CAAC,YAAY,EAAE,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBACtE,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAsB,CAAC,CAAC;IACrD,CAAC;CACF;AAlDD,8BAkDC"}
|
package/lib/destination.d.ts
CHANGED
|
@@ -52,6 +52,15 @@ export declare class FarosDestination extends AirbyteDestination<DestinationConf
|
|
|
52
52
|
write(config: DestinationConfig, redactedConfig: AirbyteConfig, catalog: AirbyteConfiguredCatalog, stdin: NodeJS.ReadStream, dryRun: boolean): AsyncGenerator<AirbyteStateMessage>;
|
|
53
53
|
private writeEntries;
|
|
54
54
|
private shouldResetData;
|
|
55
|
+
/**
|
|
56
|
+
* The reset deletes records of the origin that were refreshed before the
|
|
57
|
+
* earliest refreshedAt of the records upserted in this sync. Without any
|
|
58
|
+
* upserted records, that cutoff stays at its initial value (January 1,
|
|
59
|
+
* 2200) and matches every record of the origin, so only models whose full
|
|
60
|
+
* deletion is intended are reset: all models on reset syncs, and models
|
|
61
|
+
* that the source explicitly requested to delete in full.
|
|
62
|
+
*/
|
|
63
|
+
private modelsToReset;
|
|
55
64
|
handleRecordProcessingError(stats: WriteStats, processRecord: () => Promise<void>): Promise<void>;
|
|
56
65
|
private initStreamsCheckConverters;
|
|
57
66
|
private getConverter;
|
package/lib/destination.js
CHANGED
|
@@ -459,7 +459,7 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
459
459
|
}
|
|
460
460
|
: streamContext, stats, this);
|
|
461
461
|
try {
|
|
462
|
-
for await (const stateMessage of this.writeEntries(config, streamContext, stdin, streams, converterDependencies, stats, writer, () => accountContext?.logFiles, async (isResetSync) => await graphQLClient.resetData(streamContext,
|
|
462
|
+
for await (const stateMessage of this.writeEntries(config, streamContext, stdin, streams, converterDependencies, stats, writer, () => accountContext?.logFiles, async (models, isResetSync, fullResetModels) => await graphQLClient.resetData(streamContext, models, isResetSync, { fullResetModels }), ensureAccountSync, sourceVersion)) {
|
|
463
463
|
await graphQLClient.flush();
|
|
464
464
|
yield stateMessage;
|
|
465
465
|
}
|
|
@@ -544,7 +544,9 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
544
544
|
}
|
|
545
545
|
}
|
|
546
546
|
}
|
|
547
|
-
async *writeEntries(config, ctx, stdin, streams, converterDependencies, stats, writer, getLogFiles,
|
|
547
|
+
async *writeEntries(config, ctx, stdin, streams, converterDependencies, stats, writer, getLogFiles,
|
|
548
|
+
// fullResetModels may be reset even if no records were written
|
|
549
|
+
resetData, ensureAccountSync, sourceVersion) {
|
|
548
550
|
const recordsToBeProcessedLast = [];
|
|
549
551
|
const convertersUsed = new Map();
|
|
550
552
|
// NOTE: readline.createInterface() will start to consume the input stream once invoked.
|
|
@@ -573,6 +575,9 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
573
575
|
let stateMessagesReceived = 0;
|
|
574
576
|
let hasNonEmptyStreamState = false;
|
|
575
577
|
let streamStatusReceived = false;
|
|
578
|
+
// Records written before the model reset, excluding operations such as
|
|
579
|
+
// deletions. Only these lower the reset cutoff (see modelsToReset).
|
|
580
|
+
let recordsUpserted = 0;
|
|
576
581
|
const processedStreams = new Set();
|
|
577
582
|
const failedStreams = new Set();
|
|
578
583
|
// Process input & write records
|
|
@@ -659,7 +664,9 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
659
664
|
// Register all models for reset before marking streams
|
|
660
665
|
ctx.registerStreamResetModels('faros_feed', new Set(model_names_1.ALL_MODEL_NAMES));
|
|
661
666
|
ctx.markAllStreamsForReset();
|
|
662
|
-
|
|
667
|
+
// No records are written yet, so allow a full reset
|
|
668
|
+
const models = Array.from(ctx.getModelsForReset());
|
|
669
|
+
await resetData?.(models, false, models);
|
|
663
670
|
}
|
|
664
671
|
// Idempotent fallback: the identity message normally initializes
|
|
665
672
|
// the account first, but a source that does not emit one still
|
|
@@ -732,7 +739,9 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
732
739
|
convertersUsed.set(stream, converter);
|
|
733
740
|
}
|
|
734
741
|
const writeRecord = async (context) => {
|
|
735
|
-
|
|
742
|
+
const recordsWritten = await this.writeRecord(converter, unpacked, stats, context, writer);
|
|
743
|
+
stats.recordsWritten += recordsWritten;
|
|
744
|
+
recordsUpserted += recordsWritten;
|
|
736
745
|
stats.recordsProcessed++;
|
|
737
746
|
};
|
|
738
747
|
// Check if any converters depend on this record stream.
|
|
@@ -780,7 +789,9 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
780
789
|
// Invoke on processing complete handler on all the active converters
|
|
781
790
|
for (const converter of convertersUsed.values()) {
|
|
782
791
|
const results = await converter.onProcessingComplete(ctx);
|
|
783
|
-
|
|
792
|
+
const recordsWritten = await this.writeConvertedRecords(converter, results, stats, writer);
|
|
793
|
+
stats.recordsWritten += recordsWritten;
|
|
794
|
+
recordsUpserted += recordsWritten;
|
|
784
795
|
}
|
|
785
796
|
const allOverwrite = Object.values(streams)
|
|
786
797
|
.map((s) => s.destination_sync_mode)
|
|
@@ -796,7 +807,7 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
796
807
|
const perStreamResetDetected = stateMessagesReceived > 0 &&
|
|
797
808
|
stateMessagesReceived === stats.messagesRead &&
|
|
798
809
|
!hasNonEmptyStreamState;
|
|
799
|
-
const isResetSync = process.env.WORKER_JOB_ID &&
|
|
810
|
+
const isResetSync = !!process.env.WORKER_JOB_ID &&
|
|
800
811
|
(legacyResetDetected || perStreamResetDetected) &&
|
|
801
812
|
stats.recordsProcessed === 0 &&
|
|
802
813
|
!sourceConfigReceived &&
|
|
@@ -829,7 +840,10 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
829
840
|
if (!streamStatusReceived) {
|
|
830
841
|
ctx.markAllStreamsForReset();
|
|
831
842
|
}
|
|
832
|
-
|
|
843
|
+
const models = this.modelsToReset(ctx, isResetSync, recordsUpserted);
|
|
844
|
+
if (models.length) {
|
|
845
|
+
await resetData?.(models, isResetSync, Array.from(ctx.getFullResetModels()));
|
|
846
|
+
}
|
|
833
847
|
}
|
|
834
848
|
if (isResetSync) {
|
|
835
849
|
if (config.faros_source_id && this.farosClient) {
|
|
@@ -885,6 +899,15 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
885
899
|
return true;
|
|
886
900
|
}
|
|
887
901
|
if (sourceSucceeded) {
|
|
902
|
+
if (isFarosSource) {
|
|
903
|
+
// Faros Sources report a status for each stream they sync, so no
|
|
904
|
+
// stream was synced, e.g. all were skipped by stream_freshness_hours
|
|
905
|
+
// or filtered out before reading. Marking all streams for reset
|
|
906
|
+
// would delete the models of streams that did not run.
|
|
907
|
+
this.logger.warn('Faros Source succeeded without reporting any stream status, so no streams were synced.' +
|
|
908
|
+
' Skipping reset of non-incremental models.');
|
|
909
|
+
return false;
|
|
910
|
+
}
|
|
888
911
|
this.logger.info('Source succeeded. Resetting non-incremental models.');
|
|
889
912
|
return true;
|
|
890
913
|
}
|
|
@@ -900,6 +923,28 @@ class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
|
|
|
900
923
|
this.logger.warn('No success status received from Airbyte Source. Skipping reset of non-incremental models.');
|
|
901
924
|
return false;
|
|
902
925
|
}
|
|
926
|
+
/**
|
|
927
|
+
* The reset deletes records of the origin that were refreshed before the
|
|
928
|
+
* earliest refreshedAt of the records upserted in this sync. Without any
|
|
929
|
+
* upserted records, that cutoff stays at its initial value (January 1,
|
|
930
|
+
* 2200) and matches every record of the origin, so only models whose full
|
|
931
|
+
* deletion is intended are reset: all models on reset syncs, and models
|
|
932
|
+
* that the source explicitly requested to delete in full.
|
|
933
|
+
*/
|
|
934
|
+
modelsToReset(ctx, isResetSync, recordsUpserted) {
|
|
935
|
+
const models = Array.from(ctx.getModelsForReset());
|
|
936
|
+
if (isResetSync || recordsUpserted > 0) {
|
|
937
|
+
return models;
|
|
938
|
+
}
|
|
939
|
+
const fullResetModels = ctx.getFullResetModels();
|
|
940
|
+
const skippedModels = models.filter((m) => !fullResetModels.has(m));
|
|
941
|
+
if (skippedModels.length) {
|
|
942
|
+
this.logger.warn('No records were upserted, so the reset would delete all records of the reset models for the origin.' +
|
|
943
|
+
` Skipping reset of ${skippedModels.length} non-incremental model(s):` +
|
|
944
|
+
` ${skippedModels.join(', ')}`);
|
|
945
|
+
}
|
|
946
|
+
return models.filter((m) => fullResetModels.has(m));
|
|
947
|
+
}
|
|
903
948
|
async handleRecordProcessingError(stats, processRecord) {
|
|
904
949
|
try {
|
|
905
950
|
await processRecord();
|