airbyte-faros-destination 0.23.21 → 0.23.22

package/lib/converters/openai-platform/common.d.ts

@@ -0,0 +1,34 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { Converter, DestinationRecord, StreamContext } from '../converter';
+export declare const AI_ACCOUNT_MODEL = "ai_Account";
+export declare const AI_PROJECT_MODEL = "ai_Project";
+export declare const AI_PRINCIPAL_MODEL = "ai_Principal";
+export declare const AI_CREDENTIAL_MODEL = "ai_Credential";
+export declare const AI_USAGE_MODEL = "ai_Usage";
+export declare const AI_COST_MODEL = "ai_Cost";
+export interface AIAccountKey {
+    uid: string;
+    source: string;
+}
+export interface AIAccountEntityKey {
+    uid: string;
+    account: AIAccountKey;
+}
+interface AIDimensions {
+    account: AIAccountKey;
+    project: AIAccountEntityKey | null;
+    projectName?: string | null;
+    principal?: AIAccountEntityKey | null;
+    principalEmail?: string | null;
+    credential: AIAccountEntityKey | null;
+}
+export declare abstract class OpenAIPlatformConverter extends Converter {
+    readonly source = "OpenAI_Platform";
+    private readonly seenDimensions;
+    id(record: AirbyteRecord): any;
+    protected account(ctx: StreamContext): AIAccountKey;
+    protected accountEntity(uid: string | null | undefined, account: AIAccountKey): AIAccountEntityKey | null;
+    protected destinationDimensionRecord(uid: string, destinationRecord: DestinationRecord): DestinationRecord | null;
+    protected destinationDimensions({ account, project, projectName, principal, principalEmail, credential, }: AIDimensions): DestinationRecord[];
+}
+export {};

package/lib/converters/openai-platform/common.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIPlatformConverter = exports.AI_COST_MODEL = exports.AI_USAGE_MODEL = exports.AI_CREDENTIAL_MODEL = exports.AI_PRINCIPAL_MODEL = exports.AI_PROJECT_MODEL = exports.AI_ACCOUNT_MODEL = void 0;
+const converter_1 = require("../converter");
+exports.AI_ACCOUNT_MODEL = 'ai_Account';
+exports.AI_PROJECT_MODEL = 'ai_Project';
+exports.AI_PRINCIPAL_MODEL = 'ai_Principal';
+exports.AI_CREDENTIAL_MODEL = 'ai_Credential';
+exports.AI_USAGE_MODEL = 'ai_Usage';
+exports.AI_COST_MODEL = 'ai_Cost';
+class OpenAIPlatformConverter extends converter_1.Converter {
+    constructor() {
+        super(...arguments);
+        this.source = 'OpenAI_Platform';
+        this.seenDimensions = new Set();
+    }
+    id(record) {
+        return record?.record?.data?._record_id;
+    }
+    account(ctx) {
+        return {
+            uid: ctx.getSourceConfig().organization_id,
+            source: this.source,
+        };
+    }
+    accountEntity(uid, account) {
+        return uid ? { uid, account } : null;
+    }
+    destinationDimensionRecord(uid, destinationRecord) {
+        const { model } = destinationRecord;
+        const key = `${model}:${uid}`;
+        if (this.seenDimensions.has(key))
+            return null;
+        this.seenDimensions.add(key);
+        return destinationRecord;
+    }
+    destinationDimensions({ account, project, projectName, principal, principalEmail, credential, }) {
+        const result = [];
+        const accountRecord = this.destinationDimensionRecord(account.uid, {
+            model: exports.AI_ACCOUNT_MODEL,
+            record: account,
+        });
+        if (accountRecord)
+            result.push(accountRecord);
+        if (project) {
+            const projectRecord = this.destinationDimensionRecord(project.uid, {
+                model: exports.AI_PROJECT_MODEL,
+                record: { ...project, name: projectName ?? null },
+            });
+            if (projectRecord)
+                result.push(projectRecord);
+        }
+        if (principal) {
+            const principalRecord = this.destinationDimensionRecord(principal.uid, {
+                model: exports.AI_PRINCIPAL_MODEL,
+                record: { ...principal, email: principalEmail ?? null },
+            });
+            if (principalRecord)
+                result.push(principalRecord);
+        }
+        if (credential) {
+            const credentialRecord = this.destinationDimensionRecord(credential.uid, {
+                model: exports.AI_CREDENTIAL_MODEL,
+                record: { ...credential },
+            });
+            if (credentialRecord)
+                result.push(credentialRecord);
+        }
+        return result;
+    }
+}
+exports.OpenAIPlatformConverter = OpenAIPlatformConverter;
+//# sourceMappingURL=common.js.map

package/lib/converters/openai-platform/common.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"common.js","sourceRoot":"","sources":["../../../src/converters/openai-platform/common.ts"],"names":[],"mappings":";;;AAEA,4CAAyE;AAE5D,QAAA,gBAAgB,GAAG,YAAY,CAAC;AAChC,QAAA,gBAAgB,GAAG,YAAY,CAAC;AAChC,QAAA,kBAAkB,GAAG,cAAc,CAAC;AACpC,QAAA,mBAAmB,GAAG,eAAe,CAAC;AACtC,QAAA,cAAc,GAAG,UAAU,CAAC;AAC5B,QAAA,aAAa,GAAG,SAAS,CAAC;AAqBvC,MAAsB,uBAAwB,SAAQ,qBAAS;IAA/D;;QACW,WAAM,GAAG,iBAAiB,CAAC;QACnB,mBAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAqEtD,CAAC;IAnEC,EAAE,CAAC,MAAqB;QACtB,OAAO,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;IAC1C,CAAC;IAES,OAAO,CAAC,GAAkB;QAClC,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,eAAe,EAAE,CAAC,eAAe;YAC1C,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;IACJ,CAAC;IAES,aAAa,CACrB,GAA8B,EAC9B,OAAqB;QAErB,OAAO,GAAG,CAAC,CAAC,CAAC,EAAC,GAAG,EAAE,OAAO,EAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;IAES,0BAA0B,CAClC,GAAW,EACX,iBAAoC;QAEpC,MAAM,EAAC,KAAK,EAAC,GAAG,iBAAiB,CAAC;QAClC,MAAM,GAAG,GAAG,GAAG,KAAK,IAAI,GAAG,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAC9C,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAES,qBAAqB,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,WAAW,EACX,SAAS,EACT,cAAc,EACd,UAAU,GACG;QACb,MAAM,MAAM,GAAwB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,GAAG,EAAE;YACjE,KAAK,EAAE,wBAAgB;YACvB,MAAM,EAAE,OAAO;SAChB,CAAC,CAAC;QACH,IAAI,aAAa;YAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE9C,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,aAAa,GAAG,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,GAAG,EAAE;gBACjE,KAAK,EAAE,wBAAgB;gBACvB,MAAM,EAAE,EAAC,GAAG,OAAO,EAAE,IAAI,EAAE,WAAW,IAAI,IAAI,EAAC;aAChD,CAAC,CAAC;YACH,IAAI,aAAa;gBAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,eAAe,GAAG,IAAI,CAAC,0BAA0B,CAAC,SAAS,CAAC,GAAG,EAAE;gBACrE,KAAK,EAAE,0BAAkB;gBACzB,MAAM,EAAE,EAAC,GAAG,SAAS,EAAE,KAAK,EAAE,cAAc,IAAI,IAAI,EAAC;aACtD,CAAC,CAAC;YACH,IAAI,eAAe;gBAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,gBAAgB,GAAG,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,GAAG,EAAE;gBACvE,KAAK,EAAE,2BAAmB;gBAC1B,MAAM,EAAE,EAAC,GAAG,UAAU,EAAC;aACxB,CAAC,CAAC;YACH,IAAI,gBAAgB;gBAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAvED,0DAuEC"}

package/lib/converters/openai-platform/completions_usage.d.ts

@@ -0,0 +1,26 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { OpenAIPlatformConverter } from './common';
+export interface OpenAICompletionsUsageItem {
+    _record_id: string;
+    start_time: number;
+    end_time: number;
+    project_id?: string | null;
+    project_name?: string | null;
+    user_id?: string | null;
+    principal_email?: string | null;
+    api_key_id?: string | null;
+    model?: string | null;
+    batch?: boolean | null;
+    service_tier?: string | null;
+    num_model_requests?: number | null;
+    input_tokens?: number | null;
+    input_cached_tokens?: number | null;
+    input_cache_write_tokens?: number | null;
+    input_uncached_tokens?: number | null;
+    output_tokens?: number | null;
+}
+export declare class CompletionsUsage extends OpenAIPlatformConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/openai-platform/completions_usage.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CompletionsUsage = void 0;
+const faros_js_client_1 = require("faros-js-client");
+const common_1 = require("./common");
+class CompletionsUsage extends common_1.OpenAIPlatformConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            common_1.AI_ACCOUNT_MODEL,
+            common_1.AI_PROJECT_MODEL,
+            common_1.AI_PRINCIPAL_MODEL,
+            common_1.AI_CREDENTIAL_MODEL,
+            common_1.AI_USAGE_MODEL,
+        ];
+    }
+    async convert(record, ctx) {
+        const item = record.record.data;
+        const account = this.account(ctx);
+        const project = this.accountEntity(item.project_id, account);
+        const principal = this.accountEntity(item.user_id, account);
+        const credential = this.accountEntity(item.api_key_id, account);
+        const result = this.destinationDimensions({
+            account,
+            project,
+            projectName: item.project_name,
+            principal,
+            principalEmail: item.principal_email,
+            credential,
+        });
+        result.push({
+            model: common_1.AI_USAGE_MODEL,
+            record: {
+                uid: item._record_id,
+                source: this.source,
+                account,
+                project,
+                principal,
+                credential,
+                startedAt: faros_js_client_1.Utils.toDate(item.start_time * 1000),
+                endedAt: faros_js_client_1.Utils.toDate(item.end_time * 1000),
+                model: item.model ?? null,
+                batch: item.batch ?? null,
+                serviceTier: item.service_tier ?? null,
+                requests: item.num_model_requests ?? null,
+                // OpenAI input_tokens is the inclusive total. The following cached,
+                // cache-write, and uncached counters are its components.
+                inputTokens: item.input_tokens ?? null,
+                cachedInputTokens: item.input_cached_tokens ?? null,
+                cacheWriteInputTokens: item.input_cache_write_tokens ?? null,
+                uncachedInputTokens: item.input_uncached_tokens ?? null,
+                outputTokens: item.output_tokens ?? null,
+            },
+        });
+        return result;
+    }
+}
+exports.CompletionsUsage = CompletionsUsage;
+//# sourceMappingURL=completions_usage.js.map

package/lib/converters/openai-platform/completions_usage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"completions_usage.js","sourceRoot":"","sources":["../../../src/converters/openai-platform/completions_usage.ts"],"names":[],"mappings":";;;AACA,qDAAsC;AAGtC,qCAOkB;AAsBlB,MAAa,gBAAiB,SAAQ,gCAAuB;IAA7D;;QACW,sBAAiB,GAAoC;YAC5D,yBAAgB;YAChB,yBAAgB;YAChB,2BAAkB;YAClB,4BAAmB;YACnB,uBAAc;SACf,CAAC;IA8CJ,CAAC;IA5CC,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAkC,CAAC;QAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC;YACxC,OAAO;YACP,OAAO;YACP,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,SAAS;YACT,cAAc,EAAE,IAAI,CAAC,eAAe;YACpC,UAAU;SACX,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,uBAAc;YACrB,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI,CAAC,UAAU;gBACpB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO;gBACP,OAAO;gBACP,SAAS;gBACT,UAAU;gBACV,SAAS,EAAE,uBAAK,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBAC/C,OAAO,EAAE,uBAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBAC3C,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;gBACzB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;gBACzB,WAAW,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;gBACtC,QAAQ,EAAE,IAAI,CAAC,kBAAkB,IAAI,IAAI;gBACzC,oEAAoE;gBACpE,yDAAyD;gBACzD,WAAW,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;gBACtC,iBAAiB,EAAE,IAAI,CAAC,mBAAmB,IAAI,IAAI;gBACnD,qBAAqB,EAAE,IAAI,CAAC,wBAAwB,IAAI,IAAI;gBAC5D,mBAAmB,EAAE,IAAI,CAAC,qBAAqB,IAAI,IAAI;gBACvD,YAAY,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;aACzC;SACF,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AArDD,4CAqDC"}

package/lib/converters/openai-platform/costs.d.ts

@@ -0,0 +1,20 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { OpenAIPlatformConverter } from './common';
+export interface OpenAICostItem {
+    _record_id: string;
+    start_time: number;
+    end_time: number;
+    project_id?: string | null;
+    project_name?: string | null;
+    api_key_id?: string | null;
+    line_item?: string | null;
+    amount?: number | null;
+    currency?: string | null;
+    quantity?: number | null;
+    quantity_unit?: string | null;
+}
+export declare class Costs extends OpenAIPlatformConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/openai-platform/costs.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Costs = void 0;
+const faros_js_client_1 = require("faros-js-client");
+const common_1 = require("./common");
+class Costs extends common_1.OpenAIPlatformConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            common_1.AI_ACCOUNT_MODEL,
+            common_1.AI_PROJECT_MODEL,
+            common_1.AI_CREDENTIAL_MODEL,
+            common_1.AI_COST_MODEL,
+        ];
+    }
+    async convert(record, ctx) {
+        const item = record.record.data;
+        const account = this.account(ctx);
+        const project = this.accountEntity(item.project_id, account);
+        const credential = this.accountEntity(item.api_key_id, account);
+        const result = this.destinationDimensions({
+            account,
+            project,
+            projectName: item.project_name,
+            credential,
+        });
+        result.push({
+            model: common_1.AI_COST_MODEL,
+            record: {
+                uid: item._record_id,
+                source: this.source,
+                account,
+                project,
+                credential,
+                startedAt: faros_js_client_1.Utils.toDate(item.start_time * 1000),
+                endedAt: faros_js_client_1.Utils.toDate(item.end_time * 1000),
+                lineItem: item.line_item ?? null,
+                model: null,
+                amount: item.amount ?? null,
+                currency: item.currency ?? null,
+                quantity: item.quantity ?? null,
+                quantityUnit: item.quantity_unit ?? null,
+            },
+        });
+        return result;
+    }
+}
+exports.Costs = Costs;
+//# sourceMappingURL=costs.js.map

package/lib/converters/openai-platform/costs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"costs.js","sourceRoot":"","sources":["../../../src/converters/openai-platform/costs.ts"],"names":[],"mappings":";;;AACA,qDAAsC;AAGtC,qCAMkB;AAgBlB,MAAa,KAAM,SAAQ,gCAAuB;IAAlD;;QACW,sBAAiB,GAAoC;YAC5D,yBAAgB;YAChB,yBAAgB;YAChB,4BAAmB;YACnB,sBAAa;SACd,CAAC;IAqCJ,CAAC;IAnCC,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAsB,CAAC;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC;YACxC,OAAO;YACP,OAAO;YACP,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,UAAU;SACX,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,sBAAa;YACpB,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI,CAAC,UAAU;gBACpB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO;gBACP,OAAO;gBACP,UAAU;gBACV,SAAS,EAAE,uBAAK,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBAC/C,OAAO,EAAE,uBAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBAC3C,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;gBAChC,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;gBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;gBAC/B,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;gBAC/B,YAAY,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;aACzC;SACF,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AA3CD,sBA2CC"}