airbyte-faros-destination 0.18.0 → 0.18.2

package/lib/converters/circleci/usage.d.ts

@@ -0,0 +1,12 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { CircleCIConverter } from './common';
+export declare class Usage extends CircleCIConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    private static readonly METRIC_DEFINITION_UID;
+    private static metricDefinitionWritten;
+    private readonly axios;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+    private downloadAndParseCsv;
+    private convertCsvToMetricValues;
+}

package/lib/converters/circleci/usage.js

@@ -0,0 +1,154 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Usage = void 0;
+const sync_1 = require("csv-parse/sync");
+const faros_airbyte_cdk_1 = require("faros-airbyte-cdk");
+const faros_js_client_1 = require("faros-js-client");
+const zlib = __importStar(require("zlib"));
+const common_1 = require("./common");
+// Credit types from CircleCI usage CSV
+const CREDIT_TYPES = [
+    'COMPUTE_CREDITS',
+    'DLC_CREDITS',
+    'USER_CREDITS',
+    'STORAGE_CREDITS',
+    'NETWORK_CREDITS',
+    'LEASE_CREDITS',
+    'LEASE_OVERAGE_CREDITS',
+    'IPRANGES_CREDITS',
+    'TOTAL_CREDITS',
+];
+class Usage extends common_1.CircleCIConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            'faros_MetricDefinition',
+            'faros_MetricValue',
+        ];
+        this.axios = (0, faros_js_client_1.makeAxiosInstanceWithRetry)();
+    }
+    async convert(record, ctx) {
+        var _a;
+        const usage = record.record.data;
+        let res = [];
+        // Write metric definition once
+        if (!Usage.metricDefinitionWritten) {
+            res.push({
+                model: 'faros_MetricDefinition',
+                record: {
+                    uid: Usage.METRIC_DEFINITION_UID,
+                    name: 'CircleCI Job Usage',
+                    description: 'Credit usage metrics by CircleCI jobs',
+                    valueType: {
+                        category: 'Numeric',
+                        detail: 'Credits used per job, by type',
+                    },
+                    scorecardCompatible: false,
+                },
+            });
+            Usage.metricDefinitionWritten = true;
+        }
+        // Only process completed exports with download URLs
+        if (usage.state !== 'completed' || !((_a = usage.download_urls) === null || _a === void 0 ? void 0 : _a.length)) {
+            ctx.logger.info(`Skipping usage export job ${usage.usage_export_job_id} because it is not completed or has no download URLs`);
+            return res;
+        }
+        // Process each download URL (CSV file)
+        for (const downloadUrl of usage.download_urls) {
+            ctx.logger.info(`Processing CSV from job ${usage.usage_export_job_id}`);
+            try {
+                const csvData = await this.downloadAndParseCsv(downloadUrl, ctx);
+                const metricValues = this.convertCsvToMetricValues(csvData);
+                res = res.concat(metricValues);
+            }
+            catch (error) {
+                ctx.logger.error(`Failed to process CSV from job ${usage.usage_export_job_id} at ${downloadUrl}: ${error.message}`);
+            }
+        }
+        return res;
+    }
+    async downloadAndParseCsv(url, ctx) {
+        try {
+            // Download the CSV.gz file
+            const response = await this.axios.get(url, {
+                responseType: 'arraybuffer',
+            });
+            const buffer = Buffer.from(response.data);
+            // Decompress the gzipped CSV
+            const csvContent = zlib.gunzipSync(buffer).toString('utf8');
+            // Parse the CSV
+            const records = (0, sync_1.parse)(csvContent, {
+                columns: true,
+                skip_empty_lines: true,
+                trim: true,
+            });
+            ctx.logger.debug(`Parsed ${records.length} usage records from CSV`);
+            return records;
+        }
+        catch (error) {
+            throw new Error(`Failed to download/parse CSV: ${error.message}`);
+        }
+    }
+    convertCsvToMetricValues(records) {
+        const metricValues = [];
+        for (const record of records) {
+            const computedAt = (0, faros_airbyte_cdk_1.toDate)(record.JOB_RUN_DATE);
+            if (!computedAt) {
+                continue; // Skip records with invalid dates
+            }
+            // Create a metric value for each credit type
+            for (const creditType of CREDIT_TYPES) {
+                const value = record[creditType];
+                if (value && !isNaN(parseFloat(value)) && isFinite(parseFloat(value))) {
+                    metricValues.push({
+                        model: 'faros_MetricValue',
+                        record: {
+                            uid: `${record.JOB_ID}_${creditType.toLowerCase()}`,
+                            value: value,
+                            computedAt: computedAt.toISOString(),
+                            definition: { uid: Usage.METRIC_DEFINITION_UID },
+                        },
+                    });
+                }
+            }
+        }
+        return metricValues;
+    }
+}
+exports.Usage = Usage;
+Usage.METRIC_DEFINITION_UID = 'circleci_job_usage';
+Usage.metricDefinitionWritten = false;
+//# sourceMappingURL=usage.js.map

package/lib/converters/circleci/usage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage.js","sourceRoot":"","sources":["../../../src/converters/circleci/usage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAAqC;AACrC,yDAAwD;AACxD,qDAA2D;AAC3D,2CAA6B;AAG7B,qCAA2C;AAE3C,uCAAuC;AACvC,MAAM,YAAY,GAAG;IACnB,iBAAiB;IACjB,aAAa;IACb,cAAc;IACd,iBAAiB;IACjB,iBAAiB;IACjB,eAAe;IACf,uBAAuB;IACvB,kBAAkB;IAClB,eAAe;CACP,CAAC;AA2BX,MAAa,KAAM,SAAQ,0BAAiB;IAA5C;;QACW,sBAAiB,GAAoC;YAC5D,wBAAwB;YACxB,mBAAmB;SACpB,CAAC;QAIe,UAAK,GAAG,IAAA,4CAA0B,GAAE,CAAC;IAgHxD,CAAC;IA9GC,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;;QAElB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAsB,CAAC;QAEnD,IAAI,GAAG,GAAwB,EAAE,CAAC;QAElC,+BAA+B;QAC/B,IAAI,CAAC,KAAK,CAAC,uBAAuB,EAAE,CAAC;YACnC,GAAG,CAAC,IAAI,CAAC;gBACP,KAAK,EAAE,wBAAwB;gBAC/B,MAAM,EAAE;oBACN,GAAG,EAAE,KAAK,CAAC,qBAAqB;oBAChC,IAAI,EAAE,oBAAoB;oBAC1B,WAAW,EAAE,uCAAuC;oBACpD,SAAS,EAAE;wBACT,QAAQ,EAAE,SAAS;wBACnB,MAAM,EAAE,+BAA+B;qBACxC;oBACD,mBAAmB,EAAE,KAAK;iBAC3B;aACF,CAAC,CAAC;YACH,KAAK,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACvC,CAAC;QAED,oDAAoD;QACpD,IAAI,KAAK,CAAC,KAAK,KAAK,WAAW,IAAI,CAAC,CAAA,MAAA,KAAK,CAAC,aAAa,0CAAE,MAAM,CAAA,EAAE,CAAC;YAChE,GAAG,CAAC,MAAM,CAAC,IAAI,CACb,6BAA6B,KAAK,CAAC,mBAAmB,sDAAsD,CAC7G,CAAC;YACF,OAAO,GAAG,CAAC;QACb,CAAC;QAED,uCAAuC;QACvC,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YAC9C,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC;YACxE,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;gBACjE,MAAM,YAAY,GAAG,IAAI,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC;gBAC5D,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,GAAG,CAAC,MAAM,CAAC,KAAK,CACd,kCAAkC,KAAK,CAAC,mBAAmB,OAAO,WAAW,KAAK,KAAK,CAAC,OAAO,EAAE,CAClG,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,GAAW,EACX,GAAkB;QAElB,IAAI,CAAC;YACH,2BAA2B;YAC3B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;gBACzC,YAAY,EAAE,aAAa;aAC5B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAE1C,6BAA6B;YAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAE5D,gBAAgB;YAChB,MAAM,OAAO,GAAG,IAAA,YAAK,EAAC,UAAU,EAAE;gBAChC,OAAO,EAAE,IAAI;gBACb,gBAAgB,EAAE,IAAI;gBACtB,IAAI,EAAE,IAAI;aACX,CAAkB,CAAC;YAEpB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,OAAO,CAAC,MAAM,yBAAyB,CAAC,CAAC;YACpE,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,wBAAwB,CAC9B,OAAsB;QAEtB,MAAM,YAAY,GAAwB,EAAE,CAAC;QAE7C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,IAAA,0BAAM,EAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC/C,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,SAAS,CAAC,kCAAkC;YAC9C,CAAC;YAED,6CAA6C;YAC7C,KAAK,MAAM,UAAU,IAAI,YAAY,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;gBACjC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACtE,YAAY,CAAC,IAAI,CAAC;wBAChB,KAAK,EAAE,mBAAmB;wBAC1B,MAAM,EAAE;4BACN,GAAG,EAAE,GAAG,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,WAAW,EAAE,EAAE;4BACnD,KAAK,EAAE,KAAK;4BACZ,UAAU,EAAE,UAAU,CAAC,WAAW,EAAE;4BACpC,UAAU,EAAE,EAAC,GAAG,EAAE,KAAK,CAAC,qBAAqB,EAAC;yBAC/C;qBACF,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;;AAvHH,sBAwHC;AAlHyB,2BAAqB,GAAG,oBAAoB,AAAvB,CAAwB;AACtD,6BAAuB,GAAG,KAAK,AAAR,CAAS"}