@stackone/olap 1.8.0 → 1.10.0

package/dist/index.cjs

@@ -1 +1 @@
-let e=require(`@stackone/redaction`),t=require(`@stackone/utils`),n=require(`@aws-sdk/client-s3`),r=require(`kafkajs`);const i=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],a=10*1024*1024;var o=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,t,r){if(!r?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending to advanced sink`,category:`AdvancedSink`});return}if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}if(`mode`in e&&this.isDataSyncRequest(e)){this.#t?.debug({message:`Data sync request detected, skipping advanced log storage for actionRunId ${t.actionRunId}`,category:`AdvancedSink`});return}try{let i=await this.createS3Item(e,t,r.ttl);await this.#e.send(new n.PutObjectCommand({Bucket:i.Bucket,Key:i.Key,Body:i.Body,ContentType:i.ContentType,Expires:i.Expires,Tagging:i.Tagging}))}catch(e){throw this.#t?.error({message:`Failed to write advanced logs to S3 account ${t.accountSecureId}`,error:e,code:`AdvancedLogWriteError`,category:`AdvancedSink`,context:{actionRunId:t.actionRunId}}),e}}async createS3Item(e,t,n){let{projectSecureId:r,organizationId:i}=t,a=`ttl=30d`;typeof n==`number`&&(n===1?a=`ttl=1d`:n===7&&(a=`ttl=7d`));let o=Math.floor(Date.now()/1e3)+(n??30)*24*60*60,s=process.env.ADVANCED_LOGS_BUCKET;if(!s)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);return{Bucket:s,Key:this.generateS3Key(t,i,r),Body:this.serialize(e,t,o,10485760),ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:a}}generateS3Key(e,t,n){return`actionId`in e?`${t}/${n}/${e.actionRunId}/${e.actionRunId}.json`:`${t}/${n}/${e.actionRunId}/steps/${e.stepId}.json`}serialize(e,t,n,r){return`actionId`in t?JSON.stringify(this.serializeActionResult(e,t,n,r)):JSON.stringify(this.serializeStepResult(e,t))}serializeActionResult(t,n,r,i){let a=(0,e.redactObject)({req:{...t.headers&&{headers:{...t.headers}}},res:{...n.headers&&{headers:{...n.headers}}}},e.CensorType.PARTIAL),o=a.req?.headers,s=a.res?.headers,c=(0,e.redactObject)(t.body,e.CensorType.PARTIAL),l=(0,e.redactObject)(n.body,e.CensorType.PARTIAL),u=this.isBackgroundLog(t);return{data:{request:{id:n.actionRunId,actionId:n.actionId,method:n.httpMethod,headers:this.filterHeaders(o),url:{url:t.url,path:t.pathParams,queryParams:t.queryParams},body:c},response:{statusCode:n.statusCode??500,headers:this.filterHeaders(s),body:l},...u?{isBackgroundLog:!0}:{}},metadata:{...u?{isBackgroundLog:!0}:{},expirationTime:r},expirationTime:r}}serializeStepResult(e,t){return{}}filterHeaders(e){if(!e)return;let t={},n=i.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return(0,t.notMissing)(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const s=(e,t)=>{try{return new n.S3Client(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}},c=(e,t)=>{try{return new r.Kafka(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},l=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var u=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:r.Partitioners.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,n,r){if(!r?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending to log sink`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let i=l(this.buildLog(t,n));this.#n?.debug({message:`Sending to topic ${e}: ${i}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:i}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e,t){if(this.isActionResult(t))return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(t))return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,stepId:t.stepId,skipped:t.skipped,status:t.status,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const d={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function f(e){return{...d,...e}}var p=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=c,getS3Client:t=s,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new u(this.#e,this.#n),this.#i=new o(this.#t,this.#n)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=f(n);if(r?.enabled)try{await this.#r?.send(`actions`,e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=f(n);if(r?.enabled)try{await this.#r?.send(`steps`,e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const m=async({getKafkaClient:e=c,getS3Client:t=s,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new p({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var h=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=m}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=p,exports.OlapClientManager=h;
+let e=require(`@stackone/redaction`),t=require(`@stackone/utils`),n=require(`@aws-sdk/client-s3`),r=require(`kafkajs`);const i=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],a=10*1024*1024;var o=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let r=process.env.ADVANCED_LOGS_BUCKET;if(!r)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let i=`ttl=30d`;typeof t==`number`&&(t===1?i=`ttl=1d`:t===7&&(i=`ttl=7d`));let a=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;try{await this.#e.send(new n.PutObjectCommand({Bucket:r,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(a*1e3),Tagging:i}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,a),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,a),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(t,n,r,i){let a=(0,e.redactObject)({req:{...t.headers&&{headers:{...t.headers}}},res:{...n.headers&&{headers:{...n.headers}}}},e.CensorType.PARTIAL),o=a.req?.headers,s=a.res?.headers,c=(0,e.redactObject)(t.body,e.CensorType.PARTIAL),l=(0,e.redactObject)(n.body,e.CensorType.PARTIAL),u=this.isBackgroundLog(t),d={data:{request:{id:n.actionRunId,actionId:n.actionId,method:n.httpMethod,headers:this.filterHeaders(o),url:{url:t.url,path:t.pathParams,queryParams:t.queryParams},body:c},response:{statusCode:n.statusCode??500,headers:this.filterHeaders(s),body:l},...u?{isBackgroundLog:!0}:{}},metadata:{...u?{isBackgroundLog:!0}:{},expirationTime:r},expirationTime:r};return this.serializeAndLimit(d,i)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,t){let n=JSON.stringify(e);if(Buffer.byteLength(n,`utf8`)>t){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};n=JSON.stringify(t)}return n}filterHeaders(e){if(!e)return;let t={},n=i.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return(0,t.notMissing)(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const s=(e,t)=>{try{return new n.S3Client(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}},c=(e,t)=>{try{return new r.Kafka(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},l=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var u=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:r.Partitioners.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async send(e,t){if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=l(t);this.#n?.debug({message:`Sending to topic ${e}: ${n}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:n}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const d={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function f(e){return{...d,...e}}var p=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=c,getS3Client:t=s,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new u(this.#e,this.#n),this.#i=new o(this.#t,this.#n)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=f(n);if(r?.enabled)try{await this.#r?.sendAction(e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.sendAction(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=f(n);if(r?.enabled)try{await this.#r?.sendStep(e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.sendStep(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const m=async({getKafkaClient:e=c,getS3Client:t=s,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new p({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var h=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=m}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=p,exports.OlapClientManager=h;

package/dist/index.d.cts

@@ -122,15 +122,18 @@ type ActionResult = {
   endTime?: Date;
 };
 type StepInput = {
-  inputs?: unknown;
-};
-type StepResult = {
   actionRunId: string;
+  stepIndex: number;
   stepId: string;
   organizationId: string;
   projectSecureId: string;
   accountSecureId: string;
-  status?: string;
+  inputs?: {
+    [key: string]: unknown;
+  };
+};
+type StepResult = {
+  success?: boolean;
   message?: string;
   outputs?: unknown;
   errors?: unknown;

package/dist/index.d.mts

@@ -122,15 +122,18 @@ type ActionResult = {
   endTime?: Date;
 };
 type StepInput = {
-  inputs?: unknown;
-};
-type StepResult = {
   actionRunId: string;
+  stepIndex: number;
   stepId: string;
   organizationId: string;
   projectSecureId: string;
   accountSecureId: string;
-  status?: string;
+  inputs?: {
+    [key: string]: unknown;
+  };
+};
+type StepResult = {
+  success?: boolean;
   message?: string;
   outputs?: unknown;
   errors?: unknown;

package/dist/index.mjs

@@ -1 +1 @@
-import{CensorType as e,redactObject as t}from"@stackone/redaction";import{notMissing as n}from"@stackone/utils";import{PutObjectCommand as r,S3Client as i}from"@aws-sdk/client-s3";import{Kafka as a,Partitioners as o}from"kafkajs";const s=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`];var c=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,t,n){if(!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending to advanced sink`,category:`AdvancedSink`});return}if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}if(`mode`in e&&this.isDataSyncRequest(e)){this.#t?.debug({message:`Data sync request detected, skipping advanced log storage for actionRunId ${t.actionRunId}`,category:`AdvancedSink`});return}try{let i=await this.createS3Item(e,t,n.ttl);await this.#e.send(new r({Bucket:i.Bucket,Key:i.Key,Body:i.Body,ContentType:i.ContentType,Expires:i.Expires,Tagging:i.Tagging}))}catch(e){throw this.#t?.error({message:`Failed to write advanced logs to S3 account ${t.accountSecureId}`,error:e,code:`AdvancedLogWriteError`,category:`AdvancedSink`,context:{actionRunId:t.actionRunId}}),e}}async createS3Item(e,t,n){let{projectSecureId:r,organizationId:i}=t,a=`ttl=30d`;typeof n==`number`&&(n===1?a=`ttl=1d`:n===7&&(a=`ttl=7d`));let o=Math.floor(Date.now()/1e3)+(n??30)*24*60*60,s=process.env.ADVANCED_LOGS_BUCKET;if(!s)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);return{Bucket:s,Key:this.generateS3Key(t,i,r),Body:this.serialize(e,t,o,10485760),ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:a}}generateS3Key(e,t,n){return`actionId`in e?`${t}/${n}/${e.actionRunId}/${e.actionRunId}.json`:`${t}/${n}/${e.actionRunId}/steps/${e.stepId}.json`}serialize(e,t,n,r){return`actionId`in t?JSON.stringify(this.serializeActionResult(e,t,n,r)):JSON.stringify(this.serializeStepResult(e,t))}serializeActionResult(n,r,i,a){let o=t({req:{...n.headers&&{headers:{...n.headers}}},res:{...r.headers&&{headers:{...r.headers}}}},e.PARTIAL),s=o.req?.headers,c=o.res?.headers,l=t(n.body,e.PARTIAL),u=t(r.body,e.PARTIAL),d=this.isBackgroundLog(n);return{data:{request:{id:r.actionRunId,actionId:r.actionId,method:r.httpMethod,headers:this.filterHeaders(s),url:{url:n.url,path:n.pathParams,queryParams:n.queryParams},body:l},response:{statusCode:r.statusCode??500,headers:this.filterHeaders(c),body:u},...d?{isBackgroundLog:!0}:{}},metadata:{...d?{isBackgroundLog:!0}:{},expirationTime:i},expirationTime:i}}serializeStepResult(e,t){return{}}filterHeaders(e){if(!e)return;let t={},n=s.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return n(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const l=(e,t)=>{try{return new i(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}},u=(e,t)=>{try{return new a(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},d=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var f=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:o.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,n,r){if(!r?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending to log sink`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let i=d(this.buildLog(t,n));this.#n?.debug({message:`Sending to topic ${e}: ${i}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:i}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e,t){if(this.isActionResult(t))return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(t))return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,stepId:t.stepId,skipped:t.skipped,status:t.status,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const p={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function m(e){return{...p,...e}}var h=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=u,getS3Client:t=l,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new f(this.#e,this.#n),this.#i=new c(this.#t,this.#n)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=m(n);if(r?.enabled)try{await this.#r?.send(`actions`,e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=m(n);if(r?.enabled)try{await this.#r?.send(`steps`,e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const g=async({getKafkaClient:e=u,getS3Client:t=l,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new h({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var _=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=g}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{h as OlapClient,_ as OlapClientManager};
+import{CensorType as e,redactObject as t}from"@stackone/redaction";import{notMissing as n}from"@stackone/utils";import{PutObjectCommand as r,S3Client as i}from"@aws-sdk/client-s3";import{Kafka as a,Partitioners as o}from"kafkajs";const s=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],c=10*1024*1024;var l=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let n=process.env.ADVANCED_LOGS_BUCKET;if(!n)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let i=`ttl=30d`;typeof t==`number`&&(t===1?i=`ttl=1d`:t===7&&(i=`ttl=7d`));let a=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;try{await this.#e.send(new r({Bucket:n,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(a*1e3),Tagging:i}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,c),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,c),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(n,r,i,a){let o=t({req:{...n.headers&&{headers:{...n.headers}}},res:{...r.headers&&{headers:{...r.headers}}}},e.PARTIAL),s=o.req?.headers,c=o.res?.headers,l=t(n.body,e.PARTIAL),u=t(r.body,e.PARTIAL),d=this.isBackgroundLog(n),f={data:{request:{id:r.actionRunId,actionId:r.actionId,method:r.httpMethod,headers:this.filterHeaders(s),url:{url:n.url,path:n.pathParams,queryParams:n.queryParams},body:l},response:{statusCode:r.statusCode??500,headers:this.filterHeaders(c),body:u},...d?{isBackgroundLog:!0}:{}},metadata:{...d?{isBackgroundLog:!0}:{},expirationTime:i},expirationTime:i};return this.serializeAndLimit(f,a)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,t){let n=JSON.stringify(e);if(Buffer.byteLength(n,`utf8`)>t){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};n=JSON.stringify(t)}return n}filterHeaders(e){if(!e)return;let t={},n=s.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return n(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const u=(e,t)=>{try{return new i(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}},d=(e,t)=>{try{return new a(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},f=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var p=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:o.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async send(e,t){if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=f(t);this.#n?.debug({message:`Sending to topic ${e}: ${n}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:n}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const m={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function h(e){return{...m,...e}}var g=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=d,getS3Client:t=u,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new p(this.#e,this.#n),this.#i=new l(this.#t,this.#n)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=h(n);if(r?.enabled)try{await this.#r?.sendAction(e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.sendAction(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=h(n);if(r?.enabled)try{await this.#r?.sendStep(e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.sendStep(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const _=async({getKafkaClient:e=d,getS3Client:t=u,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new g({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var v=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=_}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{g as OlapClient,v as OlapClientManager};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stackone/olap",
-    "version": "1.8.0",
+    "version": "1.10.0",
     "description": "",
     "main": "dist/index.cjs",
     "module": "dist/index.mjs",
@@ -42,6 +42,7 @@
     "license": "ISC",
     "dependencies": {
         "@aws-sdk/client-s3": "3.913.0",
+        "@stackone/utils": "*",
         "kafkajs": "2.2.4",
         "date-fns": "2.30.0"
     },