@stackone/olap 1.5.0 → 1.6.0

package/dist/index.cjs

@@ -1 +1 @@
-let e=require(`@stackone/redaction`),t=require(`@stackone/utils`),n=require(`@aws-sdk/client-s3`),r=require(`date-fns`),i=require(`kafkajs`);const a=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}},o=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],s=[`application/octet-stream`,`application/pdf`,`application/msword`,`application/vnd.openxmlformats-officedocument.wordprocessingml.document`,`application/vnd.ms-excel`,`application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`,`application/zip`,`application/x-zip-compressed`],c=[`image/`,`audio/`,`video/`,`application/vnd.`,`binary`],l=10*1024*1024;var u=class{#e;constructor({logger:e}={}){this.#e=e}async write(e,t,n,r){try{let i=r?.ttl??30,a=await this.generateS3Command(t,n,i);await e.send(a)}catch(e){throw this.#e?.error({message:`Failed to write advanced logs to S3`,category:`AdvancedWriter`,code:`AdvancedWriteError`,error:e}),e}}generateS3Key(e,t,n,r){let i=`${t}/${n}/${e}`;return r?`${i}/steps/${r}.json`:`${i}/${e}.json`}async generateS3Command(e,t,i){let a=process.env.ADVANCED_LOGS_BUCKET;if(!a)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let o=this.generateS3Key(t.actionRunId,t.organizationId,t.projectSecureId),s=(0,r.getUnixTime)((0,r.addDays)(new Date,i)),c=this.serialize(e,s,10485760),l=`ttl=30d`;return i===1?l=`ttl=1d`:i===7&&(l=`ttl=7d`),new n.PutObjectCommand({Bucket:a,Key:o,Body:c,ContentType:`application/json`,Expires:new Date(s*1e3),Tagging:l})}filterHeaders(e){if(!e)return;let t={},n=o.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return(0,t.notMissing)(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}serialize(e,n,r){let i=e.response?.body;((0,t.isString)(i)&&i.length>r||Buffer.isBuffer(i)&&i.byteLength>r||i&&(0,t.isObject)(i)&&!Array.isArray(i)&&Object.values(i).some(e=>(0,t.isString)(e)&&e.length>r||Buffer.isBuffer(e)&&e.byteLength>r))&&(e={...e,response:{...e.response,body:{error:`Error.TOO_LARGE`}}});let a=JSON.stringify({data:e,expirationTime:n});if(Buffer.byteLength(a,`utf8`)>r){let t={...e,response:{...e.response,body:{error:`Error.TOO_LARGE`}}};a=JSON.stringify({data:t,expirationTime:n})}return a}processContent(e,t,n){let r=this.getContentType(e);if(!r)return a(t);let i=r.toLowerCase().split(`;`)[0].trim();return s.includes(i)||c.some(e=>i.startsWith(e))?{type:`binary`,action:n,contentType:r}:a(t)}};const d=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],f=10*1024*1024;var p=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n??new u({logger:t})}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,t,r){if(!r?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending to advanced sink`,category:`AdvancedSink`});return}if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}if(`mode`in t&&this.isDataSyncRequest(t)){this.#t?.debug({message:`Data sync request detected, skipping advanced log storage for actionRunId ${t.actionRunId}`,category:`AdvancedSink`});return}try{let i=await this.createS3Item(e,t,r.ttl);await this.#e.send(new n.PutObjectCommand({Bucket:i.Bucket,Key:i.Key,Body:i.Body,ContentType:i.ContentType,Expires:i.Expires,Tagging:i.Tagging}))}catch(e){throw this.#t?.error({message:`Failed to write advanced logs to S3 account ${t.accountSecureId}`,error:e,code:`AdvancedLogWriteError`,category:`AdvancedSink`,context:{actionRunId:t.actionRunId}}),e}}async createS3Item(e,t,n){let{projectSecureId:r,organizationId:i}=t,a=`ttl=30d`;typeof n==`number`&&(n===1?a=`ttl=1d`:n===7&&(a=`ttl=7d`));let o=Math.floor(Date.now()/1e3)+(n??30)*24*60*60,s=process.env.ADVANCED_LOGS_BUCKET;if(!s)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);return{Bucket:s,Key:this.generateS3Key(t,i,r),Body:this.serialize(e,t,o,10485760),ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:a}}generateS3Key(e,t,n){return`actionId`in e?`${t}/${n}/${e.actionRunId}/${e.actionRunId}.json`:`${t}/${n}/${e.actionRunId}/steps/${e.stepId}.json`}serialize(e,t,n,r){return`actionId`in t?JSON.stringify(this.serializeActionResult(e,t,n,r)):JSON.stringify(this.serializeStepResult(e,t))}serializeActionResult(t,n,r,i){let a=(0,e.redactObject)({req:{...t.headers&&{headers:{...t.headers}}},res:{...n.headers&&{headers:{...n.headers}}}},e.CensorType.PARTIAL),o=a.req?.headers,s=a.res?.headers,c=(0,e.redactObject)(t.body,e.CensorType.PARTIAL),l=(0,e.redactObject)(n.body,e.CensorType.PARTIAL),u=this.isBackgroundLog(n);return{data:{request:{id:n.actionRunId,actionId:n.actionId,method:n.httpMethod,headers:this.filterHeaders(o),url:{url:t.url,path:t.pathParams,queryParams:t.queryParams},body:c},response:{statusCode:this.getStatusCode(n.status),headers:this.filterHeaders(s),body:l},...u?{isBackgroundLog:!0}:{}},metadata:{...u?{isBackgroundLog:!0}:{},expirationTime:r},expirationTime:r}}serializeStepResult(e,t){return{}}getStatusCode(e){let t=parseInt(e??``,10);return isNaN(t)?e===`success`?200:500:t}filterHeaders(e){if(!e)return;let t={},n=d.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return(0,t.notMissing)(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const m=(e,t)=>{try{return new n.S3Client(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}},h=(e,t)=>{try{return new i.Kafka(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}};var g=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:i.Partitioners.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping send to topic ${e}`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let r=a(this.buildLog(t));this.#n?.debug({message:`Sending to topic ${e}: ${r}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:r}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e){if(this.isActionResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,actionId:e.actionId,organizationId:e.organizationId,projectSecureId:e.projectSecureId,accountSecureId:e.accountSecureId,mode:e.mode,connectorKey:e.connectorKey,connectorVersion:e.connectorVersion,actionType:e.actionType,category:e.category,originOwnerId:e.originOwnerId,originOwnerName:e.originOwnerName,httpMethod:e.httpMethod,url:e.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,status:e.status,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepId:e.stepId,skipped:e.skipped,status:e.status,message:e.message,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const _={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function v(e){return{..._,...e}}var y=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=h,getS3Client:t=m,kafkaClientConfig:n,s3ClientConfig:r,logger:i,advancedWriter:a}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new g(this.#e,this.#n),this.#i=new p(this.#t,this.#n,a)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=v(n);if(r?.enabled)try{await this.#r?.send(`actions`,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=v(n);if(r?.enabled)try{await this.#r?.send(`steps`,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const b=async({getKafkaClient:e=h,getS3Client:t=m,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new y({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var x=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=b}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=y,exports.OlapClientManager=x;
+let e=require(`@stackone/redaction`),t=require(`@stackone/utils`),n=require(`@aws-sdk/client-s3`),r=require(`date-fns`),i=require(`kafkajs`);const a=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}},o=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],s=[`application/octet-stream`,`application/pdf`,`application/msword`,`application/vnd.openxmlformats-officedocument.wordprocessingml.document`,`application/vnd.ms-excel`,`application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`,`application/zip`,`application/x-zip-compressed`],c=[`image/`,`audio/`,`video/`,`application/vnd.`,`binary`],l=10*1024*1024;var u=class{#e;constructor({logger:e}={}){this.#e=e}async write(e,t,n,r){try{let i=r?.ttl??30,a=await this.generateS3Command(t,n,i);await e.send(a)}catch(e){throw this.#e?.error({message:`Failed to write advanced logs to S3`,category:`AdvancedWriter`,code:`AdvancedWriteError`,error:e}),e}}generateS3Key(e,t,n,r){let i=`${t}/${n}/${e}`;return r?`${i}/steps/${r}.json`:`${i}/${e}.json`}async generateS3Command(e,t,i){let a=process.env.ADVANCED_LOGS_BUCKET;if(!a)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let o=this.generateS3Key(t.actionRunId,t.organizationId,t.projectSecureId),s=(0,r.getUnixTime)((0,r.addDays)(new Date,i)),c=this.serialize(e,s,10485760),l=`ttl=30d`;return i===1?l=`ttl=1d`:i===7&&(l=`ttl=7d`),new n.PutObjectCommand({Bucket:a,Key:o,Body:c,ContentType:`application/json`,Expires:new Date(s*1e3),Tagging:l})}filterHeaders(e){if(!e)return;let t={},n=o.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return(0,t.notMissing)(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}serialize(e,n,r){let i=e.response?.body;((0,t.isString)(i)&&i.length>r||Buffer.isBuffer(i)&&i.byteLength>r||i&&(0,t.isObject)(i)&&!Array.isArray(i)&&Object.values(i).some(e=>(0,t.isString)(e)&&e.length>r||Buffer.isBuffer(e)&&e.byteLength>r))&&(e={...e,response:{...e.response,body:{error:`Error.TOO_LARGE`}}});let a=JSON.stringify({data:e,expirationTime:n});if(Buffer.byteLength(a,`utf8`)>r){let t={...e,response:{...e.response,body:{error:`Error.TOO_LARGE`}}};a=JSON.stringify({data:t,expirationTime:n})}return a}processContent(e,t,n){let r=this.getContentType(e);if(!r)return a(t);let i=r.toLowerCase().split(`;`)[0].trim();return s.includes(i)||c.some(e=>i.startsWith(e))?{type:`binary`,action:n,contentType:r}:a(t)}};const d=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],f=10*1024*1024;var p=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n??new u({logger:t})}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,t,r){if(!r?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending to advanced sink`,category:`AdvancedSink`});return}if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}if(`mode`in t&&this.isDataSyncRequest(t)){this.#t?.debug({message:`Data sync request detected, skipping advanced log storage for actionRunId ${t.actionRunId}`,category:`AdvancedSink`});return}try{let i=await this.createS3Item(e,t,r.ttl);await this.#e.send(new n.PutObjectCommand({Bucket:i.Bucket,Key:i.Key,Body:i.Body,ContentType:i.ContentType,Expires:i.Expires,Tagging:i.Tagging}))}catch(e){throw this.#t?.error({message:`Failed to write advanced logs to S3 account ${t.accountSecureId}`,error:e,code:`AdvancedLogWriteError`,category:`AdvancedSink`,context:{actionRunId:t.actionRunId}}),e}}async createS3Item(e,t,n){let{projectSecureId:r,organizationId:i}=t,a=`ttl=30d`;typeof n==`number`&&(n===1?a=`ttl=1d`:n===7&&(a=`ttl=7d`));let o=Math.floor(Date.now()/1e3)+(n??30)*24*60*60,s=process.env.ADVANCED_LOGS_BUCKET;if(!s)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);return{Bucket:s,Key:this.generateS3Key(t,i,r),Body:this.serialize(e,t,o,10485760),ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:a}}generateS3Key(e,t,n){return`actionId`in e?`${t}/${n}/${e.actionRunId}/${e.actionRunId}.json`:`${t}/${n}/${e.actionRunId}/steps/${e.stepId}.json`}serialize(e,t,n,r){return`actionId`in t?JSON.stringify(this.serializeActionResult(e,t,n,r)):JSON.stringify(this.serializeStepResult(e,t))}serializeActionResult(t,n,r,i){let a=(0,e.redactObject)({req:{...t.headers&&{headers:{...t.headers}}},res:{...n.headers&&{headers:{...n.headers}}}},e.CensorType.PARTIAL),o=a.req?.headers,s=a.res?.headers,c=(0,e.redactObject)(t.body,e.CensorType.PARTIAL),l=(0,e.redactObject)(n.body,e.CensorType.PARTIAL),u=this.isBackgroundLog(n);return{data:{request:{id:n.actionRunId,actionId:n.actionId,method:n.httpMethod,headers:this.filterHeaders(o),url:{url:t.url,path:t.pathParams,queryParams:t.queryParams},body:c},response:{statusCode:n.statusCode??500,headers:this.filterHeaders(s),body:l},...u?{isBackgroundLog:!0}:{}},metadata:{...u?{isBackgroundLog:!0}:{},expirationTime:r},expirationTime:r}}serializeStepResult(e,t){return{}}filterHeaders(e){if(!e)return;let t={},n=d.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return(0,t.notMissing)(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const m=(e,t)=>{try{return new n.S3Client(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}},h=(e,t)=>{try{return new i.Kafka(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}};var g=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:i.Partitioners.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,n,r){if(!r?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending to log sink`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let i=a(this.buildLog(t,n));this.#n?.debug({message:`Sending to topic ${e}: ${i}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:i}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e,t){if(this.isActionResult(t))return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:t.organizationId,projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:t.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(t))return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,stepId:t.stepId,skipped:t.skipped,status:t.status,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const _={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function v(e){return{..._,...e}}var y=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=h,getS3Client:t=m,kafkaClientConfig:n,s3ClientConfig:r,logger:i,advancedWriter:a}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new g(this.#e,this.#n),this.#i=new p(this.#t,this.#n,a)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=v(n);if(r?.enabled)try{await this.#r?.send(`actions`,e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=v(n);if(r?.enabled)try{await this.#r?.send(`steps`,e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const b=async({getKafkaClient:e=h,getS3Client:t=m,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new y({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var x=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=b}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=y,exports.OlapClientManager=x;

package/dist/index.d.cts

@@ -90,6 +90,9 @@ type ActionInput = {
   queryParams?: unknown;
   body?: unknown;
   headers?: Record<string, string>;
+  sourceId?: string;
+  sourceType?: string;
+  sourceValue?: string;
 };
 type ActionResult = {
   actionRunId: string;
@@ -107,13 +110,11 @@ type ActionResult = {
   httpMethod?: string;
   url?: string;
   path?: string;
-  sourceId?: string;
-  sourceType?: string;
-  sourceValue?: string;
   resource?: string;
   subResource?: string;
   childResource?: string;
-  status?: string;
+  success?: boolean;
+  statusCode?: number;
   message?: string;
   headers?: Record<string, string>;
   body?: unknown;

package/dist/index.d.mts

@@ -90,6 +90,9 @@ type ActionInput = {
   queryParams?: unknown;
   body?: unknown;
   headers?: Record<string, string>;
+  sourceId?: string;
+  sourceType?: string;
+  sourceValue?: string;
 };
 type ActionResult = {
   actionRunId: string;
@@ -107,13 +110,11 @@ type ActionResult = {
   httpMethod?: string;
   url?: string;
   path?: string;
-  sourceId?: string;
-  sourceType?: string;
-  sourceValue?: string;
   resource?: string;
   subResource?: string;
   childResource?: string;
-  status?: string;
+  success?: boolean;
+  statusCode?: number;
   message?: string;
   headers?: Record<string, string>;
   body?: unknown;

package/dist/index.mjs

@@ -1 +1 @@
-import{CensorType as e,redactObject as t}from"@stackone/redaction";import{isObject as n,isString as r,notMissing as i}from"@stackone/utils";import{PutObjectCommand as a,S3Client as o}from"@aws-sdk/client-s3";import{addDays as s,getUnixTime as c}from"date-fns";import{Kafka as l,Partitioners as u}from"kafkajs";const d=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}},f=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],p=[`application/octet-stream`,`application/pdf`,`application/msword`,`application/vnd.openxmlformats-officedocument.wordprocessingml.document`,`application/vnd.ms-excel`,`application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`,`application/zip`,`application/x-zip-compressed`],m=[`image/`,`audio/`,`video/`,`application/vnd.`,`binary`];var h=class{#e;constructor({logger:e}={}){this.#e=e}async write(e,t,n,r){try{let i=r?.ttl??30,a=await this.generateS3Command(t,n,i);await e.send(a)}catch(e){throw this.#e?.error({message:`Failed to write advanced logs to S3`,category:`AdvancedWriter`,code:`AdvancedWriteError`,error:e}),e}}generateS3Key(e,t,n,r){let i=`${t}/${n}/${e}`;return r?`${i}/steps/${r}.json`:`${i}/${e}.json`}async generateS3Command(e,t,n){let r=process.env.ADVANCED_LOGS_BUCKET;if(!r)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let i=this.generateS3Key(t.actionRunId,t.organizationId,t.projectSecureId),o=c(s(new Date,n)),l=this.serialize(e,o,10485760),u=`ttl=30d`;return n===1?u=`ttl=1d`:n===7&&(u=`ttl=7d`),new a({Bucket:r,Key:i,Body:l,ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:u})}filterHeaders(e){if(!e)return;let t={},n=f.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return i(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}serialize(e,t,i){let a=e.response?.body;(r(a)&&a.length>i||Buffer.isBuffer(a)&&a.byteLength>i||a&&n(a)&&!Array.isArray(a)&&Object.values(a).some(e=>r(e)&&e.length>i||Buffer.isBuffer(e)&&e.byteLength>i))&&(e={...e,response:{...e.response,body:{error:`Error.TOO_LARGE`}}});let o=JSON.stringify({data:e,expirationTime:t});if(Buffer.byteLength(o,`utf8`)>i){let n={...e,response:{...e.response,body:{error:`Error.TOO_LARGE`}}};o=JSON.stringify({data:n,expirationTime:t})}return o}processContent(e,t,n){let r=this.getContentType(e);if(!r)return d(t);let i=r.toLowerCase().split(`;`)[0].trim();return p.includes(i)||m.some(e=>i.startsWith(e))?{type:`binary`,action:n,contentType:r}:d(t)}};const g=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`];var _=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n??new h({logger:t})}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,t,n){if(!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending to advanced sink`,category:`AdvancedSink`});return}if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}if(`mode`in t&&this.isDataSyncRequest(t)){this.#t?.debug({message:`Data sync request detected, skipping advanced log storage for actionRunId ${t.actionRunId}`,category:`AdvancedSink`});return}try{let r=await this.createS3Item(e,t,n.ttl);await this.#e.send(new a({Bucket:r.Bucket,Key:r.Key,Body:r.Body,ContentType:r.ContentType,Expires:r.Expires,Tagging:r.Tagging}))}catch(e){throw this.#t?.error({message:`Failed to write advanced logs to S3 account ${t.accountSecureId}`,error:e,code:`AdvancedLogWriteError`,category:`AdvancedSink`,context:{actionRunId:t.actionRunId}}),e}}async createS3Item(e,t,n){let{projectSecureId:r,organizationId:i}=t,a=`ttl=30d`;typeof n==`number`&&(n===1?a=`ttl=1d`:n===7&&(a=`ttl=7d`));let o=Math.floor(Date.now()/1e3)+(n??30)*24*60*60,s=process.env.ADVANCED_LOGS_BUCKET;if(!s)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);return{Bucket:s,Key:this.generateS3Key(t,i,r),Body:this.serialize(e,t,o,10485760),ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:a}}generateS3Key(e,t,n){return`actionId`in e?`${t}/${n}/${e.actionRunId}/${e.actionRunId}.json`:`${t}/${n}/${e.actionRunId}/steps/${e.stepId}.json`}serialize(e,t,n,r){return`actionId`in t?JSON.stringify(this.serializeActionResult(e,t,n,r)):JSON.stringify(this.serializeStepResult(e,t))}serializeActionResult(n,r,i,a){let o=t({req:{...n.headers&&{headers:{...n.headers}}},res:{...r.headers&&{headers:{...r.headers}}}},e.PARTIAL),s=o.req?.headers,c=o.res?.headers,l=t(n.body,e.PARTIAL),u=t(r.body,e.PARTIAL),d=this.isBackgroundLog(r);return{data:{request:{id:r.actionRunId,actionId:r.actionId,method:r.httpMethod,headers:this.filterHeaders(s),url:{url:n.url,path:n.pathParams,queryParams:n.queryParams},body:l},response:{statusCode:this.getStatusCode(r.status),headers:this.filterHeaders(c),body:u},...d?{isBackgroundLog:!0}:{}},metadata:{...d?{isBackgroundLog:!0}:{},expirationTime:i},expirationTime:i}}serializeStepResult(e,t){return{}}getStatusCode(e){let t=parseInt(e??``,10);return isNaN(t)?e===`success`?200:500:t}filterHeaders(e){if(!e)return;let t={},n=g.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return i(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const v=(e,t)=>{try{return new o(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}},y=(e,t)=>{try{return new l(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}};var b=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:u.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping send to topic ${e}`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let r=d(this.buildLog(t));this.#n?.debug({message:`Sending to topic ${e}: ${r}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:r}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e){if(this.isActionResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,actionId:e.actionId,organizationId:e.organizationId,projectSecureId:e.projectSecureId,accountSecureId:e.accountSecureId,mode:e.mode,connectorKey:e.connectorKey,connectorVersion:e.connectorVersion,actionType:e.actionType,category:e.category,originOwnerId:e.originOwnerId,originOwnerName:e.originOwnerName,httpMethod:e.httpMethod,url:e.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,status:e.status,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepId:e.stepId,skipped:e.skipped,status:e.status,message:e.message,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const x={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function S(e){return{...x,...e}}var C=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=y,getS3Client:t=v,kafkaClientConfig:n,s3ClientConfig:r,logger:i,advancedWriter:a}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new b(this.#e,this.#n),this.#i=new _(this.#t,this.#n,a)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=S(n);if(r?.enabled)try{await this.#r?.send(`actions`,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=S(n);if(r?.enabled)try{await this.#r?.send(`steps`,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const w=async({getKafkaClient:e=y,getS3Client:t=v,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new C({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var T=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=w}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{C as OlapClient,T as OlapClientManager};
+import{CensorType as e,redactObject as t}from"@stackone/redaction";import{isObject as n,isString as r,notMissing as i}from"@stackone/utils";import{PutObjectCommand as a,S3Client as o}from"@aws-sdk/client-s3";import{addDays as s,getUnixTime as c}from"date-fns";import{Kafka as l,Partitioners as u}from"kafkajs";const d=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}},f=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],p=[`application/octet-stream`,`application/pdf`,`application/msword`,`application/vnd.openxmlformats-officedocument.wordprocessingml.document`,`application/vnd.ms-excel`,`application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`,`application/zip`,`application/x-zip-compressed`],m=[`image/`,`audio/`,`video/`,`application/vnd.`,`binary`];var h=class{#e;constructor({logger:e}={}){this.#e=e}async write(e,t,n,r){try{let i=r?.ttl??30,a=await this.generateS3Command(t,n,i);await e.send(a)}catch(e){throw this.#e?.error({message:`Failed to write advanced logs to S3`,category:`AdvancedWriter`,code:`AdvancedWriteError`,error:e}),e}}generateS3Key(e,t,n,r){let i=`${t}/${n}/${e}`;return r?`${i}/steps/${r}.json`:`${i}/${e}.json`}async generateS3Command(e,t,n){let r=process.env.ADVANCED_LOGS_BUCKET;if(!r)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let i=this.generateS3Key(t.actionRunId,t.organizationId,t.projectSecureId),o=c(s(new Date,n)),l=this.serialize(e,o,10485760),u=`ttl=30d`;return n===1?u=`ttl=1d`:n===7&&(u=`ttl=7d`),new a({Bucket:r,Key:i,Body:l,ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:u})}filterHeaders(e){if(!e)return;let t={},n=f.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return i(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}serialize(e,t,i){let a=e.response?.body;(r(a)&&a.length>i||Buffer.isBuffer(a)&&a.byteLength>i||a&&n(a)&&!Array.isArray(a)&&Object.values(a).some(e=>r(e)&&e.length>i||Buffer.isBuffer(e)&&e.byteLength>i))&&(e={...e,response:{...e.response,body:{error:`Error.TOO_LARGE`}}});let o=JSON.stringify({data:e,expirationTime:t});if(Buffer.byteLength(o,`utf8`)>i){let n={...e,response:{...e.response,body:{error:`Error.TOO_LARGE`}}};o=JSON.stringify({data:n,expirationTime:t})}return o}processContent(e,t,n){let r=this.getContentType(e);if(!r)return d(t);let i=r.toLowerCase().split(`;`)[0].trim();return p.includes(i)||m.some(e=>i.startsWith(e))?{type:`binary`,action:n,contentType:r}:d(t)}};const g=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`];var _=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n??new h({logger:t})}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,t,n){if(!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending to advanced sink`,category:`AdvancedSink`});return}if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}if(`mode`in t&&this.isDataSyncRequest(t)){this.#t?.debug({message:`Data sync request detected, skipping advanced log storage for actionRunId ${t.actionRunId}`,category:`AdvancedSink`});return}try{let r=await this.createS3Item(e,t,n.ttl);await this.#e.send(new a({Bucket:r.Bucket,Key:r.Key,Body:r.Body,ContentType:r.ContentType,Expires:r.Expires,Tagging:r.Tagging}))}catch(e){throw this.#t?.error({message:`Failed to write advanced logs to S3 account ${t.accountSecureId}`,error:e,code:`AdvancedLogWriteError`,category:`AdvancedSink`,context:{actionRunId:t.actionRunId}}),e}}async createS3Item(e,t,n){let{projectSecureId:r,organizationId:i}=t,a=`ttl=30d`;typeof n==`number`&&(n===1?a=`ttl=1d`:n===7&&(a=`ttl=7d`));let o=Math.floor(Date.now()/1e3)+(n??30)*24*60*60,s=process.env.ADVANCED_LOGS_BUCKET;if(!s)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);return{Bucket:s,Key:this.generateS3Key(t,i,r),Body:this.serialize(e,t,o,10485760),ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:a}}generateS3Key(e,t,n){return`actionId`in e?`${t}/${n}/${e.actionRunId}/${e.actionRunId}.json`:`${t}/${n}/${e.actionRunId}/steps/${e.stepId}.json`}serialize(e,t,n,r){return`actionId`in t?JSON.stringify(this.serializeActionResult(e,t,n,r)):JSON.stringify(this.serializeStepResult(e,t))}serializeActionResult(n,r,i,a){let o=t({req:{...n.headers&&{headers:{...n.headers}}},res:{...r.headers&&{headers:{...r.headers}}}},e.PARTIAL),s=o.req?.headers,c=o.res?.headers,l=t(n.body,e.PARTIAL),u=t(r.body,e.PARTIAL),d=this.isBackgroundLog(r);return{data:{request:{id:r.actionRunId,actionId:r.actionId,method:r.httpMethod,headers:this.filterHeaders(s),url:{url:n.url,path:n.pathParams,queryParams:n.queryParams},body:l},response:{statusCode:r.statusCode??500,headers:this.filterHeaders(c),body:u},...d?{isBackgroundLog:!0}:{}},metadata:{...d?{isBackgroundLog:!0}:{},expirationTime:i},expirationTime:i}}serializeStepResult(e,t){return{}}filterHeaders(e){if(!e)return;let t={},n=g.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return i(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const v=(e,t)=>{try{return new o(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}},y=(e,t)=>{try{return new l(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}};var b=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:u.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,n,r){if(!r?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending to log sink`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let i=d(this.buildLog(t,n));this.#n?.debug({message:`Sending to topic ${e}: ${i}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:i}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e,t){if(this.isActionResult(t))return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:t.organizationId,projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:t.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(t))return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,stepId:t.stepId,skipped:t.skipped,status:t.status,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const x={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function S(e){return{...x,...e}}var C=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=y,getS3Client:t=v,kafkaClientConfig:n,s3ClientConfig:r,logger:i,advancedWriter:a}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new b(this.#e,this.#n),this.#i=new _(this.#t,this.#n,a)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=S(n);if(r?.enabled)try{await this.#r?.send(`actions`,e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=S(n);if(r?.enabled)try{await this.#r?.send(`steps`,e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const w=async({getKafkaClient:e=y,getS3Client:t=v,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new C({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var T=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=w}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{C as OlapClient,T as OlapClientManager};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stackone/olap",
-    "version": "1.5.0",
+    "version": "1.6.0",
     "description": "",
     "main": "dist/index.cjs",
     "module": "dist/index.mjs",