@stackone/olap 1.16.0 → 1.17.1

package/dist/index.cjs

@@ -1 +1 @@
-var e=Object.defineProperty,__name=(t,n)=>e(t,`name`,{value:n,configurable:!0});let t=require(`@stackone/redaction`),n=require(`@stackone/utils`),r=require(`@aws-sdk/client-s3`),i=require(`kafkajs`);const a=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],o=[`refresh_authentication`],s=10*1024*1024;var AdvancedSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send action`,category:`AdvancedSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}if(n.errorsOnly&&t.success){this.#t?.debug({message:`Advanced sink errorsOnly is enabled, skipping successful action for ${e.mode||`action`}`,category:`AdvancedSink`});return}if(this.isBackgroundLog(e)&&!n.includeBackground){this.#t?.debug({message:`Advanced sink is configured to exclude background logs, skipping action with mode ${e.mode}`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send step`,category:`AdvancedSink`,context:{actionRunId:e.actionRunId,success:t.success,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let n=process.env.ADVANCED_LOGS_BUCKET;if(!n)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let i=`ttl=30d`;typeof t==`number`&&(t===1?i=`ttl=1d`:t===7&&(i=`ttl=7d`));let a=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;this.#t?.debug({message:`Storing advanced log in S3 bucket ${n} with ttl of ${i}`,category:`AdvancedSink`,context:{bucket:n,key:e.Key,ttlTag:i,expiresAt:new Date(a*1e3).toISOString()}});try{await this.#e.send(new r.PutObjectCommand({Bucket:n,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(a*1e3),Tagging:i}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,s),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,s),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(e,n,r,i){let a=(0,t.redactObject)({req:{...e.headers&&{headers:{...e.headers}}},res:{...n.headers&&{headers:{...n.headers}}}},t.CensorType.PARTIAL),o=a.req?.headers,s=a.res?.headers,c=(0,t.redactObject)(e.body,t.CensorType.PARTIAL),l=(0,t.redactObject)(n.body,t.CensorType.PARTIAL),u=this.isBackgroundLog(e),d={data:{request:{id:n.actionRunId,actionId:n.actionId,method:n.httpMethod,headers:this.filterHeaders(o),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:c},response:{statusCode:n.statusCode??500,headers:this.filterHeaders(s),body:l},...u?{isBackgroundLog:!0}:{}},metadata:{...u?{isBackgroundLog:!0}:{},expirationTime:r},expirationTime:r};return this.serializeAndLimit(d,i)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,t){let n=JSON.stringify(e);if(Buffer.byteLength(n,`utf8`)>t){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};n=JSON.stringify(t)}return n}filterHeaders(e){if(!e)return;let t={},n=a.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return(0,n.notMissing)(e.mode)&&o.includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const buildS3Client=(e,t)=>{try{return new r.S3Client(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}};var DefenderSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, defender sink will not function`,category:`DefenderSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Defender sink called to send action`,category:`DefenderSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled}}}),!n?.enabled){this.#t?.debug({message:`Defender sink is disabled, skipping sending action to defender sink`,category:`DefenderSink`});return}let r=this.createActionS3(t);await this.send(r)}async send(e){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to defender sink`,category:`DefenderSink`});return}let t=process.env.DEFENDER_LOGS_BUCKET;if(!t)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);this.#t?.debug({message:`Storing defender log in S3 bucket ${t}`,category:`DefenderSink`,context:{bucket:t,key:e.Key}});try{await this.#e.send(new r.PutObjectCommand({Bucket:t,Key:e.Key,Body:e.Body,ContentType:e.ContentType}))}catch(t){throw this.#t?.error({message:`Failed to write defender logs to S3 at ${e.Key}`,error:t,code:`DefenderLogWriteError`,category:`DefenderSink`}),t}}createActionS3(e){return{Key:`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/defender.json`,Body:this.serializeDefenderContext(e),ContentType:`application/json`}}serializeDefenderContext(e){let t=e.defenderContext;return t?JSON.stringify(t):JSON.stringify({data:null,metadata:null})}};const buildKafkaClient=(e,t)=>{try{return new i.Kafka(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},safeSerialize=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var LogsSink=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:i.Partitioners.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async send(e,t){if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=safeSerialize(t);this.#n?.debug({message:`Sending to topic ${e}: ${n}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:n}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,riskLevel:t.defenderContext?.riskLevel,tier2Score:t.defenderContext?.tier2Score,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const c={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1,includeBackground:!1},defender:{enabled:!0}};function resolveOlapOptions(e){return{logs:{...c.logs,...e?.logs},advanced:{...c.advanced,...e?.advanced},defender:{...c.defender,...e?.defender}}}var OlapClient=class{#e;#t;#n;#r;#i;#a;constructor({getKafkaClient:e=buildKafkaClient,getS3Client:t=buildS3Client,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new LogsSink(this.#e,this.#n),this.#i=new AdvancedSink(this.#t,this.#n),this.#a=new DefenderSink(this.#t,this.#n)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize(),await this.#a?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i,defender:a}=resolveOlapOptions(n);if(this.#n?.debug({message:`Olap client called to record action`,category:`OlapClient`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,options:n,resolvedOptions:{logs:r,advanced:i,defender:a}}}),r?.enabled)try{await this.#r?.sendAction(e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.sendAction(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to advanced logs s3: ${e.message}`,category:`OlapClient`,error:e})}if(a?.enabled)try{await this.#a?.sendAction(e,t,a)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to defender logs s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=resolveOlapOptions(n);if(r?.enabled)try{await this.#r?.sendStep(e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.sendStep(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const buildOlapClientInstance=async({getKafkaClient:e=buildKafkaClient,getS3Client:t=buildS3Client,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new OlapClient({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var OlapClientManager=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=buildOlapClientInstance}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=OlapClient,exports.OlapClientManager=OlapClientManager;
+var e=Object.create,t=Object.defineProperty,__name=(e,n)=>t(e,`name`,{value:n,configurable:!0}),n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,__copyProps=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},__toESM=(n,r,a)=>(a=n==null?{}:e(i(n)),__copyProps(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n));let o=require(`@stackone/utils`),s=require(`@stackone/redaction`),c=require(`@aws-sdk/client-s3`),l=require(`@stackone/transport`),u=require(`kafkajs`),d=require(`node:http`);d=__toESM(d);const f=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],p=[`refresh_authentication`],m=10*1024*1024;var AdvancedSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send action`,category:`AdvancedSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}if(n.errorsOnly&&t.success){this.#t?.debug({message:`Advanced sink errorsOnly is enabled, skipping successful action for ${e.mode||`action`}`,category:`AdvancedSink`});return}if(this.isBackgroundLog(e)&&!n.includeBackground){this.#t?.debug({message:`Advanced sink is configured to exclude background logs, skipping action with mode ${e.mode}`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send step`,category:`AdvancedSink`,context:{actionRunId:e.actionRunId,success:t.success,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from advanced sink`,category:`AdvancedSink`});return}let r=process.env.ADVANCED_LOGS_BUCKET;if(!r)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let i=new c.GetObjectCommand({Bucket:r,Key:`${e}/${t}/${n}/${n}.json`}),a=await this.#e.send(i);if(!a.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`AdvancedSink`});return}let o=await a.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved advanced log from S3 for actionRunId ${n}`,category:`AdvancedSink`}),o}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Advanced log not found in S3 for actionRunId ${n}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting advanced log from S3 for actionRunId ${n}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let n=process.env.ADVANCED_LOGS_BUCKET;if(!n)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let r=`ttl=30d`;typeof t==`number`&&(t===1?r=`ttl=1d`:t===7&&(r=`ttl=7d`));let i=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;this.#t?.debug({message:`Storing advanced log in S3 bucket ${n} with ttl of ${r}`,category:`AdvancedSink`,context:{bucket:n,key:e.Key,ttlTag:r,expiresAt:new Date(i*1e3).toISOString()}});try{await this.#e.send(new c.PutObjectCommand({Bucket:n,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(i*1e3),Tagging:r}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,m),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,m),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(e,t,n,r){let i=(0,s.redactObject)({req:{...e.headers&&{headers:{...e.headers}}},res:{...t.headers&&{headers:{...t.headers}}}},s.CensorType.PARTIAL),a=i.req?.headers,o=i.res?.headers,c=(0,s.redactObject)(e.body,s.CensorType.PARTIAL),l=(0,s.redactObject)(t.body,s.CensorType.PARTIAL),u=this.isBackgroundLog(e),d={data:{request:{id:t.actionRunId,actionId:t.actionId,method:t.httpMethod,headers:this.filterHeaders(a),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:c},response:{statusCode:t.statusCode??500,headers:this.filterHeaders(o),body:l},...u?{isBackgroundLog:!0}:{}},metadata:{...u?{isBackgroundLog:!0}:{},expirationTime:n},expirationTime:n};return this.serializeAndLimit(d,r)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,t){let n=JSON.stringify(e);if(Buffer.byteLength(n,`utf8`)>t){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};n=JSON.stringify(t)}return n}filterHeaders(e){if(!e)return;let t={},n=f.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return(0,o.notMissing)(e.mode)&&p.includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const buildS3Client=(e,t)=>{try{return new c.S3Client(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}};var DefenderSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, defender sink will not function`,category:`DefenderSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Defender sink called to send action`,category:`DefenderSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled}}}),!n?.enabled){this.#t?.debug({message:`Defender sink is disabled, skipping sending action to defender sink`,category:`DefenderSink`});return}let r=this.createActionS3(t);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from defender sink`,category:`DefenderSink`});return}let r=process.env.DEFENDER_LOGS_BUCKET;if(!r)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);try{let i=new c.GetObjectCommand({Bucket:r,Key:this.generateS3Key(e,t,n)}),a=await this.#e.send(i);if(!a.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`DefenderSink`});return}let o=await a.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved defender log from S3 for actionRunId ${n}`,category:`DefenderSink`}),o}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Defender log not found in S3 for actionRunId ${n}`,category:`DefenderSink`});return}throw this.#t?.error({message:`Error when getting defender log from S3 for actionRunId ${n}`,error:e,code:`DefenderLogReadError`,category:`DefenderSink`}),e}}async send(e){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to defender sink`,category:`DefenderSink`});return}let t=process.env.DEFENDER_LOGS_BUCKET;if(!t)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);this.#t?.debug({message:`Storing defender log in S3 bucket ${t}`,category:`DefenderSink`,context:{bucket:t,key:e.Key}});try{await this.#e.send(new c.PutObjectCommand({Bucket:t,Key:e.Key,Body:e.Body,ContentType:e.ContentType}))}catch(t){throw this.#t?.error({message:`Failed to write defender logs to S3 at ${e.Key}`,error:t,code:`DefenderLogWriteError`,category:`DefenderSink`}),t}}generateS3Key(e,t,n){return`${e}/${t}/${n}/defender.json`}createActionS3(e){let t=e.organizationId,n=e.projectSecureId;return{Key:this.generateS3Key(t,n,e.actionRunId),Body:this.serializeDefenderContext(e),ContentType:`application/json`}}serializeDefenderContext(e){let t=e.defenderContext;return t?JSON.stringify(t):JSON.stringify({data:null,metadata:null})}};const buildHttpClient=e=>{try{return new l.HttpClient({logger:e})}catch(t){let n=t;e?.error({message:`Error building http client: ${n.message}`,error:n,code:`BuildHttpClientError`,category:`buildHttpClient`});return}},buildKafkaClient=(e,t)=>{try{return new u.Kafka(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},safeSerialize=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var TinybirdClient=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n}async query(e){if(!this.#e)throw this.#n?.error({message:`HTTP client not initialized, cannot perform Tinybird query`,category:`TinybirdClient`,code:`HttpClientNotReady`}),Error(`HTTP client is not initialized`);if(!this.#t?.token)throw this.#n?.warning({message:`Missing OLAP token, cannot perform Tinybird query`,category:`TinybirdClient`}),Error(`TinybirdClient - Missing token`);let t=new URL(`/v0/pipes/${e.endpoint}`,this.#t.baseUrl);try{this.#n?.debug({message:`Querying Tinybird endpoint: ${e.endpoint}`,category:`TinybirdClient`,context:{endpoint:e.endpoint,params:e.params}});let n=await this.#e.request({method:`post`,url:t.toString(),payload:e.params,headers:{"Content-Type":`application/json`,Authorization:`Bearer ${this.#t.token}`},...this.#t.allowHttp&&{httpAgent:this.#r()}}),r=n.data.data;return this.#n?.debug({message:`Tinybird query returned ${r.length} rows`,category:`TinybirdClient`,context:{endpoint:e.endpoint,rows:n.data.rows,statistics:n.data.statistics}}),r}catch(t){let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryWithPagination(e){if(!this.#e)throw this.#n?.error({message:`HTTP client not initialized, cannot perform Tinybird query`,category:`TinybirdClient`,code:`HttpClientNotReady`}),Error(`HTTP client is not initialized`);if(!this.#t?.token)throw this.#n?.warning({message:`Missing OLAP token, cannot perform Tinybird query`,category:`TinybirdClient`}),Error(`TinybirdClient - Missing token`);let t=new URL(`/v0/pipes/${e.endpoint}`,this.#t.baseUrl);try{let n=await this.#e.request({method:`post`,url:t.toString(),payload:e.params,headers:{"Content-Type":`application/json`,Authorization:`Bearer ${this.#t.token}`},...this.#t.allowHttp&&{httpAgent:this.#r()}}),r=n.data.data;return{data:r,total:n.data.rows_before_limit_at_least??n.data.rows??r.length,pageNumber:e.pageNumber??1,pageSize:e.pageSize??r.length}}catch(t){let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}isReady(){return!!(this.#e&&this.#t?.token)}#r(){return new d.default.Agent({})}};const h=`project_actions_logs.json`,g=`project_steps_logs.json`;var LogsSink=class{#e;#t;#n;#r;constructor(e,t,n,r){this.#e=e,this.#r=r,this.#n=new TinybirdClient(t,n,r)}async initialize(){if(!this.#e){this.#r?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(this.#n?.isReady()||this.#r?.warning({message:`Tinybird client is not ready (missing HTTP client or Tinybird config/token), logs sink will not be able to query`,category:`LogsSink`}),!this.#t){try{this.#t=this.#e.producer({createPartitioner:u.Partitioners.DefaultPartitioner})}catch(e){this.#r?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#r?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#r?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async getAction(e){return(await this.getActions({actionRunId:e})).actionLogs[0]}async getActions(e){try{this.#r?.info({message:`Querying action logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_actions_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} action logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{actionLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#n?.isReady()?this.#r?.warning({message:`Failed to query Tinybird for action logs, returning empty results`,category:`LogsSink`,error:t}):this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getSteps(e){return(await this.querySteps({actionRunId:e})).stepLogs}async querySteps(e){try{this.#r?.info({message:`Querying step logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_steps_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} step logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{stepLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async send(e,t){if(!this.#t)throw this.#r?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=safeSerialize(t);this.#r?.debug({message:`Sending to topic ${e}: ${n}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:n}]});this.#r?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#r?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,riskLevel:t.defenderContext?.riskLevel,tier2Score:t.defenderContext?.tier2Score,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const _={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1,includeBackground:!1},defender:{enabled:!0}};function resolveOlapOptions(e){return{logs:{..._.logs,...e?.logs},advanced:{..._.advanced,...e?.advanced},defender:{..._.defender,...e?.defender}}}var OlapClient=class{#e;#t;#n;#r;#i;#a;#o;constructor({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={}){this.name=`OlapClient`,this.#e=e(r,o),this.#t=t(o),this.#n=n(i,o),this.#r=o,this.#i=new LogsSink(this.#e,this.#t,a,this.#r),this.#a=new AdvancedSink(this.#n,this.#r),this.#o=new DefenderSink(this.#n,this.#r)}async initialize(){await this.#i?.initialize(),await this.#a?.initialize(),await this.#o?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i,defender:a}=resolveOlapOptions(n);if(this.#r?.debug({message:`Olap client called to record action`,category:`OlapClient`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,options:n,resolvedOptions:{logs:r,advanced:i,defender:a}}}),r?.enabled)try{await this.#i?.sendAction(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendAction(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to advanced logs s3: ${e.message}`,category:`OlapClient`,error:e})}if(a?.enabled)try{await this.#o?.sendAction(e,t,a)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to defender logs s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=resolveOlapOptions(n);if(r?.enabled)try{await this.#i?.sendStep(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendStep(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async getActions(e,t,n){return this.#r?.debug({message:`Querying action logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getActions({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query action logs, returning empty result`,category:`OlapClient`}),{actionLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getAction(e,t,n){let r=await this.#i?.getActions({organizationId:e,projectSecureId:t,actionRunId:n});if((0,o.isMissing)(r)||r.actionLogs.length===0){this.#r?.debug({message:`Failed to retrieve action log for actionRunId ${n}`,category:`OlapClient`});return}return r.actionLogs[0]}async getAdvanced(e,t,n){let r=await this.#a?.getAction(e,t,n);if((0,o.isMissing)(r)){this.#r?.debug({message:`Failed to retrieve advanced log for actionRunId ${n}`,category:`OlapClient`});return}return r}async getDefender(e,t,n){let r=await this.#o?.getAction(e,t,n);if((0,o.isMissing)(r)){this.#r?.debug({message:`Failed to retrieve defender log for actionRunId ${n}`,category:`OlapClient`});return}return r}};const buildOlapClientInstance=async({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={})=>{let s=new OlapClient({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o});return await s.initialize(),s};var OlapClientManager=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o,getOlapClient:s=buildOlapClientInstance}={}){return this.olapClientPromise||=s({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=OlapClient,exports.OlapClientManager=OlapClientManager;

package/dist/index.d.cts

@@ -1,4 +1,9 @@
 import { S3Client } from "@aws-sdk/client-s3";
+import * as redis from "redis";
+import { AxiosInstance } from "axios";
+import http from "node:http";
+import https from "node:https";
+import { Readable } from "node:stream";
 import { Kafka } from "kafkajs";
 
 //#region rolldown:runtime
@@ -85,10 +90,487 @@ type DefenderOptions = {
   enabled?: boolean;
 };
 //#endregion
+//#region ../transport/src/cacheClient/types.d.ts
+interface ICacheClient<ClientType = unknown> {
+  getData<T>(key: string): Promise<T | null>;
+  listData<T>({
+    partialKey,
+    limit,
+    cursor
+  }: {
+    partialKey: string;
+    limit?: number;
+    cursor?: string;
+  }): Promise<{
+    items: T[] | null;
+    cursor?: string;
+  }>;
+  setData<T>({
+    key,
+    value,
+    cacheTTL,
+    groupKey
+  }: {
+    key: string;
+    value: T;
+    cacheTTL: number;
+    groupKey?: string;
+  }): Promise<boolean>;
+  executeScript?<T>({
+    sha1,
+    keys,
+    args
+  }: {
+    sha1: string;
+    keys: string[];
+    args: string[];
+  }): Promise<T | null>;
+  loadScript?(script: string): Promise<string | null>;
+  scriptExists?(shas: string[]): Promise<boolean[]>;
+  increment?(key: string, cacheTTL: number): Promise<number | null>;
+  decrement?(key: string, cacheTTL: number): Promise<number | null>;
+  subscribe?<T extends boolean = false>(pattern: string, listener: PubSubListener<T>): Promise<boolean>;
+  unsubscribe?(pattern: string): Promise<boolean>;
+  publish?(channel: string, message: string): Promise<number | null>;
+  getClient?(): ClientType | null;
+  deleteData?(key: string): Promise<boolean>;
+  close?(): Promise<void> | void;
+}
+type PubSubListener<ReturnsBuffer extends boolean = false> = <T extends (ReturnsBuffer extends true ? Buffer : string)>(message: T, channel: T) => unknown;
+//#endregion
+//#region ../transport/src/redisClient/types.d.ts
+type RedisClientType = redis.RedisClientType;
+interface RedisClientConfig {
+  getRedisClient?: typeof redis.createClient;
+  host?: string;
+  port?: number;
+  tls?: boolean;
+  reconnect?: boolean;
+  database?: number;
+}
+type RedisClientBuilder = (config: RedisClientConfig, logger?: ILogger, invoker?: string) => Promise<ICacheClient<RedisClientType> | undefined>;
+//#endregion
+//#region ../transport/src/concurrencyManager/types.d.ts
+type ConcurrencySubPoolConfig = {
+  subPoolKey: string;
+  urlPattern: RegExp | string;
+  maxConcurrency: number;
+};
+type ConcurrencyConfig = {
+  mainMaxConcurrency: number;
+  subPools?: ConcurrencySubPoolConfig[];
+};
+//#endregion
+//#region ../transport/src/rateLimitManager/types.d.ts
+type RateLimitSubPoolConfig = {
+  subPoolKey: string;
+  urlPattern: RegExp | string;
+  rateLimit: number;
+};
+type RetryUnit = 'seconds' | 'milliseconds' | 'date';
+type MappedRateLimitErrorConfig = {
+  errorStatus: number;
+  errorMessage: string | RegExp;
+  errorMessagePath?: string;
+  retryAfterPath?: string;
+  retryAfterUnit?: RetryUnit;
+  retryAfterValue?: number;
+};
+type RateLimitConfig = {
+  mainRatelimit: number;
+  subPools?: RateLimitSubPoolConfig[];
+  mappedRateLimitErrors?: MappedRateLimitErrorConfig[];
+};
+//#endregion
+//#region ../transport/src/interceptors/types.d.ts
+type RequestConfig = {
+  rateLimits?: RateLimitConfig;
+  concurrency?: ConcurrencyConfig;
+};
+//#endregion
+//#region ../transport/src/parsers/types.d.ts
+declare const QueryArrayFormats: readonly ["repeat", "brackets", "comma"];
+type QueryArrayFormat = (typeof QueryArrayFormats)[number];
+//#endregion
+//#region ../transport/src/httpClient/types.d.ts
+type HttpHeader = string | string[];
+type HttpHeaders = {
+  [key: string]: HttpHeader;
+};
+type HttpQueryParamValue = {
+  value: string | string[];
+  arrayFormat?: QueryArrayFormat;
+};
+type HttpQueryParams = {
+  [key: string]: string | string[] | HttpQueryParamValue;
+};
+declare const HttpMethods: readonly ["get", "post", "put", "delete", "patch"];
+type HttpMethod = (typeof HttpMethods)[number];
+type HttpResponse<T = any, P = any> = {
+  data: T;
+  status: number;
+  headers: HttpHeaders;
+  requestUrl: string;
+  responseType?: string;
+  responseTime?: Date;
+  message?: string;
+  body?: P;
+};
+type StreamHttpResponse = {
+  status: number;
+  headers: HttpHeaders;
+  stream: Readable;
+  requestUrl: string;
+};
+interface IHttpClient {
+  request<P, T>({
+    headers,
+    url,
+    method,
+    queryParams,
+    maxRedirects,
+    responseType,
+    cacheTTL,
+    context,
+    payload,
+    httpsAgent,
+    httpAgent,
+    requestConfig,
+    httpsAgentConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    method?: HttpMethod;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    responseType?: 'arraybuffer' | 'blob' | 'document' | 'json' | 'text';
+    cacheTTL?: number;
+    context?: RequestContext;
+    payload?: P;
+    httpsAgent?: https.Agent;
+    httpAgent?: http.Agent;
+    requestConfig?: RequestConfig;
+    httpsAgentConfig?: https.AgentOptions;
+  }): Promise<HttpResponse<T>>;
+  get<T>({
+    headers,
+    url,
+    queryParams,
+    maxRedirects,
+    cacheTTL,
+    context,
+    requestConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    cacheTTL?: number;
+    context?: RequestContext;
+    requestConfig?: RequestConfig;
+  }): Promise<HttpResponse<T>>;
+  post<P, T>({
+    headers,
+    url,
+    maxRedirects,
+    cacheTTL,
+    context,
+    payload,
+    requestConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    maxRedirects?: number;
+    cacheTTL?: number;
+    context?: RequestContext;
+    payload?: P;
+    requestConfig?: RequestConfig;
+  }): Promise<HttpResponse<T>>;
+  requestStream({
+    headers,
+    url,
+    method,
+    queryParams,
+    maxRedirects,
+    context,
+    traceId,
+    payload,
+    httpsAgent,
+    httpAgent,
+    requestConfig,
+    httpsAgentConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    method?: HttpMethod;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    context?: RequestContext;
+    traceId?: string;
+    payload?: unknown;
+    httpsAgent?: https.Agent;
+    httpAgent?: http.Agent;
+    requestConfig?: RequestConfig;
+    httpsAgentConfig?: https.AgentOptions;
+  }): Promise<StreamHttpResponse>;
+}
+interface OperationSetting {
+  url?: string;
+  query?: Record<string, string>;
+  config?: Partial<APIConfig>;
+  custom_settings?: Record<string, unknown>;
+}
+type OperationSettings = Record<string, OperationSetting>;
+type AccountSettings = Record<string, string | number | boolean | object>;
+type HttpClientBehaviour = 'CONCURRENCY' | 'RETRY';
+type APIConfig = Record<string, unknown>;
+type RequestContext = {
+  accountSecureId?: string;
+  projectSecureId?: string;
+  organizationId?: number | string;
+  environment?: string;
+  authConfigKey?: string;
+  provider?: string;
+  service?: string;
+  resource?: string;
+  subResource?: string;
+  childResource?: string;
+  operation?: string;
+  action?: string;
+  mode?: string;
+  testerUniqueToken?: string;
+  externalKey?: string;
+  dataKey?: string;
+  isDataSync?: boolean;
+  operationSettings?: OperationSettings;
+  accountSettings?: AccountSettings;
+  behaviours?: HttpClientBehaviour[];
+};
+type ErrorMappingFn<TError extends Error = Error> = (error: Error) => TError | undefined;
+type TransportFactory = ({
+  logger,
+  redisClientConfig,
+  context,
+  requestConfig,
+  httpsAgentConfig
+}: {
+  logger?: ILogger;
+  redisClientConfig?: RedisClientConfig;
+  context?: RequestContext;
+  requestConfig?: RequestConfig;
+  httpsAgentConfig?: https.AgentOptions;
+}) => Promise<AxiosInstance>;
+//#endregion
+//#region ../transport/src/httpClient/httpClient.d.ts
+declare class HttpClient<TError extends Error = Error> implements IHttpClient {
+  #private;
+  name: string;
+  constructor({
+    transportFactory,
+    getRedisClient,
+    logger,
+    redisClientConfig,
+    errorMappingFn
+  }?: {
+    transportFactory?: TransportFactory;
+    getRedisClient?: RedisClientBuilder;
+    logger?: ILogger;
+    redisClientConfig?: RedisClientConfig;
+    errorMappingFn?: ErrorMappingFn<TError>;
+  });
+  request<P, T>({
+    headers,
+    url,
+    method,
+    queryParams,
+    maxRedirects,
+    responseType,
+    cacheTTL,
+    context,
+    traceId,
+    payload,
+    httpsAgent,
+    httpAgent,
+    requestConfig,
+    httpsAgentConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    method?: HttpMethod;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    responseType?: 'arraybuffer' | 'blob' | 'document' | 'json' | 'text';
+    cacheTTL?: number;
+    context?: RequestContext;
+    traceId?: string;
+    payload?: P;
+    httpsAgent?: https.Agent;
+    httpAgent?: http.Agent;
+    requestConfig?: RequestConfig;
+    httpsAgentConfig?: https.AgentOptions;
+  }): Promise<HttpResponse<T>>;
+  get<T>({
+    headers,
+    url,
+    queryParams,
+    maxRedirects,
+    cacheTTL,
+    context,
+    traceId,
+    requestConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    cacheTTL?: number;
+    context?: RequestContext;
+    traceId?: string;
+    requestConfig?: RequestConfig;
+  }): Promise<HttpResponse<T>>;
+  post<P, T>({
+    headers,
+    url,
+    maxRedirects,
+    cacheTTL,
+    context,
+    traceId,
+    payload,
+    requestConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    maxRedirects?: number;
+    cacheTTL?: number;
+    context?: RequestContext;
+    traceId?: string;
+    payload?: P;
+    requestConfig?: RequestConfig;
+  }): Promise<HttpResponse<T>>;
+  requestStream({
+    headers,
+    url,
+    method,
+    queryParams,
+    maxRedirects,
+    context,
+    traceId,
+    payload,
+    httpsAgent,
+    httpAgent,
+    requestConfig,
+    httpsAgentConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    method?: HttpMethod;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    context?: RequestContext;
+    traceId?: string;
+    payload?: unknown;
+    httpsAgent?: https.Agent;
+    httpAgent?: http.Agent;
+    requestConfig?: RequestConfig;
+    httpsAgentConfig?: https.AgentOptions;
+  }): Promise<StreamHttpResponse>;
+}
+//#endregion
+//#region src/logs/types.d.ts
+type ActionsQuery = {
+  organizationId?: string | string[];
+  projectSecureId?: string | string[];
+  actionRunId?: string | string[];
+  accountSecureId?: string | string[];
+  actionId?: string | string[];
+  connectorKey?: string | string[];
+  mode?: string | string[];
+  category?: string | string[];
+  success?: boolean;
+  startTime?: Date;
+  endTime?: Date;
+  pageNumber?: number;
+  pageSize?: number;
+};
+type ActionsQueryResult = {
+  actionLogs: ActionLog[];
+  total: number;
+  pageNumber: number;
+  pageSize: number;
+};
+type StepsQuery = {
+  organizationId?: string | string[];
+  projectSecureId?: string | string[];
+  accountSecureId?: string | string[];
+  actionRunId?: string | string[];
+  stepId?: string | string[];
+  success?: boolean;
+  startTime?: Date;
+  endTime?: Date;
+  pageNumber?: number;
+  pageSize?: number;
+};
+type StepsQueryResult = {
+  stepLogs: StepLog[];
+  total: number;
+  pageNumber: number;
+  pageSize: number;
+};
+type HttpClientBuilder = (logger?: ILogger) => HttpClient | undefined;
+interface KafkaClientConfig {
+  brokers: string[];
+}
+type KafkaClientBuilder = (config?: KafkaClientConfig, logger?: ILogger) => Kafka | undefined;
+type ActionLog = {
+  actionRunId: string;
+  actionId: string;
+  connectorKey: string;
+  connectorVersion: string;
+  organizationId: string;
+  projectSecureId: string;
+  accountSecureId: string;
+  mode?: string;
+  category?: string;
+  originOwnerId?: string;
+  originOwnerName?: string;
+  httpMethod?: string;
+  url?: string;
+  sourceType?: string;
+  sourceId?: string;
+  sourceValue?: string;
+  success?: boolean;
+  statusCode?: number;
+  startTime?: Date;
+  endTime?: Date;
+  durationMs?: number;
+  eventTime?: Date;
+  riskLevel?: string;
+  tier2Score?: number;
+};
+type StepLog = {
+  actionRunId: string;
+  stepIndex: number;
+  stepId: string;
+  organizationId: string;
+  projectSecureId: string;
+  accountSecureId: string;
+  skipped?: boolean;
+  success?: boolean;
+  message?: string;
+  startTime?: Date;
+  endTime?: Date;
+  durationMs?: number;
+  eventTime?: Date;
+};
+//#endregion
 //#region src/types.d.ts
 interface IOlapClient {
   recordAction(actionInput: ActionInput, actionResult: ActionResult, options?: OlapOptions): void;
   recordStep(stepInput: StepInput, stepResult: StepResult, options?: OlapOptions): void;
+  getActions(organizationId: string, projectSecureId: string, query: ActionsQuery): Promise<ActionsQueryResult>;
+  getAction(organizationId: string, projectSecureId: string, actionRunId: string): Promise<ActionLog | undefined>;
+  getAdvanced(organizationId: string, projectSecureId: string, actionRunId: string): Promise<string | undefined>;
+  getDefender(organizationId: string, projectSecureId: string, actionRunId: string): Promise<string | undefined>;
 }
 type DefenderContext = {
   riskLevel: string;
@@ -161,11 +643,30 @@ interface S3ClientConfig {
 }
 type S3ClientBuilder = (config?: S3ClientConfig, logger?: ILogger) => S3Client | undefined;
 //#endregion
-//#region src/logs/types.d.ts
-interface KafkaClientConfig {
-  brokers: string[];
+//#region src/logs/tinybirdClient.d.ts
+interface TinybirdConfig {
+  baseUrl: string;
+  token: string;
+  allowHttp?: boolean;
+}
+interface TinybirdResponse<T> {
+  meta: Array<{
+    name: string;
+    type: string;
+  }>;
+  data: T;
+  rows: number;
+  rows_before_limit_at_least?: number;
+  statistics?: {
+    elapsed: number;
+    rows_read: number;
+    bytes_read: number;
+  };
+}
+interface TinybirdQueryOptions<TParams = Record<string, unknown>> {
+  endpoint: string;
+  params?: TParams;
 }
-type KafkaClientBuilder = (config?: KafkaClientConfig, logger?: ILogger) => Kafka | undefined;
 //#endregion
 //#region src/olap/olapClient.d.ts
 declare class OlapClient implements IOlapClient {
@@ -173,54 +674,70 @@ declare class OlapClient implements IOlapClient {
   name: string;
   constructor({
     getKafkaClient,
+    getHttpClient,
     getS3Client,
     kafkaClientConfig,
     s3ClientConfig,
+    tinybirdConfig,
     logger
   }?: {
     getKafkaClient?: KafkaClientBuilder;
+    getHttpClient?: HttpClientBuilder;
     getS3Client?: S3ClientBuilder;
     kafkaClientConfig?: KafkaClientConfig;
     s3ClientConfig?: S3ClientConfig;
+    tinybirdConfig?: TinybirdConfig;
     logger?: ILogger;
   });
   initialize(): Promise<void>;
   recordAction(actionInput: ActionInput, actionResult: ActionResult, options?: OlapOptions): Promise<void>;
   recordStep(stepInput: StepInput, stepResult: StepResult, options?: OlapOptions): Promise<void>;
+  getActions(organizationId: string, projectSecureId: string, query: ActionsQuery): Promise<ActionsQueryResult>;
+  getAction(organizationId: string, projectSecureId: string, actionRunId: string): Promise<ActionLog | undefined>;
+  getAdvanced(organizationId: string, projectSecureId: string, actionRunId: string): Promise<string | undefined>;
+  getDefender(organizationId: string, projectSecureId: string, actionRunId: string): Promise<string | undefined>;
 }
 //#endregion
 //#region src/olap/olapClientManager.d.ts
 declare const buildOlapClientInstance: ({
   getKafkaClient,
+  getHttpClient,
   getS3Client,
   kafkaClientConfig,
   s3ClientConfig,
+  tinybirdConfig,
   logger
 }?: {
   getKafkaClient?: KafkaClientBuilder;
+  getHttpClient?: HttpClientBuilder;
   getS3Client?: S3ClientBuilder;
   kafkaClientConfig?: KafkaClientConfig;
   s3ClientConfig?: S3ClientConfig;
+  tinybirdConfig?: TinybirdConfig;
   logger?: ILogger;
 }) => Promise<IOlapClient>;
 declare class OlapClientManager {
   private static olapClientPromise;
   static getInstance({
     getKafkaClient,
+    getHttpClient,
     getS3Client,
     kafkaClientConfig,
     s3ClientConfig,
+    tinybirdConfig,
     logger,
     getOlapClient
   }?: {
     getKafkaClient?: KafkaClientBuilder;
+    getHttpClient?: HttpClientBuilder;
     getS3Client?: S3ClientBuilder;
     kafkaClientConfig?: KafkaClientConfig;
     s3ClientConfig?: S3ClientConfig;
+    tinybirdConfig?: TinybirdConfig;
     logger?: ILogger;
     getOlapClient?: typeof buildOlapClientInstance;
   }): Promise<IOlapClient>;
   static resetInstance(): void;
 }
 //#endregion
-export { type ActionInput, type ActionResult, type AdvancedOptions, type DefenderContext, type DefenderOptions, type IOlapClient, type LogsOptions, OlapClient, OlapClientManager, type OlapOptions, type StepInput, type StepResult };
+export { type ActionInput, type ActionLog, type ActionResult, type ActionsQuery, type ActionsQueryResult, type AdvancedOptions, type DefenderContext, type DefenderOptions, type IOlapClient, type LogsOptions, OlapClient, OlapClientManager, type OlapOptions, type StepInput, type StepLog, type StepResult, type StepsQuery, type StepsQueryResult, type TinybirdConfig, type TinybirdQueryOptions, type TinybirdResponse };

package/dist/index.d.mts

@@ -1,6 +1,12 @@
 import { t as __name } from "./chunk-Cfxk5zVN.mjs";
+import "@stackone/utils";
 import { S3Client } from "@aws-sdk/client-s3";
 import { Kafka } from "kafkajs";
+import http from "node:http";
+import * as redis from "redis";
+import { AxiosInstance } from "axios";
+import https from "node:https";
+import { Readable } from "node:stream";
 
 //#region ../logger/src/types.d.ts
 interface ILogger {
@@ -84,10 +90,487 @@ type DefenderOptions = {
   enabled?: boolean;
 };
 //#endregion
+//#region ../transport/src/cacheClient/types.d.ts
+interface ICacheClient<ClientType = unknown> {
+  getData<T>(key: string): Promise<T | null>;
+  listData<T>({
+    partialKey,
+    limit,
+    cursor
+  }: {
+    partialKey: string;
+    limit?: number;
+    cursor?: string;
+  }): Promise<{
+    items: T[] | null;
+    cursor?: string;
+  }>;
+  setData<T>({
+    key,
+    value,
+    cacheTTL,
+    groupKey
+  }: {
+    key: string;
+    value: T;
+    cacheTTL: number;
+    groupKey?: string;
+  }): Promise<boolean>;
+  executeScript?<T>({
+    sha1,
+    keys,
+    args
+  }: {
+    sha1: string;
+    keys: string[];
+    args: string[];
+  }): Promise<T | null>;
+  loadScript?(script: string): Promise<string | null>;
+  scriptExists?(shas: string[]): Promise<boolean[]>;
+  increment?(key: string, cacheTTL: number): Promise<number | null>;
+  decrement?(key: string, cacheTTL: number): Promise<number | null>;
+  subscribe?<T extends boolean = false>(pattern: string, listener: PubSubListener<T>): Promise<boolean>;
+  unsubscribe?(pattern: string): Promise<boolean>;
+  publish?(channel: string, message: string): Promise<number | null>;
+  getClient?(): ClientType | null;
+  deleteData?(key: string): Promise<boolean>;
+  close?(): Promise<void> | void;
+}
+type PubSubListener<ReturnsBuffer extends boolean = false> = <T extends (ReturnsBuffer extends true ? Buffer : string)>(message: T, channel: T) => unknown;
+//#endregion
+//#region ../transport/src/redisClient/types.d.ts
+type RedisClientType = redis.RedisClientType;
+interface RedisClientConfig {
+  getRedisClient?: typeof redis.createClient;
+  host?: string;
+  port?: number;
+  tls?: boolean;
+  reconnect?: boolean;
+  database?: number;
+}
+type RedisClientBuilder = (config: RedisClientConfig, logger?: ILogger, invoker?: string) => Promise<ICacheClient<RedisClientType> | undefined>;
+//#endregion
+//#region ../transport/src/concurrencyManager/types.d.ts
+type ConcurrencySubPoolConfig = {
+  subPoolKey: string;
+  urlPattern: RegExp | string;
+  maxConcurrency: number;
+};
+type ConcurrencyConfig = {
+  mainMaxConcurrency: number;
+  subPools?: ConcurrencySubPoolConfig[];
+};
+//#endregion
+//#region ../transport/src/rateLimitManager/types.d.ts
+type RateLimitSubPoolConfig = {
+  subPoolKey: string;
+  urlPattern: RegExp | string;
+  rateLimit: number;
+};
+type RetryUnit = 'seconds' | 'milliseconds' | 'date';
+type MappedRateLimitErrorConfig = {
+  errorStatus: number;
+  errorMessage: string | RegExp;
+  errorMessagePath?: string;
+  retryAfterPath?: string;
+  retryAfterUnit?: RetryUnit;
+  retryAfterValue?: number;
+};
+type RateLimitConfig = {
+  mainRatelimit: number;
+  subPools?: RateLimitSubPoolConfig[];
+  mappedRateLimitErrors?: MappedRateLimitErrorConfig[];
+};
+//#endregion
+//#region ../transport/src/interceptors/types.d.ts
+type RequestConfig = {
+  rateLimits?: RateLimitConfig;
+  concurrency?: ConcurrencyConfig;
+};
+//#endregion
+//#region ../transport/src/parsers/types.d.ts
+declare const QueryArrayFormats: readonly ["repeat", "brackets", "comma"];
+type QueryArrayFormat = (typeof QueryArrayFormats)[number];
+//#endregion
+//#region ../transport/src/httpClient/types.d.ts
+type HttpHeader = string | string[];
+type HttpHeaders = {
+  [key: string]: HttpHeader;
+};
+type HttpQueryParamValue = {
+  value: string | string[];
+  arrayFormat?: QueryArrayFormat;
+};
+type HttpQueryParams = {
+  [key: string]: string | string[] | HttpQueryParamValue;
+};
+declare const HttpMethods: readonly ["get", "post", "put", "delete", "patch"];
+type HttpMethod = (typeof HttpMethods)[number];
+type HttpResponse<T = any, P = any> = {
+  data: T;
+  status: number;
+  headers: HttpHeaders;
+  requestUrl: string;
+  responseType?: string;
+  responseTime?: Date;
+  message?: string;
+  body?: P;
+};
+type StreamHttpResponse = {
+  status: number;
+  headers: HttpHeaders;
+  stream: Readable;
+  requestUrl: string;
+};
+interface IHttpClient {
+  request<P, T>({
+    headers,
+    url,
+    method,
+    queryParams,
+    maxRedirects,
+    responseType,
+    cacheTTL,
+    context,
+    payload,
+    httpsAgent,
+    httpAgent,
+    requestConfig,
+    httpsAgentConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    method?: HttpMethod;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    responseType?: 'arraybuffer' | 'blob' | 'document' | 'json' | 'text';
+    cacheTTL?: number;
+    context?: RequestContext;
+    payload?: P;
+    httpsAgent?: https.Agent;
+    httpAgent?: http.Agent;
+    requestConfig?: RequestConfig;
+    httpsAgentConfig?: https.AgentOptions;
+  }): Promise<HttpResponse<T>>;
+  get<T>({
+    headers,
+    url,
+    queryParams,
+    maxRedirects,
+    cacheTTL,
+    context,
+    requestConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    cacheTTL?: number;
+    context?: RequestContext;
+    requestConfig?: RequestConfig;
+  }): Promise<HttpResponse<T>>;
+  post<P, T>({
+    headers,
+    url,
+    maxRedirects,
+    cacheTTL,
+    context,
+    payload,
+    requestConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    maxRedirects?: number;
+    cacheTTL?: number;
+    context?: RequestContext;
+    payload?: P;
+    requestConfig?: RequestConfig;
+  }): Promise<HttpResponse<T>>;
+  requestStream({
+    headers,
+    url,
+    method,
+    queryParams,
+    maxRedirects,
+    context,
+    traceId,
+    payload,
+    httpsAgent,
+    httpAgent,
+    requestConfig,
+    httpsAgentConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    method?: HttpMethod;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    context?: RequestContext;
+    traceId?: string;
+    payload?: unknown;
+    httpsAgent?: https.Agent;
+    httpAgent?: http.Agent;
+    requestConfig?: RequestConfig;
+    httpsAgentConfig?: https.AgentOptions;
+  }): Promise<StreamHttpResponse>;
+}
+interface OperationSetting {
+  url?: string;
+  query?: Record<string, string>;
+  config?: Partial<APIConfig>;
+  custom_settings?: Record<string, unknown>;
+}
+type OperationSettings = Record<string, OperationSetting>;
+type AccountSettings = Record<string, string | number | boolean | object>;
+type HttpClientBehaviour = 'CONCURRENCY' | 'RETRY';
+type APIConfig = Record<string, unknown>;
+type RequestContext = {
+  accountSecureId?: string;
+  projectSecureId?: string;
+  organizationId?: number | string;
+  environment?: string;
+  authConfigKey?: string;
+  provider?: string;
+  service?: string;
+  resource?: string;
+  subResource?: string;
+  childResource?: string;
+  operation?: string;
+  action?: string;
+  mode?: string;
+  testerUniqueToken?: string;
+  externalKey?: string;
+  dataKey?: string;
+  isDataSync?: boolean;
+  operationSettings?: OperationSettings;
+  accountSettings?: AccountSettings;
+  behaviours?: HttpClientBehaviour[];
+};
+type ErrorMappingFn<TError extends Error = Error> = (error: Error) => TError | undefined;
+type TransportFactory = ({
+  logger,
+  redisClientConfig,
+  context,
+  requestConfig,
+  httpsAgentConfig
+}: {
+  logger?: ILogger;
+  redisClientConfig?: RedisClientConfig;
+  context?: RequestContext;
+  requestConfig?: RequestConfig;
+  httpsAgentConfig?: https.AgentOptions;
+}) => Promise<AxiosInstance>;
+//#endregion
+//#region ../transport/src/httpClient/httpClient.d.ts
+declare class HttpClient<TError extends Error = Error> implements IHttpClient {
+  #private;
+  name: string;
+  constructor({
+    transportFactory,
+    getRedisClient,
+    logger,
+    redisClientConfig,
+    errorMappingFn
+  }?: {
+    transportFactory?: TransportFactory;
+    getRedisClient?: RedisClientBuilder;
+    logger?: ILogger;
+    redisClientConfig?: RedisClientConfig;
+    errorMappingFn?: ErrorMappingFn<TError>;
+  });
+  request<P, T>({
+    headers,
+    url,
+    method,
+    queryParams,
+    maxRedirects,
+    responseType,
+    cacheTTL,
+    context,
+    traceId,
+    payload,
+    httpsAgent,
+    httpAgent,
+    requestConfig,
+    httpsAgentConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    method?: HttpMethod;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    responseType?: 'arraybuffer' | 'blob' | 'document' | 'json' | 'text';
+    cacheTTL?: number;
+    context?: RequestContext;
+    traceId?: string;
+    payload?: P;
+    httpsAgent?: https.Agent;
+    httpAgent?: http.Agent;
+    requestConfig?: RequestConfig;
+    httpsAgentConfig?: https.AgentOptions;
+  }): Promise<HttpResponse<T>>;
+  get<T>({
+    headers,
+    url,
+    queryParams,
+    maxRedirects,
+    cacheTTL,
+    context,
+    traceId,
+    requestConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    cacheTTL?: number;
+    context?: RequestContext;
+    traceId?: string;
+    requestConfig?: RequestConfig;
+  }): Promise<HttpResponse<T>>;
+  post<P, T>({
+    headers,
+    url,
+    maxRedirects,
+    cacheTTL,
+    context,
+    traceId,
+    payload,
+    requestConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    maxRedirects?: number;
+    cacheTTL?: number;
+    context?: RequestContext;
+    traceId?: string;
+    payload?: P;
+    requestConfig?: RequestConfig;
+  }): Promise<HttpResponse<T>>;
+  requestStream({
+    headers,
+    url,
+    method,
+    queryParams,
+    maxRedirects,
+    context,
+    traceId,
+    payload,
+    httpsAgent,
+    httpAgent,
+    requestConfig,
+    httpsAgentConfig
+  }: {
+    headers?: HttpHeaders;
+    url: string;
+    method?: HttpMethod;
+    queryParams?: HttpQueryParams;
+    maxRedirects?: number;
+    context?: RequestContext;
+    traceId?: string;
+    payload?: unknown;
+    httpsAgent?: https.Agent;
+    httpAgent?: http.Agent;
+    requestConfig?: RequestConfig;
+    httpsAgentConfig?: https.AgentOptions;
+  }): Promise<StreamHttpResponse>;
+}
+//#endregion
+//#region src/logs/types.d.ts
+type ActionsQuery = {
+  organizationId?: string | string[];
+  projectSecureId?: string | string[];
+  actionRunId?: string | string[];
+  accountSecureId?: string | string[];
+  actionId?: string | string[];
+  connectorKey?: string | string[];
+  mode?: string | string[];
+  category?: string | string[];
+  success?: boolean;
+  startTime?: Date;
+  endTime?: Date;
+  pageNumber?: number;
+  pageSize?: number;
+};
+type ActionsQueryResult = {
+  actionLogs: ActionLog[];
+  total: number;
+  pageNumber: number;
+  pageSize: number;
+};
+type StepsQuery = {
+  organizationId?: string | string[];
+  projectSecureId?: string | string[];
+  accountSecureId?: string | string[];
+  actionRunId?: string | string[];
+  stepId?: string | string[];
+  success?: boolean;
+  startTime?: Date;
+  endTime?: Date;
+  pageNumber?: number;
+  pageSize?: number;
+};
+type StepsQueryResult = {
+  stepLogs: StepLog[];
+  total: number;
+  pageNumber: number;
+  pageSize: number;
+};
+type HttpClientBuilder = (logger?: ILogger) => HttpClient | undefined;
+interface KafkaClientConfig {
+  brokers: string[];
+}
+type KafkaClientBuilder = (config?: KafkaClientConfig, logger?: ILogger) => Kafka | undefined;
+type ActionLog = {
+  actionRunId: string;
+  actionId: string;
+  connectorKey: string;
+  connectorVersion: string;
+  organizationId: string;
+  projectSecureId: string;
+  accountSecureId: string;
+  mode?: string;
+  category?: string;
+  originOwnerId?: string;
+  originOwnerName?: string;
+  httpMethod?: string;
+  url?: string;
+  sourceType?: string;
+  sourceId?: string;
+  sourceValue?: string;
+  success?: boolean;
+  statusCode?: number;
+  startTime?: Date;
+  endTime?: Date;
+  durationMs?: number;
+  eventTime?: Date;
+  riskLevel?: string;
+  tier2Score?: number;
+};
+type StepLog = {
+  actionRunId: string;
+  stepIndex: number;
+  stepId: string;
+  organizationId: string;
+  projectSecureId: string;
+  accountSecureId: string;
+  skipped?: boolean;
+  success?: boolean;
+  message?: string;
+  startTime?: Date;
+  endTime?: Date;
+  durationMs?: number;
+  eventTime?: Date;
+};
+//#endregion
 //#region src/types.d.ts
 interface IOlapClient {
   recordAction(actionInput: ActionInput, actionResult: ActionResult, options?: OlapOptions): void;
   recordStep(stepInput: StepInput, stepResult: StepResult, options?: OlapOptions): void;
+  getActions(organizationId: string, projectSecureId: string, query: ActionsQuery): Promise<ActionsQueryResult>;
+  getAction(organizationId: string, projectSecureId: string, actionRunId: string): Promise<ActionLog | undefined>;
+  getAdvanced(organizationId: string, projectSecureId: string, actionRunId: string): Promise<string | undefined>;
+  getDefender(organizationId: string, projectSecureId: string, actionRunId: string): Promise<string | undefined>;
 }
 type DefenderContext = {
   riskLevel: string;
@@ -160,11 +643,30 @@ interface S3ClientConfig {
 }
 type S3ClientBuilder = (config?: S3ClientConfig, logger?: ILogger) => S3Client | undefined;
 //#endregion
-//#region src/logs/types.d.ts
-interface KafkaClientConfig {
-  brokers: string[];
+//#region src/logs/tinybirdClient.d.ts
+interface TinybirdConfig {
+  baseUrl: string;
+  token: string;
+  allowHttp?: boolean;
+}
+interface TinybirdResponse<T> {
+  meta: Array<{
+    name: string;
+    type: string;
+  }>;
+  data: T;
+  rows: number;
+  rows_before_limit_at_least?: number;
+  statistics?: {
+    elapsed: number;
+    rows_read: number;
+    bytes_read: number;
+  };
+}
+interface TinybirdQueryOptions<TParams = Record<string, unknown>> {
+  endpoint: string;
+  params?: TParams;
 }
-type KafkaClientBuilder = (config?: KafkaClientConfig, logger?: ILogger) => Kafka | undefined;
 //#endregion
 //#region src/olap/olapClient.d.ts
 declare class OlapClient implements IOlapClient {
@@ -172,54 +674,70 @@ declare class OlapClient implements IOlapClient {
   name: string;
   constructor({
     getKafkaClient,
+    getHttpClient,
     getS3Client,
     kafkaClientConfig,
     s3ClientConfig,
+    tinybirdConfig,
     logger
   }?: {
     getKafkaClient?: KafkaClientBuilder;
+    getHttpClient?: HttpClientBuilder;
     getS3Client?: S3ClientBuilder;
     kafkaClientConfig?: KafkaClientConfig;
     s3ClientConfig?: S3ClientConfig;
+    tinybirdConfig?: TinybirdConfig;
     logger?: ILogger;
   });
   initialize(): Promise<void>;
   recordAction(actionInput: ActionInput, actionResult: ActionResult, options?: OlapOptions): Promise<void>;
   recordStep(stepInput: StepInput, stepResult: StepResult, options?: OlapOptions): Promise<void>;
+  getActions(organizationId: string, projectSecureId: string, query: ActionsQuery): Promise<ActionsQueryResult>;
+  getAction(organizationId: string, projectSecureId: string, actionRunId: string): Promise<ActionLog | undefined>;
+  getAdvanced(organizationId: string, projectSecureId: string, actionRunId: string): Promise<string | undefined>;
+  getDefender(organizationId: string, projectSecureId: string, actionRunId: string): Promise<string | undefined>;
 }
 //#endregion
 //#region src/olap/olapClientManager.d.ts
 declare const buildOlapClientInstance: ({
   getKafkaClient,
+  getHttpClient,
   getS3Client,
   kafkaClientConfig,
   s3ClientConfig,
+  tinybirdConfig,
   logger
 }?: {
   getKafkaClient?: KafkaClientBuilder;
+  getHttpClient?: HttpClientBuilder;
   getS3Client?: S3ClientBuilder;
   kafkaClientConfig?: KafkaClientConfig;
   s3ClientConfig?: S3ClientConfig;
+  tinybirdConfig?: TinybirdConfig;
   logger?: ILogger;
 }) => Promise<IOlapClient>;
 declare class OlapClientManager {
   private static olapClientPromise;
   static getInstance({
     getKafkaClient,
+    getHttpClient,
     getS3Client,
     kafkaClientConfig,
     s3ClientConfig,
+    tinybirdConfig,
     logger,
     getOlapClient
   }?: {
     getKafkaClient?: KafkaClientBuilder;
+    getHttpClient?: HttpClientBuilder;
     getS3Client?: S3ClientBuilder;
     kafkaClientConfig?: KafkaClientConfig;
     s3ClientConfig?: S3ClientConfig;
+    tinybirdConfig?: TinybirdConfig;
     logger?: ILogger;
     getOlapClient?: typeof buildOlapClientInstance;
   }): Promise<IOlapClient>;
   static resetInstance(): void;
 }
 //#endregion
-export { type ActionInput, type ActionResult, type AdvancedOptions, type DefenderContext, type DefenderOptions, type IOlapClient, type LogsOptions, OlapClient, OlapClientManager, type OlapOptions, type StepInput, type StepResult };
+export { type ActionInput, type ActionLog, type ActionResult, type ActionsQuery, type ActionsQueryResult, type AdvancedOptions, type DefenderContext, type DefenderOptions, type IOlapClient, type LogsOptions, OlapClient, OlapClientManager, type OlapOptions, type StepInput, type StepLog, type StepResult, type StepsQuery, type StepsQueryResult, type TinybirdConfig, type TinybirdQueryOptions, type TinybirdResponse };

package/dist/index.mjs

@@ -1 +1 @@
-import{t as e}from"./chunk-Cfxk5zVN.mjs";import{CensorType as t,redactObject as n}from"@stackone/redaction";import{notMissing as r}from"@stackone/utils";import{PutObjectCommand as i,S3Client as a}from"@aws-sdk/client-s3";import{Kafka as o,Partitioners as s}from"kafkajs";const c=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],l=[`refresh_authentication`],u=10*1024*1024;var AdvancedSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send action`,category:`AdvancedSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}if(n.errorsOnly&&t.success){this.#t?.debug({message:`Advanced sink errorsOnly is enabled, skipping successful action for ${e.mode||`action`}`,category:`AdvancedSink`});return}if(this.isBackgroundLog(e)&&!n.includeBackground){this.#t?.debug({message:`Advanced sink is configured to exclude background logs, skipping action with mode ${e.mode}`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send step`,category:`AdvancedSink`,context:{actionRunId:e.actionRunId,success:t.success,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let n=process.env.ADVANCED_LOGS_BUCKET;if(!n)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let r=`ttl=30d`;typeof t==`number`&&(t===1?r=`ttl=1d`:t===7&&(r=`ttl=7d`));let a=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;this.#t?.debug({message:`Storing advanced log in S3 bucket ${n} with ttl of ${r}`,category:`AdvancedSink`,context:{bucket:n,key:e.Key,ttlTag:r,expiresAt:new Date(a*1e3).toISOString()}});try{await this.#e.send(new i({Bucket:n,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(a*1e3),Tagging:r}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,u),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,u),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(e,r,i,a){let o=n({req:{...e.headers&&{headers:{...e.headers}}},res:{...r.headers&&{headers:{...r.headers}}}},t.PARTIAL),s=o.req?.headers,c=o.res?.headers,l=n(e.body,t.PARTIAL),u=n(r.body,t.PARTIAL),d=this.isBackgroundLog(e),f={data:{request:{id:r.actionRunId,actionId:r.actionId,method:r.httpMethod,headers:this.filterHeaders(s),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:l},response:{statusCode:r.statusCode??500,headers:this.filterHeaders(c),body:u},...d?{isBackgroundLog:!0}:{}},metadata:{...d?{isBackgroundLog:!0}:{},expirationTime:i},expirationTime:i};return this.serializeAndLimit(f,a)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,t){let n=JSON.stringify(e);if(Buffer.byteLength(n,`utf8`)>t){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};n=JSON.stringify(t)}return n}filterHeaders(e){if(!e)return;let t={},n=c.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return r(e.mode)&&l.includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const buildS3Client=(e,t)=>{try{return new a(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}};var DefenderSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, defender sink will not function`,category:`DefenderSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Defender sink called to send action`,category:`DefenderSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled}}}),!n?.enabled){this.#t?.debug({message:`Defender sink is disabled, skipping sending action to defender sink`,category:`DefenderSink`});return}let r=this.createActionS3(t);await this.send(r)}async send(e){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to defender sink`,category:`DefenderSink`});return}let t=process.env.DEFENDER_LOGS_BUCKET;if(!t)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);this.#t?.debug({message:`Storing defender log in S3 bucket ${t}`,category:`DefenderSink`,context:{bucket:t,key:e.Key}});try{await this.#e.send(new i({Bucket:t,Key:e.Key,Body:e.Body,ContentType:e.ContentType}))}catch(t){throw this.#t?.error({message:`Failed to write defender logs to S3 at ${e.Key}`,error:t,code:`DefenderLogWriteError`,category:`DefenderSink`}),t}}createActionS3(e){return{Key:`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/defender.json`,Body:this.serializeDefenderContext(e),ContentType:`application/json`}}serializeDefenderContext(e){let t=e.defenderContext;return t?JSON.stringify(t):JSON.stringify({data:null,metadata:null})}};const buildKafkaClient=(e,t)=>{try{return new o(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},safeSerialize=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var LogsSink=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:s.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async send(e,t){if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=safeSerialize(t);this.#n?.debug({message:`Sending to topic ${e}: ${n}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:n}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,riskLevel:t.defenderContext?.riskLevel,tier2Score:t.defenderContext?.tier2Score,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const d={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1,includeBackground:!1},defender:{enabled:!0}};function resolveOlapOptions(e){return{logs:{...d.logs,...e?.logs},advanced:{...d.advanced,...e?.advanced},defender:{...d.defender,...e?.defender}}}var OlapClient=class{#e;#t;#n;#r;#i;#a;constructor({getKafkaClient:e=buildKafkaClient,getS3Client:t=buildS3Client,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new LogsSink(this.#e,this.#n),this.#i=new AdvancedSink(this.#t,this.#n),this.#a=new DefenderSink(this.#t,this.#n)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize(),await this.#a?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i,defender:a}=resolveOlapOptions(n);if(this.#n?.debug({message:`Olap client called to record action`,category:`OlapClient`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,options:n,resolvedOptions:{logs:r,advanced:i,defender:a}}}),r?.enabled)try{await this.#r?.sendAction(e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.sendAction(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to advanced logs s3: ${e.message}`,category:`OlapClient`,error:e})}if(a?.enabled)try{await this.#a?.sendAction(e,t,a)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to defender logs s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=resolveOlapOptions(n);if(r?.enabled)try{await this.#r?.sendStep(e,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.sendStep(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const buildOlapClientInstance=async({getKafkaClient:e=buildKafkaClient,getS3Client:t=buildS3Client,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new OlapClient({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var OlapClientManager=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=buildOlapClientInstance}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{OlapClient,OlapClientManager};
+import{t as e}from"./chunk-Cfxk5zVN.mjs";import{isMissing as t,notMissing as n}from"@stackone/utils";import{CensorType as r,redactObject as i}from"@stackone/redaction";import{GetObjectCommand as a,PutObjectCommand as o,S3Client as s}from"@aws-sdk/client-s3";import{HttpClient as c}from"@stackone/transport";import{Kafka as l,Partitioners as u}from"kafkajs";import d from"node:http";const f=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],p=[`refresh_authentication`],m=10*1024*1024;var AdvancedSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send action`,category:`AdvancedSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}if(n.errorsOnly&&t.success){this.#t?.debug({message:`Advanced sink errorsOnly is enabled, skipping successful action for ${e.mode||`action`}`,category:`AdvancedSink`});return}if(this.isBackgroundLog(e)&&!n.includeBackground){this.#t?.debug({message:`Advanced sink is configured to exclude background logs, skipping action with mode ${e.mode}`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send step`,category:`AdvancedSink`,context:{actionRunId:e.actionRunId,success:t.success,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from advanced sink`,category:`AdvancedSink`});return}let r=process.env.ADVANCED_LOGS_BUCKET;if(!r)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let i=new a({Bucket:r,Key:`${e}/${t}/${n}/${n}.json`}),o=await this.#e.send(i);if(!o.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`AdvancedSink`});return}let s=await o.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved advanced log from S3 for actionRunId ${n}`,category:`AdvancedSink`}),s}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Advanced log not found in S3 for actionRunId ${n}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting advanced log from S3 for actionRunId ${n}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let n=process.env.ADVANCED_LOGS_BUCKET;if(!n)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let r=`ttl=30d`;typeof t==`number`&&(t===1?r=`ttl=1d`:t===7&&(r=`ttl=7d`));let i=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;this.#t?.debug({message:`Storing advanced log in S3 bucket ${n} with ttl of ${r}`,category:`AdvancedSink`,context:{bucket:n,key:e.Key,ttlTag:r,expiresAt:new Date(i*1e3).toISOString()}});try{await this.#e.send(new o({Bucket:n,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(i*1e3),Tagging:r}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,m),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,m),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(e,t,n,a){let o=i({req:{...e.headers&&{headers:{...e.headers}}},res:{...t.headers&&{headers:{...t.headers}}}},r.PARTIAL),s=o.req?.headers,c=o.res?.headers,l=i(e.body,r.PARTIAL),u=i(t.body,r.PARTIAL),d=this.isBackgroundLog(e),f={data:{request:{id:t.actionRunId,actionId:t.actionId,method:t.httpMethod,headers:this.filterHeaders(s),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:l},response:{statusCode:t.statusCode??500,headers:this.filterHeaders(c),body:u},...d?{isBackgroundLog:!0}:{}},metadata:{...d?{isBackgroundLog:!0}:{},expirationTime:n},expirationTime:n};return this.serializeAndLimit(f,a)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,t){let n=JSON.stringify(e);if(Buffer.byteLength(n,`utf8`)>t){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};n=JSON.stringify(t)}return n}filterHeaders(e){if(!e)return;let t={},n=f.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return n(e.mode)&&p.includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const buildS3Client=(e,t)=>{try{return new s(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}};var DefenderSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, defender sink will not function`,category:`DefenderSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Defender sink called to send action`,category:`DefenderSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled}}}),!n?.enabled){this.#t?.debug({message:`Defender sink is disabled, skipping sending action to defender sink`,category:`DefenderSink`});return}let r=this.createActionS3(t);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from defender sink`,category:`DefenderSink`});return}let r=process.env.DEFENDER_LOGS_BUCKET;if(!r)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);try{let i=new a({Bucket:r,Key:this.generateS3Key(e,t,n)}),o=await this.#e.send(i);if(!o.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`DefenderSink`});return}let s=await o.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved defender log from S3 for actionRunId ${n}`,category:`DefenderSink`}),s}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Defender log not found in S3 for actionRunId ${n}`,category:`DefenderSink`});return}throw this.#t?.error({message:`Error when getting defender log from S3 for actionRunId ${n}`,error:e,code:`DefenderLogReadError`,category:`DefenderSink`}),e}}async send(e){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to defender sink`,category:`DefenderSink`});return}let t=process.env.DEFENDER_LOGS_BUCKET;if(!t)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);this.#t?.debug({message:`Storing defender log in S3 bucket ${t}`,category:`DefenderSink`,context:{bucket:t,key:e.Key}});try{await this.#e.send(new o({Bucket:t,Key:e.Key,Body:e.Body,ContentType:e.ContentType}))}catch(t){throw this.#t?.error({message:`Failed to write defender logs to S3 at ${e.Key}`,error:t,code:`DefenderLogWriteError`,category:`DefenderSink`}),t}}generateS3Key(e,t,n){return`${e}/${t}/${n}/defender.json`}createActionS3(e){let t=e.organizationId,n=e.projectSecureId;return{Key:this.generateS3Key(t,n,e.actionRunId),Body:this.serializeDefenderContext(e),ContentType:`application/json`}}serializeDefenderContext(e){let t=e.defenderContext;return t?JSON.stringify(t):JSON.stringify({data:null,metadata:null})}};const buildHttpClient=e=>{try{return new c({logger:e})}catch(t){let n=t;e?.error({message:`Error building http client: ${n.message}`,error:n,code:`BuildHttpClientError`,category:`buildHttpClient`});return}},buildKafkaClient=(e,t)=>{try{return new l(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},safeSerialize=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var TinybirdClient=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n}async query(e){if(!this.#e)throw this.#n?.error({message:`HTTP client not initialized, cannot perform Tinybird query`,category:`TinybirdClient`,code:`HttpClientNotReady`}),Error(`HTTP client is not initialized`);if(!this.#t?.token)throw this.#n?.warning({message:`Missing OLAP token, cannot perform Tinybird query`,category:`TinybirdClient`}),Error(`TinybirdClient - Missing token`);let t=new URL(`/v0/pipes/${e.endpoint}`,this.#t.baseUrl);try{this.#n?.debug({message:`Querying Tinybird endpoint: ${e.endpoint}`,category:`TinybirdClient`,context:{endpoint:e.endpoint,params:e.params}});let n=await this.#e.request({method:`post`,url:t.toString(),payload:e.params,headers:{"Content-Type":`application/json`,Authorization:`Bearer ${this.#t.token}`},...this.#t.allowHttp&&{httpAgent:this.#r()}}),r=n.data.data;return this.#n?.debug({message:`Tinybird query returned ${r.length} rows`,category:`TinybirdClient`,context:{endpoint:e.endpoint,rows:n.data.rows,statistics:n.data.statistics}}),r}catch(t){let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryWithPagination(e){if(!this.#e)throw this.#n?.error({message:`HTTP client not initialized, cannot perform Tinybird query`,category:`TinybirdClient`,code:`HttpClientNotReady`}),Error(`HTTP client is not initialized`);if(!this.#t?.token)throw this.#n?.warning({message:`Missing OLAP token, cannot perform Tinybird query`,category:`TinybirdClient`}),Error(`TinybirdClient - Missing token`);let t=new URL(`/v0/pipes/${e.endpoint}`,this.#t.baseUrl);try{let n=await this.#e.request({method:`post`,url:t.toString(),payload:e.params,headers:{"Content-Type":`application/json`,Authorization:`Bearer ${this.#t.token}`},...this.#t.allowHttp&&{httpAgent:this.#r()}}),r=n.data.data;return{data:r,total:n.data.rows_before_limit_at_least??n.data.rows??r.length,pageNumber:e.pageNumber??1,pageSize:e.pageSize??r.length}}catch(t){let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}isReady(){return!!(this.#e&&this.#t?.token)}#r(){return new d.Agent({})}},LogsSink=class{#e;#t;#n;#r;constructor(e,t,n,r){this.#e=e,this.#r=r,this.#n=new TinybirdClient(t,n,r)}async initialize(){if(!this.#e){this.#r?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(this.#n?.isReady()||this.#r?.warning({message:`Tinybird client is not ready (missing HTTP client or Tinybird config/token), logs sink will not be able to query`,category:`LogsSink`}),!this.#t){try{this.#t=this.#e.producer({createPartitioner:u.DefaultPartitioner})}catch(e){this.#r?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#r?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#r?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async getAction(e){return(await this.getActions({actionRunId:e})).actionLogs[0]}async getActions(e){try{this.#r?.info({message:`Querying action logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_actions_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} action logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{actionLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#n?.isReady()?this.#r?.warning({message:`Failed to query Tinybird for action logs, returning empty results`,category:`LogsSink`,error:t}):this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getSteps(e){return(await this.querySteps({actionRunId:e})).stepLogs}async querySteps(e){try{this.#r?.info({message:`Querying step logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_steps_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} step logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{stepLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async send(e,t){if(!this.#t)throw this.#r?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=safeSerialize(t);this.#r?.debug({message:`Sending to topic ${e}: ${n}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:n}]});this.#r?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#r?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,riskLevel:t.defenderContext?.riskLevel,tier2Score:t.defenderContext?.tier2Score,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const h={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1,includeBackground:!1},defender:{enabled:!0}};function resolveOlapOptions(e){return{logs:{...h.logs,...e?.logs},advanced:{...h.advanced,...e?.advanced},defender:{...h.defender,...e?.defender}}}var OlapClient=class{#e;#t;#n;#r;#i;#a;#o;constructor({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={}){this.name=`OlapClient`,this.#e=e(r,o),this.#t=t(o),this.#n=n(i,o),this.#r=o,this.#i=new LogsSink(this.#e,this.#t,a,this.#r),this.#a=new AdvancedSink(this.#n,this.#r),this.#o=new DefenderSink(this.#n,this.#r)}async initialize(){await this.#i?.initialize(),await this.#a?.initialize(),await this.#o?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i,defender:a}=resolveOlapOptions(n);if(this.#r?.debug({message:`Olap client called to record action`,category:`OlapClient`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,options:n,resolvedOptions:{logs:r,advanced:i,defender:a}}}),r?.enabled)try{await this.#i?.sendAction(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendAction(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to advanced logs s3: ${e.message}`,category:`OlapClient`,error:e})}if(a?.enabled)try{await this.#o?.sendAction(e,t,a)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to defender logs s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=resolveOlapOptions(n);if(r?.enabled)try{await this.#i?.sendStep(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendStep(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async getActions(e,t,n){return this.#r?.debug({message:`Querying action logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getActions({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query action logs, returning empty result`,category:`OlapClient`}),{actionLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getAction(e,n,r){let i=await this.#i?.getActions({organizationId:e,projectSecureId:n,actionRunId:r});if(t(i)||i.actionLogs.length===0){this.#r?.debug({message:`Failed to retrieve action log for actionRunId ${r}`,category:`OlapClient`});return}return i.actionLogs[0]}async getAdvanced(e,n,r){let i=await this.#a?.getAction(e,n,r);if(t(i)){this.#r?.debug({message:`Failed to retrieve advanced log for actionRunId ${r}`,category:`OlapClient`});return}return i}async getDefender(e,n,r){let i=await this.#o?.getAction(e,n,r);if(t(i)){this.#r?.debug({message:`Failed to retrieve defender log for actionRunId ${r}`,category:`OlapClient`});return}return i}};const buildOlapClientInstance=async({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={})=>{let s=new OlapClient({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o});return await s.initialize(),s};var OlapClientManager=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o,getOlapClient:s=buildOlapClientInstance}={}){return this.olapClientPromise||=s({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{OlapClient,OlapClientManager};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stackone/olap",
-    "version": "1.16.0",
+    "version": "1.17.1",
     "description": "",
     "main": "dist/index.cjs",
     "module": "dist/index.mjs",
@@ -42,7 +42,9 @@
     "license": "ISC",
     "dependencies": {
         "@aws-sdk/client-s3": "3.913.0",
+        "@stackone/logger": "*",
         "@stackone/redaction": "*",
+        "@stackone/transport": "*",
         "@stackone/utils": "*",
         "kafkajs": "2.2.4",
         "date-fns": "2.30.0"