@stackone/olap 1.21.0 → 1.22.0

package/dist/index.cjs

@@ -1 +1 @@
-var e=Object.create,t=Object.defineProperty,__name=(e,n)=>t(e,`name`,{value:n,configurable:!0}),n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,__copyProps=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},__toESM=(n,r,a)=>(a=n==null?{}:e(i(n)),__copyProps(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n));let o=require(`@stackone/utils`),s=require(`@stackone/redaction`),c=require(`@aws-sdk/client-s3`),l=require(`@stackone/transport`),u=require(`kafkajs`),d=require(`node:http`);d=__toESM(d);const f=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],p=[`refresh_authentication`],m=10*1024*1024;var AdvancedSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send action`,category:`AdvancedSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}if(n.errorsOnly&&t.success){this.#t?.debug({message:`Advanced sink errorsOnly is enabled, skipping successful action for ${e.mode||`action`}`,category:`AdvancedSink`});return}if(this.isBackgroundLog(e)&&!n.includeBackground){this.#t?.debug({message:`Advanced sink is configured to exclude background logs, skipping action with mode ${e.mode}`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send step`,category:`AdvancedSink`,context:{actionRunId:e.actionRunId,success:t.success,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from advanced sink`,category:`AdvancedSink`});return}let r=process.env.ADVANCED_LOGS_BUCKET;if(!r)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let i=new c.GetObjectCommand({Bucket:r,Key:`${e}/${t}/${n}/${n}.json`}),a=await this.#e.send(i);if(!a.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`AdvancedSink`});return}let o=await a.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved advanced log from S3 for actionRunId ${n}`,category:`AdvancedSink`}),JSON.parse(o)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Advanced log not found in S3 for actionRunId ${n}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting advanced log from S3 for actionRunId ${n}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async getStep(e,t,n,r){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get step log from advanced sink`,category:`AdvancedSink`});return}let i=process.env.ADVANCED_LOGS_BUCKET;if(!i)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let a=new c.GetObjectCommand({Bucket:i,Key:`${e}/${t}/${n}/steps/${r}.json`}),o=await this.#e.send(a);if(!o.Body){this.#t?.warning({message:`Received empty body when trying to get step log from S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`});return}let s=await o.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved step log from S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`}),JSON.parse(s)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Step log not found in S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting step log from S3 for actionRunId ${n} stepIndex ${r}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let n=process.env.ADVANCED_LOGS_BUCKET;if(!n)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let r=`ttl=30d`;typeof t==`number`&&(t===1?r=`ttl=1d`:t===7&&(r=`ttl=7d`));let i=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;this.#t?.debug({message:`Storing advanced log in S3 bucket ${n} with ttl of ${r}`,category:`AdvancedSink`,context:{bucket:n,key:e.Key,ttlTag:r,expiresAt:new Date(i*1e3).toISOString()}});try{await this.#e.send(new c.PutObjectCommand({Bucket:n,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(i*1e3),Tagging:r}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,m),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,m),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(e,t,n,r){let i=(0,s.redactObject)({req:{...e.headers&&{headers:{...e.headers}}},res:{...t.headers&&{headers:{...t.headers}}}},s.CensorType.PARTIAL),a=i.req?.headers,o=i.res?.headers,c=(0,s.redactObject)(e.body,s.CensorType.PARTIAL),l=(0,s.redactObject)(t.body,s.CensorType.PARTIAL),u=this.isBackgroundLog(e),d={data:{request:{id:t.actionRunId,actionId:t.actionId,method:t.httpMethod,headers:this.filterHeaders(a),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:c},response:{statusCode:t.statusCode??500,headers:this.filterHeaders(o),body:l},...u?{isBackgroundLog:!0}:{}},metadata:{...u?{isBackgroundLog:!0}:{},expirationTime:n}};return this.serializeAndLimit(d,r)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,t){let n=JSON.stringify(e);if(Buffer.byteLength(n,`utf8`)>t){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};n=JSON.stringify(t)}return n}filterHeaders(e){if(!e)return;let t={},n=f.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return(0,o.notMissing)(e.mode)&&p.includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const buildS3Client=(e,t)=>{try{return new c.S3Client(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}};var DefenderSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, defender sink will not function`,category:`DefenderSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Defender sink called to send action`,category:`DefenderSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled}}}),(0,o.isMissing)(t.defenderContext)){this.#t?.debug({message:`No defender context in action result, skipping sending action to defender sink`,category:`DefenderSink`,context:{actionRunId:t.actionRunId}});return}if(!n?.enabled){this.#t?.debug({message:`Defender sink is disabled, skipping sending action to defender sink`,category:`DefenderSink`});return}let r=this.createActionS3(t);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from defender sink`,category:`DefenderSink`});return}let r=process.env.DEFENDER_LOGS_BUCKET;if(!r)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);try{let i=new c.GetObjectCommand({Bucket:r,Key:this.generateS3Key(e,t,n)}),a=await this.#e.send(i);if(!a.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`DefenderSink`});return}let o=await a.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved defender log from S3 for actionRunId ${n}`,category:`DefenderSink`}),JSON.parse(o)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Defender log not found in S3 for actionRunId ${n}`,category:`DefenderSink`});return}throw this.#t?.error({message:`Error when getting defender log from S3 for actionRunId ${n}`,error:e,code:`DefenderLogReadError`,category:`DefenderSink`}),e}}async send(e){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to defender sink`,category:`DefenderSink`});return}let t=process.env.DEFENDER_LOGS_BUCKET;if(!t)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);this.#t?.debug({message:`Storing defender log in S3 bucket ${t}`,category:`DefenderSink`,context:{bucket:t,key:e.Key}});try{await this.#e.send(new c.PutObjectCommand({Bucket:t,Key:e.Key,Body:e.Body,ContentType:e.ContentType}))}catch(t){throw this.#t?.error({message:`Failed to write defender logs to S3 at ${e.Key}`,error:t,code:`DefenderLogWriteError`,category:`DefenderSink`}),t}}generateS3Key(e,t,n){return`${e}/${t}/${n}/defender.json`}createActionS3(e){let t=e.organizationId,n=e.projectSecureId;return{Key:this.generateS3Key(t,n,e.actionRunId),Body:JSON.stringify(e.defenderContext),ContentType:`application/json`}}};const buildHttpClient=e=>{try{return new l.HttpClient({logger:e})}catch(t){let n=t;e?.error({message:`Error building http client: ${n.message}`,error:n,code:`BuildHttpClientError`,category:`buildHttpClient`});return}},buildKafkaClient=(e,t)=>{try{return new u.Kafka(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},safeSerialize=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var SchemaValidationError=class extends Error{constructor(e,t){super(e),this.name=`SchemaValidationError`,this.cause=t}},TinybirdClient=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n}async query(e){try{let t=await this.#o(e.endpoint,e.params);return this.#a(t.data,e.schema)}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryWithPagination(e){try{let t=await this.#o(e.endpoint,e.params),n=this.#a(t.data,e.schema);return{data:n,total:t.rows_before_limit_at_least??t.rows??n.length,pageNumber:e.pageNumber??1,pageSize:e.pageSize??n.length}}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryRaw(e){try{let t=await this.#o(e.endpoint,e.params);return t.data=this.#a(t.data,e.schema),t}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}isReady(){return!!(this.#e&&this.#t?.token)}#r(){return new d.default.Agent({})}#i(){if(!this.#e)throw this.#n?.error({message:`HTTP client not initialized, cannot perform Tinybird query`,category:`TinybirdClient`,code:`HttpClientNotReady`}),Error(`HTTP client is not initialized`);if(!this.#t?.token)throw this.#n?.warning({message:`Missing OLAP token, cannot perform Tinybird query`,category:`TinybirdClient`}),Error(`TinybirdClient - Missing token`)}#a(e,t){if(!t)return e;try{let n=e.map(e=>t.parse(e));return this.#n?.debug({message:`Schema validation passed for ${n.length} rows`,category:`TinybirdClient`}),n}catch(e){let t=e instanceof Error?e.message:String(e);throw this.#n?.error({message:`Schema validation failed: ${t}`,category:`TinybirdClient`,code:`SchemaValidationError`,error:e}),new SchemaValidationError(t,e)}}async#o(e,t){this.#i();let n=this.#e,r=this.#t;if(!n||!r)throw Error(`Client validation failed`);let i=new URL(`/v0/pipes/${e}`,r.baseUrl);this.#n?.debug({message:`Querying Tinybird endpoint: ${e}`,category:`TinybirdClient`,context:{endpoint:e,params:t}});let a=await n.request({method:`post`,url:i.toString(),payload:t,headers:{"Content-Type":`application/json`,Authorization:`Bearer ${r.token}`},...r.allowHttp&&{httpAgent:this.#r()}});return this.#n?.debug({message:`Tinybird query returned ${a.data.data.length} rows`,category:`TinybirdClient`,context:{endpoint:e,rows:a.data.rows,statistics:a.data.statistics}}),a.data}};const h=`project_actions_logs.json`,g=`project_steps_logs.json`;var LogsSink=class{#e;#t;#n;#r;constructor(e,t,n,r){this.#e=e,this.#r=r,this.#n=new TinybirdClient(t,n,r)}get tinybirdClient(){return this.#n}async initialize(){if(!this.#e){this.#r?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(this.#n?.isReady()||this.#r?.warning({message:`Tinybird client is not ready (missing HTTP client or Tinybird config/token), logs sink will not be able to query`,category:`LogsSink`}),!this.#t){try{this.#t=this.#e.producer({createPartitioner:u.Partitioners.DefaultPartitioner,maxInFlightRequests:2,idempotent:!1})}catch(e){this.#r?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#i();try{await this.#t.connect()}catch(e){this.#r?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#r?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async getActions(e){try{this.#r?.info({message:`Querying action logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_actions_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} action logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{actionLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#n?.isReady()?this.#r?.warning({message:`Failed to query Tinybird for action logs, returning empty results`,category:`LogsSink`,error:t}):this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getAction(e,t,n){return(await this.getActions({organizationId:e,projectSecureId:t,actionRunId:n})).actionLogs[0]}async getSteps(e){try{this.#r?.info({message:`Querying step logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_steps_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} step logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{stepLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getStep(e,t,n,r){return(await this.getSteps({organizationId:e,projectSecureId:t,actionRunId:n,stepIndex:r})).stepLogs[0]}#i(){this.#t&&(this.#t.on(this.#t.events.CONNECT,()=>{this.#r?.info({message:`Kafka producer connected successfully`,category:`LogsSink`})}),this.#t.on(this.#t.events.DISCONNECT,()=>{this.#r?.warning({message:`Kafka producer disconnected`,category:`LogsSink`})}),this.#t.on(this.#t.events.REQUEST_TIMEOUT,({payload:e})=>{this.#r?.error({message:`Kafka producer request timeout`,category:`LogsSink`,code:`KafkaProducerRequestTimeout`,context:{payload:e}})}))}async send(e,t){if(!this.#t)throw this.#r?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=Date.now(),r=safeSerialize(t);this.#r?.debug({message:`Sending to topic ${e}: ${r}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:r}],acks:1,compression:u.CompressionTypes.GZIP});this.#r?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`,context:{durationMs:Date.now()-n,topic:e}})}catch(t){throw this.#r?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`,context:{durationMs:Date.now()-n,topic:e}}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,riskLevel:t.defenderContext?.riskLevel,tier2Score:t.defenderContext?.tier2Score,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const _={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1,includeBackground:!1},defender:{enabled:!0}};function resolveOlapOptions(e){return{logs:{..._.logs,...e?.logs},advanced:{..._.advanced,...e?.advanced},defender:{..._.defender,...e?.defender}}}var OlapClient=class{#e;#t;#n;#r;#i;#a;#o;constructor({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={}){this.name=`OlapClient`,this.#e=e(r,o),this.#t=t(o),this.#n=n(i,o),this.#r=o,this.#i=new LogsSink(this.#e,this.#t,a,this.#r),this.#a=new AdvancedSink(this.#n,this.#r),this.#o=new DefenderSink(this.#n,this.#r)}async initialize(){await this.#i?.initialize(),await this.#a?.initialize(),await this.#o?.initialize()}get tinybirdClient(){return this.#i?.tinybirdClient}async recordAction(e,t,n){let{logs:r,advanced:i,defender:a}=resolveOlapOptions(n);if(this.#r?.debug({message:`Olap client called to record action`,category:`OlapClient`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,options:n,resolvedOptions:{logs:r,advanced:i,defender:a}}}),r?.enabled)try{await this.#i?.sendAction(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendAction(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to advanced logs s3: ${e.message}`,category:`OlapClient`,error:e})}if(a?.enabled)try{await this.#o?.sendAction(e,t,a)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to defender logs s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=resolveOlapOptions(n);if(r?.enabled)try{await this.#i?.sendStep(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendStep(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async getActionLogs(e,t,n){return this.#r?.debug({message:`Querying action logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getActions({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query action logs, returning empty result`,category:`OlapClient`}),{actionLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getActionLog(e,t,n){let r=await this.#i?.getAction(e,t,n);if((0,o.isMissing)(r)){this.#r?.debug({message:`Failed to retrieve action log for actionRunId ${n}`,category:`OlapClient`});return}return r}async getStepLogs(e,t,n){return this.#r?.debug({message:`Querying step logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getSteps({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query step logs, returning empty result`,category:`OlapClient`}),{stepLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getStepLog(e,t,n,r){let i=await this.#i?.getStep(e,t,n,r);if((0,o.isMissing)(i)){this.#r?.debug({message:`Failed to retrieve step log for actionRunId ${n} stepIndex ${r}`,category:`OlapClient`});return}return i}async getActionAdvancedLog(e,t,n){let r=await this.#a?.getAction(e,t,n);if((0,o.isMissing)(r)){this.#r?.debug({message:`Failed to retrieve action advanced log for actionRunId ${n}`,category:`OlapClient`});return}return r}async getStepAdvancedLog(e,t,n,r){let i=await this.#a?.getStep(e,t,n,r);if((0,o.isMissing)(i)){this.#r?.debug({message:`Failed to retrieve step advanced log for actionRunId ${n} stepIndex ${r}`,category:`OlapClient`});return}return i}async getDefenderLog(e,t,n){let r=await this.#o?.getAction(e,t,n);if((0,o.isMissing)(r)){this.#r?.debug({message:`Failed to retrieve defender log for actionRunId ${n}`,category:`OlapClient`});return}return r}};const buildOlapClientInstance=async({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={})=>{let s=new OlapClient({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o});return await s.initialize(),s};var OlapClientManager=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o,getOlapClient:s=buildOlapClientInstance}={}){return this.olapClientPromise||=s({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=OlapClient,exports.OlapClientManager=OlapClientManager;
+var e=Object.create,t=Object.defineProperty,__name=(e,n)=>t(e,`name`,{value:n,configurable:!0}),n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,__copyProps=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},__toESM=(n,r,a)=>(a=n==null?{}:e(i(n)),__copyProps(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n));let o=require(`@stackone/utils`),s=require(`@stackone/redaction`),c=require(`@aws-sdk/client-s3`),l=require(`@stackone/transport`),u=require(`kafkajs`),d=require(`node:http`);d=__toESM(d);const f=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],p=[`refresh_authentication`],m=10*1024*1024;var AdvancedSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send action`,category:`AdvancedSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}if(n.errorsOnly&&t.success){this.#t?.debug({message:`Advanced sink errorsOnly is enabled, skipping successful action for ${e.mode||`action`}`,category:`AdvancedSink`});return}if(this.isBackgroundLog(e)&&!n.includeBackground){this.#t?.debug({message:`Advanced sink is configured to exclude background logs, skipping action with mode ${e.mode}`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send step`,category:`AdvancedSink`,context:{actionRunId:e.actionRunId,success:t.success,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from advanced sink`,category:`AdvancedSink`});return}let r=process.env.ADVANCED_LOGS_BUCKET;if(!r)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let i=new c.GetObjectCommand({Bucket:r,Key:`${e}/${t}/${n}/${n}.json`}),a=await this.#e.send(i);if(!a.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`AdvancedSink`});return}let o=await a.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved advanced log from S3 for actionRunId ${n}`,category:`AdvancedSink`}),JSON.parse(o)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Advanced log not found in S3 for actionRunId ${n}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting advanced log from S3 for actionRunId ${n}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async getStep(e,t,n,r){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get step log from advanced sink`,category:`AdvancedSink`});return}let i=process.env.ADVANCED_LOGS_BUCKET;if(!i)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let a=new c.GetObjectCommand({Bucket:i,Key:`${e}/${t}/${n}/steps/${r}.json`}),o=await this.#e.send(a);if(!o.Body){this.#t?.warning({message:`Received empty body when trying to get step log from S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`});return}let s=await o.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved step log from S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`}),JSON.parse(s)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Step log not found in S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting step log from S3 for actionRunId ${n} stepIndex ${r}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let n=process.env.ADVANCED_LOGS_BUCKET;if(!n)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let r=`ttl=30d`;typeof t==`number`&&(t===1?r=`ttl=1d`:t===7&&(r=`ttl=7d`));let i=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;this.#t?.debug({message:`Storing advanced log in S3 bucket ${n} with ttl of ${r}`,category:`AdvancedSink`,context:{bucket:n,key:e.Key,ttlTag:r,expiresAt:new Date(i*1e3).toISOString()}});try{await this.#e.send(new c.PutObjectCommand({Bucket:n,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(i*1e3),Tagging:r}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,m),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,m),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(e,t,n,r){let i=(0,s.redactObject)({req:{...e.headers&&{headers:{...e.headers}}},res:{...t.headers&&{headers:{...t.headers}}}},s.CensorType.PARTIAL),a=i.req?.headers,o=i.res?.headers,c=(0,s.redactObject)(e.body,s.CensorType.PARTIAL),l=(0,s.redactObject)(t.body,s.CensorType.PARTIAL),u=this.isBackgroundLog(e),d={data:{request:{id:t.actionRunId,actionId:t.actionId,method:t.httpMethod,headers:this.filterHeaders(a),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:c},response:{statusCode:t.statusCode??500,headers:this.filterHeaders(o),body:l},...u?{isBackgroundLog:!0}:{}},metadata:{...u?{isBackgroundLog:!0}:{},expirationTime:n}};return this.serializeAndLimit(d,r)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,t){if((0,o.exceedsSize)(e,t)){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};return JSON.stringify(t)}return JSON.stringify(e)}filterHeaders(e){if(!e)return;let t={},n=f.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return(0,o.notMissing)(e.mode)&&p.includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const buildS3Client=(e,t)=>{try{return new c.S3Client(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}};var DefenderSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, defender sink will not function`,category:`DefenderSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Defender sink called to send action`,category:`DefenderSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled}}}),(0,o.isMissing)(t.defenderContext)){this.#t?.debug({message:`No defender context in action result, skipping sending action to defender sink`,category:`DefenderSink`,context:{actionRunId:t.actionRunId}});return}if(!n?.enabled){this.#t?.debug({message:`Defender sink is disabled, skipping sending action to defender sink`,category:`DefenderSink`});return}let r=this.createActionS3(t);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from defender sink`,category:`DefenderSink`});return}let r=process.env.DEFENDER_LOGS_BUCKET;if(!r)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);try{let i=new c.GetObjectCommand({Bucket:r,Key:this.generateS3Key(e,t,n)}),a=await this.#e.send(i);if(!a.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`DefenderSink`});return}let o=await a.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved defender log from S3 for actionRunId ${n}`,category:`DefenderSink`}),JSON.parse(o)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Defender log not found in S3 for actionRunId ${n}`,category:`DefenderSink`});return}throw this.#t?.error({message:`Error when getting defender log from S3 for actionRunId ${n}`,error:e,code:`DefenderLogReadError`,category:`DefenderSink`}),e}}async send(e){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to defender sink`,category:`DefenderSink`});return}let t=process.env.DEFENDER_LOGS_BUCKET;if(!t)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);this.#t?.debug({message:`Storing defender log in S3 bucket ${t}`,category:`DefenderSink`,context:{bucket:t,key:e.Key}});try{await this.#e.send(new c.PutObjectCommand({Bucket:t,Key:e.Key,Body:e.Body,ContentType:e.ContentType}))}catch(t){throw this.#t?.error({message:`Failed to write defender logs to S3 at ${e.Key}`,error:t,code:`DefenderLogWriteError`,category:`DefenderSink`}),t}}generateS3Key(e,t,n){return`${e}/${t}/${n}/defender.json`}createActionS3(e){let t=e.organizationId,n=e.projectSecureId;return{Key:this.generateS3Key(t,n,e.actionRunId),Body:JSON.stringify(e.defenderContext),ContentType:`application/json`}}};const buildHttpClient=e=>{try{return new l.HttpClient({logger:e})}catch(t){let n=t;e?.error({message:`Error building http client: ${n.message}`,error:n,code:`BuildHttpClientError`,category:`buildHttpClient`});return}},buildKafkaClient=(e,t)=>{try{return new u.Kafka(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},safeSerialize=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var SchemaValidationError=class extends Error{constructor(e,t){super(e),this.name=`SchemaValidationError`,this.cause=t}},TinybirdClient=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n}async query(e){try{let t=await this.#o(e.endpoint,e.params);return this.#a(t.data,e.schema)}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryWithPagination(e){try{let t=await this.#o(e.endpoint,e.params),n=this.#a(t.data,e.schema);return{data:n,total:t.rows_before_limit_at_least??t.rows??n.length,pageNumber:e.pageNumber??1,pageSize:e.pageSize??n.length}}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryRaw(e){try{let t=await this.#o(e.endpoint,e.params);return t.data=this.#a(t.data,e.schema),t}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}isReady(){return!!(this.#e&&this.#t?.token)}#r(){return new d.default.Agent({})}#i(){if(!this.#e)throw this.#n?.error({message:`HTTP client not initialized, cannot perform Tinybird query`,category:`TinybirdClient`,code:`HttpClientNotReady`}),Error(`HTTP client is not initialized`);if(!this.#t?.token)throw this.#n?.warning({message:`Missing OLAP token, cannot perform Tinybird query`,category:`TinybirdClient`}),Error(`TinybirdClient - Missing token`)}#a(e,t){if(!t)return e;try{let n=e.map(e=>t.parse(e));return this.#n?.debug({message:`Schema validation passed for ${n.length} rows`,category:`TinybirdClient`}),n}catch(e){let t=e instanceof Error?e.message:String(e);throw this.#n?.error({message:`Schema validation failed: ${t}`,category:`TinybirdClient`,code:`SchemaValidationError`,error:e}),new SchemaValidationError(t,e)}}async#o(e,t){this.#i();let n=this.#e,r=this.#t;if(!n||!r)throw Error(`Client validation failed`);let i=new URL(`/v0/pipes/${e}`,r.baseUrl);this.#n?.debug({message:`Querying Tinybird endpoint: ${e}`,category:`TinybirdClient`,context:{endpoint:e,params:t}});let a=await n.request({method:`post`,url:i.toString(),payload:t,headers:{"Content-Type":`application/json`,Authorization:`Bearer ${r.token}`},...r.allowHttp&&{httpAgent:this.#r()}});return this.#n?.debug({message:`Tinybird query returned ${a.data.data.length} rows`,category:`TinybirdClient`,context:{endpoint:e,rows:a.data.rows,statistics:a.data.statistics}}),a.data}};const h=`project_actions_logs.json`,g=`project_steps_logs.json`;var LogsSink=class{#e;#t;#n;#r;constructor(e,t,n,r){this.#e=e,this.#r=r,this.#n=new TinybirdClient(t,n,r)}get tinybirdClient(){return this.#n}async initialize(){if(!this.#e){this.#r?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(this.#n?.isReady()||this.#r?.warning({message:`Tinybird client is not ready (missing HTTP client or Tinybird config/token), logs sink will not be able to query`,category:`LogsSink`}),!this.#t){try{this.#t=this.#e.producer({createPartitioner:u.Partitioners.DefaultPartitioner,maxInFlightRequests:2,idempotent:!1})}catch(e){this.#r?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#i();try{await this.#t.connect()}catch(e){this.#r?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#r?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async getActions(e){try{this.#r?.info({message:`Querying action logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_actions_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} action logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{actionLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#n?.isReady()?this.#r?.warning({message:`Failed to query Tinybird for action logs, returning empty results`,category:`LogsSink`,error:t}):this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getAction(e,t,n){return(await this.getActions({organizationId:e,projectSecureId:t,actionRunId:n})).actionLogs[0]}async getSteps(e){try{this.#r?.info({message:`Querying step logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_steps_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} step logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{stepLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getStep(e,t,n,r){return(await this.getSteps({organizationId:e,projectSecureId:t,actionRunId:n,stepIndex:r})).stepLogs[0]}#i(){this.#t&&(this.#t.on(this.#t.events.CONNECT,()=>{this.#r?.info({message:`Kafka producer connected successfully`,category:`LogsSink`})}),this.#t.on(this.#t.events.DISCONNECT,()=>{this.#r?.warning({message:`Kafka producer disconnected`,category:`LogsSink`})}),this.#t.on(this.#t.events.REQUEST_TIMEOUT,({payload:e})=>{this.#r?.error({message:`Kafka producer request timeout`,category:`LogsSink`,code:`KafkaProducerRequestTimeout`,context:{payload:e}})}))}async send(e,t){if(!this.#t)throw this.#r?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=Date.now(),r=safeSerialize(t);this.#r?.debug({message:`Sending to topic ${e}: ${r}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:r}],acks:1,compression:u.CompressionTypes.GZIP});this.#r?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`,context:{durationMs:Date.now()-n,topic:e}})}catch(t){throw this.#r?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`,context:{durationMs:Date.now()-n,topic:e}}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,riskLevel:t.defenderContext?.riskLevel,tier2Score:t.defenderContext?.tier2Score,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const _={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1,includeBackground:!1},defender:{enabled:!0}};function resolveOlapOptions(e){return{logs:{..._.logs,...e?.logs},advanced:{..._.advanced,...e?.advanced},defender:{..._.defender,...e?.defender}}}var OlapClient=class{#e;#t;#n;#r;#i;#a;#o;constructor({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={}){this.name=`OlapClient`,this.#e=e(r,o),this.#t=t(o),this.#n=n(i,o),this.#r=o,this.#i=new LogsSink(this.#e,this.#t,a,this.#r),this.#a=new AdvancedSink(this.#n,this.#r),this.#o=new DefenderSink(this.#n,this.#r)}async initialize(){await this.#i?.initialize(),await this.#a?.initialize(),await this.#o?.initialize()}get tinybirdClient(){return this.#i?.tinybirdClient}async recordAction(e,t,n){let{logs:r,advanced:i,defender:a}=resolveOlapOptions(n);if(this.#r?.debug({message:`Olap client called to record action`,category:`OlapClient`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,options:n,resolvedOptions:{logs:r,advanced:i,defender:a}}}),r?.enabled)try{await this.#i?.sendAction(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendAction(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to advanced logs s3: ${e.message}`,category:`OlapClient`,error:e})}if(a?.enabled)try{await this.#o?.sendAction(e,t,a)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to defender logs s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=resolveOlapOptions(n);if(r?.enabled)try{await this.#i?.sendStep(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendStep(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async getActionLogs(e,t,n){return this.#r?.debug({message:`Querying action logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getActions({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query action logs, returning empty result`,category:`OlapClient`}),{actionLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getActionLog(e,t,n){let r=await this.#i?.getAction(e,t,n);if((0,o.isMissing)(r)){this.#r?.debug({message:`Failed to retrieve action log for actionRunId ${n}`,category:`OlapClient`});return}return r}async getStepLogs(e,t,n){return this.#r?.debug({message:`Querying step logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getSteps({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query step logs, returning empty result`,category:`OlapClient`}),{stepLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getStepLog(e,t,n,r){let i=await this.#i?.getStep(e,t,n,r);if((0,o.isMissing)(i)){this.#r?.debug({message:`Failed to retrieve step log for actionRunId ${n} stepIndex ${r}`,category:`OlapClient`});return}return i}async getActionAdvancedLog(e,t,n){let r=await this.#a?.getAction(e,t,n);if((0,o.isMissing)(r)){this.#r?.debug({message:`Failed to retrieve action advanced log for actionRunId ${n}`,category:`OlapClient`});return}return r}async getStepAdvancedLog(e,t,n,r){let i=await this.#a?.getStep(e,t,n,r);if((0,o.isMissing)(i)){this.#r?.debug({message:`Failed to retrieve step advanced log for actionRunId ${n} stepIndex ${r}`,category:`OlapClient`});return}return i}async getDefenderLog(e,t,n){let r=await this.#o?.getAction(e,t,n);if((0,o.isMissing)(r)){this.#r?.debug({message:`Failed to retrieve defender log for actionRunId ${n}`,category:`OlapClient`});return}return r}};const buildOlapClientInstance=async({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={})=>{let s=new OlapClient({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o});return await s.initialize(),s};var OlapClientManager=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o,getOlapClient:s=buildOlapClientInstance}={}){return this.olapClientPromise||=s({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=OlapClient,exports.OlapClientManager=OlapClientManager;

package/dist/index.d.cts

@@ -100,6 +100,7 @@ type DefenderLog = {
 //#region ../transport/src/cacheClient/types.d.ts
 interface ICacheClient<ClientType = unknown> {
   getData<T>(key: string): Promise<T | null>;
+  getDataKeysFromGroup(groupKey: string): Promise<string[] | null>;
   listData<T>({
     partialKey,
     limit,
@@ -116,13 +117,16 @@ interface ICacheClient<ClientType = unknown> {
     key,
     value,
     cacheTTL,
-    groupKey
+    groupKeys,
+    groupKeyTTL
   }: {
     key: string;
     value: T;
     cacheTTL: number;
-    groupKey?: string;
+    groupKeys?: string[];
+    groupKeyTTL?: number;
   }): Promise<boolean>;
+  setTTL(key: string, cacheTTL: number): Promise<boolean>;
   executeScript?<T>({
     sha1,
     keys,

package/dist/index.d.mts

@@ -100,6 +100,7 @@ type DefenderLog = {
 //#region ../transport/src/cacheClient/types.d.ts
 interface ICacheClient<ClientType = unknown> {
   getData<T>(key: string): Promise<T | null>;
+  getDataKeysFromGroup(groupKey: string): Promise<string[] | null>;
   listData<T>({
     partialKey,
     limit,
@@ -116,13 +117,16 @@ interface ICacheClient<ClientType = unknown> {
     key,
     value,
     cacheTTL,
-    groupKey
+    groupKeys,
+    groupKeyTTL
   }: {
     key: string;
     value: T;
     cacheTTL: number;
-    groupKey?: string;
+    groupKeys?: string[];
+    groupKeyTTL?: number;
   }): Promise<boolean>;
+  setTTL(key: string, cacheTTL: number): Promise<boolean>;
   executeScript?<T>({
     sha1,
     keys,

package/dist/index.mjs

@@ -1 +1 @@
-import{t as e}from"./chunk-Cfxk5zVN.mjs";import{isMissing as t,notMissing as n}from"@stackone/utils";import{CensorType as r,redactObject as i}from"@stackone/redaction";import{GetObjectCommand as a,PutObjectCommand as o,S3Client as s}from"@aws-sdk/client-s3";import{HttpClient as c}from"@stackone/transport";import{CompressionTypes as l,Kafka as u,Partitioners as d}from"kafkajs";import f from"node:http";const p=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],m=[`refresh_authentication`],h=10*1024*1024;var AdvancedSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send action`,category:`AdvancedSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}if(n.errorsOnly&&t.success){this.#t?.debug({message:`Advanced sink errorsOnly is enabled, skipping successful action for ${e.mode||`action`}`,category:`AdvancedSink`});return}if(this.isBackgroundLog(e)&&!n.includeBackground){this.#t?.debug({message:`Advanced sink is configured to exclude background logs, skipping action with mode ${e.mode}`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send step`,category:`AdvancedSink`,context:{actionRunId:e.actionRunId,success:t.success,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from advanced sink`,category:`AdvancedSink`});return}let r=process.env.ADVANCED_LOGS_BUCKET;if(!r)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let i=new a({Bucket:r,Key:`${e}/${t}/${n}/${n}.json`}),o=await this.#e.send(i);if(!o.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`AdvancedSink`});return}let s=await o.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved advanced log from S3 for actionRunId ${n}`,category:`AdvancedSink`}),JSON.parse(s)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Advanced log not found in S3 for actionRunId ${n}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting advanced log from S3 for actionRunId ${n}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async getStep(e,t,n,r){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get step log from advanced sink`,category:`AdvancedSink`});return}let i=process.env.ADVANCED_LOGS_BUCKET;if(!i)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let o=new a({Bucket:i,Key:`${e}/${t}/${n}/steps/${r}.json`}),s=await this.#e.send(o);if(!s.Body){this.#t?.warning({message:`Received empty body when trying to get step log from S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`});return}let c=await s.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved step log from S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`}),JSON.parse(c)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Step log not found in S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting step log from S3 for actionRunId ${n} stepIndex ${r}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let n=process.env.ADVANCED_LOGS_BUCKET;if(!n)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let r=`ttl=30d`;typeof t==`number`&&(t===1?r=`ttl=1d`:t===7&&(r=`ttl=7d`));let i=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;this.#t?.debug({message:`Storing advanced log in S3 bucket ${n} with ttl of ${r}`,category:`AdvancedSink`,context:{bucket:n,key:e.Key,ttlTag:r,expiresAt:new Date(i*1e3).toISOString()}});try{await this.#e.send(new o({Bucket:n,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(i*1e3),Tagging:r}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,h),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,h),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(e,t,n,a){let o=i({req:{...e.headers&&{headers:{...e.headers}}},res:{...t.headers&&{headers:{...t.headers}}}},r.PARTIAL),s=o.req?.headers,c=o.res?.headers,l=i(e.body,r.PARTIAL),u=i(t.body,r.PARTIAL),d=this.isBackgroundLog(e),f={data:{request:{id:t.actionRunId,actionId:t.actionId,method:t.httpMethod,headers:this.filterHeaders(s),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:l},response:{statusCode:t.statusCode??500,headers:this.filterHeaders(c),body:u},...d?{isBackgroundLog:!0}:{}},metadata:{...d?{isBackgroundLog:!0}:{},expirationTime:n}};return this.serializeAndLimit(f,a)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,t){let n=JSON.stringify(e);if(Buffer.byteLength(n,`utf8`)>t){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};n=JSON.stringify(t)}return n}filterHeaders(e){if(!e)return;let t={},n=p.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return n(e.mode)&&m.includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const buildS3Client=(e,t)=>{try{return new s(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}};var DefenderSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, defender sink will not function`,category:`DefenderSink`});return}}async sendAction(e,n,r){if(this.#t?.debug({message:`Defender sink called to send action`,category:`DefenderSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:n.actionRunId,success:n.success,statusCode:n.statusCode,options:{enabled:r?.enabled}}}),t(n.defenderContext)){this.#t?.debug({message:`No defender context in action result, skipping sending action to defender sink`,category:`DefenderSink`,context:{actionRunId:n.actionRunId}});return}if(!r?.enabled){this.#t?.debug({message:`Defender sink is disabled, skipping sending action to defender sink`,category:`DefenderSink`});return}let i=this.createActionS3(n);await this.send(i)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from defender sink`,category:`DefenderSink`});return}let r=process.env.DEFENDER_LOGS_BUCKET;if(!r)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);try{let i=new a({Bucket:r,Key:this.generateS3Key(e,t,n)}),o=await this.#e.send(i);if(!o.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`DefenderSink`});return}let s=await o.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved defender log from S3 for actionRunId ${n}`,category:`DefenderSink`}),JSON.parse(s)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Defender log not found in S3 for actionRunId ${n}`,category:`DefenderSink`});return}throw this.#t?.error({message:`Error when getting defender log from S3 for actionRunId ${n}`,error:e,code:`DefenderLogReadError`,category:`DefenderSink`}),e}}async send(e){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to defender sink`,category:`DefenderSink`});return}let t=process.env.DEFENDER_LOGS_BUCKET;if(!t)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);this.#t?.debug({message:`Storing defender log in S3 bucket ${t}`,category:`DefenderSink`,context:{bucket:t,key:e.Key}});try{await this.#e.send(new o({Bucket:t,Key:e.Key,Body:e.Body,ContentType:e.ContentType}))}catch(t){throw this.#t?.error({message:`Failed to write defender logs to S3 at ${e.Key}`,error:t,code:`DefenderLogWriteError`,category:`DefenderSink`}),t}}generateS3Key(e,t,n){return`${e}/${t}/${n}/defender.json`}createActionS3(e){let t=e.organizationId,n=e.projectSecureId;return{Key:this.generateS3Key(t,n,e.actionRunId),Body:JSON.stringify(e.defenderContext),ContentType:`application/json`}}};const buildHttpClient=e=>{try{return new c({logger:e})}catch(t){let n=t;e?.error({message:`Error building http client: ${n.message}`,error:n,code:`BuildHttpClientError`,category:`buildHttpClient`});return}},buildKafkaClient=(e,t)=>{try{return new u(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},safeSerialize=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var SchemaValidationError=class extends Error{constructor(e,t){super(e),this.name=`SchemaValidationError`,this.cause=t}},TinybirdClient=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n}async query(e){try{let t=await this.#o(e.endpoint,e.params);return this.#a(t.data,e.schema)}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryWithPagination(e){try{let t=await this.#o(e.endpoint,e.params),n=this.#a(t.data,e.schema);return{data:n,total:t.rows_before_limit_at_least??t.rows??n.length,pageNumber:e.pageNumber??1,pageSize:e.pageSize??n.length}}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryRaw(e){try{let t=await this.#o(e.endpoint,e.params);return t.data=this.#a(t.data,e.schema),t}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}isReady(){return!!(this.#e&&this.#t?.token)}#r(){return new f.Agent({})}#i(){if(!this.#e)throw this.#n?.error({message:`HTTP client not initialized, cannot perform Tinybird query`,category:`TinybirdClient`,code:`HttpClientNotReady`}),Error(`HTTP client is not initialized`);if(!this.#t?.token)throw this.#n?.warning({message:`Missing OLAP token, cannot perform Tinybird query`,category:`TinybirdClient`}),Error(`TinybirdClient - Missing token`)}#a(e,t){if(!t)return e;try{let n=e.map(e=>t.parse(e));return this.#n?.debug({message:`Schema validation passed for ${n.length} rows`,category:`TinybirdClient`}),n}catch(e){let t=e instanceof Error?e.message:String(e);throw this.#n?.error({message:`Schema validation failed: ${t}`,category:`TinybirdClient`,code:`SchemaValidationError`,error:e}),new SchemaValidationError(t,e)}}async#o(e,t){this.#i();let n=this.#e,r=this.#t;if(!n||!r)throw Error(`Client validation failed`);let i=new URL(`/v0/pipes/${e}`,r.baseUrl);this.#n?.debug({message:`Querying Tinybird endpoint: ${e}`,category:`TinybirdClient`,context:{endpoint:e,params:t}});let a=await n.request({method:`post`,url:i.toString(),payload:t,headers:{"Content-Type":`application/json`,Authorization:`Bearer ${r.token}`},...r.allowHttp&&{httpAgent:this.#r()}});return this.#n?.debug({message:`Tinybird query returned ${a.data.data.length} rows`,category:`TinybirdClient`,context:{endpoint:e,rows:a.data.rows,statistics:a.data.statistics}}),a.data}},LogsSink=class{#e;#t;#n;#r;constructor(e,t,n,r){this.#e=e,this.#r=r,this.#n=new TinybirdClient(t,n,r)}get tinybirdClient(){return this.#n}async initialize(){if(!this.#e){this.#r?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(this.#n?.isReady()||this.#r?.warning({message:`Tinybird client is not ready (missing HTTP client or Tinybird config/token), logs sink will not be able to query`,category:`LogsSink`}),!this.#t){try{this.#t=this.#e.producer({createPartitioner:d.DefaultPartitioner,maxInFlightRequests:2,idempotent:!1})}catch(e){this.#r?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#i();try{await this.#t.connect()}catch(e){this.#r?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#r?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async getActions(e){try{this.#r?.info({message:`Querying action logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_actions_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} action logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{actionLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#n?.isReady()?this.#r?.warning({message:`Failed to query Tinybird for action logs, returning empty results`,category:`LogsSink`,error:t}):this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getAction(e,t,n){return(await this.getActions({organizationId:e,projectSecureId:t,actionRunId:n})).actionLogs[0]}async getSteps(e){try{this.#r?.info({message:`Querying step logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_steps_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} step logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{stepLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getStep(e,t,n,r){return(await this.getSteps({organizationId:e,projectSecureId:t,actionRunId:n,stepIndex:r})).stepLogs[0]}#i(){this.#t&&(this.#t.on(this.#t.events.CONNECT,()=>{this.#r?.info({message:`Kafka producer connected successfully`,category:`LogsSink`})}),this.#t.on(this.#t.events.DISCONNECT,()=>{this.#r?.warning({message:`Kafka producer disconnected`,category:`LogsSink`})}),this.#t.on(this.#t.events.REQUEST_TIMEOUT,({payload:e})=>{this.#r?.error({message:`Kafka producer request timeout`,category:`LogsSink`,code:`KafkaProducerRequestTimeout`,context:{payload:e}})}))}async send(e,t){if(!this.#t)throw this.#r?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=Date.now(),r=safeSerialize(t);this.#r?.debug({message:`Sending to topic ${e}: ${r}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:r}],acks:1,compression:l.GZIP});this.#r?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`,context:{durationMs:Date.now()-n,topic:e}})}catch(t){throw this.#r?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`,context:{durationMs:Date.now()-n,topic:e}}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,riskLevel:t.defenderContext?.riskLevel,tier2Score:t.defenderContext?.tier2Score,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const g={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1,includeBackground:!1},defender:{enabled:!0}};function resolveOlapOptions(e){return{logs:{...g.logs,...e?.logs},advanced:{...g.advanced,...e?.advanced},defender:{...g.defender,...e?.defender}}}var OlapClient=class{#e;#t;#n;#r;#i;#a;#o;constructor({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={}){this.name=`OlapClient`,this.#e=e(r,o),this.#t=t(o),this.#n=n(i,o),this.#r=o,this.#i=new LogsSink(this.#e,this.#t,a,this.#r),this.#a=new AdvancedSink(this.#n,this.#r),this.#o=new DefenderSink(this.#n,this.#r)}async initialize(){await this.#i?.initialize(),await this.#a?.initialize(),await this.#o?.initialize()}get tinybirdClient(){return this.#i?.tinybirdClient}async recordAction(e,t,n){let{logs:r,advanced:i,defender:a}=resolveOlapOptions(n);if(this.#r?.debug({message:`Olap client called to record action`,category:`OlapClient`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,options:n,resolvedOptions:{logs:r,advanced:i,defender:a}}}),r?.enabled)try{await this.#i?.sendAction(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendAction(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to advanced logs s3: ${e.message}`,category:`OlapClient`,error:e})}if(a?.enabled)try{await this.#o?.sendAction(e,t,a)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to defender logs s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=resolveOlapOptions(n);if(r?.enabled)try{await this.#i?.sendStep(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendStep(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async getActionLogs(e,t,n){return this.#r?.debug({message:`Querying action logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getActions({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query action logs, returning empty result`,category:`OlapClient`}),{actionLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getActionLog(e,n,r){let i=await this.#i?.getAction(e,n,r);if(t(i)){this.#r?.debug({message:`Failed to retrieve action log for actionRunId ${r}`,category:`OlapClient`});return}return i}async getStepLogs(e,t,n){return this.#r?.debug({message:`Querying step logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getSteps({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query step logs, returning empty result`,category:`OlapClient`}),{stepLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getStepLog(e,n,r,i){let a=await this.#i?.getStep(e,n,r,i);if(t(a)){this.#r?.debug({message:`Failed to retrieve step log for actionRunId ${r} stepIndex ${i}`,category:`OlapClient`});return}return a}async getActionAdvancedLog(e,n,r){let i=await this.#a?.getAction(e,n,r);if(t(i)){this.#r?.debug({message:`Failed to retrieve action advanced log for actionRunId ${r}`,category:`OlapClient`});return}return i}async getStepAdvancedLog(e,n,r,i){let a=await this.#a?.getStep(e,n,r,i);if(t(a)){this.#r?.debug({message:`Failed to retrieve step advanced log for actionRunId ${r} stepIndex ${i}`,category:`OlapClient`});return}return a}async getDefenderLog(e,n,r){let i=await this.#o?.getAction(e,n,r);if(t(i)){this.#r?.debug({message:`Failed to retrieve defender log for actionRunId ${r}`,category:`OlapClient`});return}return i}};const buildOlapClientInstance=async({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={})=>{let s=new OlapClient({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o});return await s.initialize(),s};var OlapClientManager=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o,getOlapClient:s=buildOlapClientInstance}={}){return this.olapClientPromise||=s({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{OlapClient,OlapClientManager};
+import{t as e}from"./chunk-Cfxk5zVN.mjs";import{exceedsSize as t,isMissing as n,notMissing as r}from"@stackone/utils";import{CensorType as i,redactObject as a}from"@stackone/redaction";import{GetObjectCommand as o,PutObjectCommand as s,S3Client as c}from"@aws-sdk/client-s3";import{HttpClient as l}from"@stackone/transport";import{CompressionTypes as u,Kafka as d,Partitioners as f}from"kafkajs";import p from"node:http";const m=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],h=[`refresh_authentication`],g=10*1024*1024;var AdvancedSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async sendAction(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send action`,category:`AdvancedSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending action to advanced sink`,category:`AdvancedSink`});return}if(n.errorsOnly&&t.success){this.#t?.debug({message:`Advanced sink errorsOnly is enabled, skipping successful action for ${e.mode||`action`}`,category:`AdvancedSink`});return}if(this.isBackgroundLog(e)&&!n.includeBackground){this.#t?.debug({message:`Advanced sink is configured to exclude background logs, skipping action with mode ${e.mode}`,category:`AdvancedSink`});return}let r=this.createActionS3(e,t,n?.ttl);await this.send(r)}async sendStep(e,t,n){if(this.#t?.debug({message:`Advanced sink called to send step`,category:`AdvancedSink`,context:{actionRunId:e.actionRunId,success:t.success,options:{enabled:n?.enabled,errorsOnly:n?.errorsOnly,includeBackground:n?.includeBackground,ttl:n?.ttl}}}),!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending step to advanced sink`,category:`AdvancedSink`});return}let r=this.createStepS3(e,t,n?.ttl);await this.send(r)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from advanced sink`,category:`AdvancedSink`});return}let r=process.env.ADVANCED_LOGS_BUCKET;if(!r)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let i=new o({Bucket:r,Key:`${e}/${t}/${n}/${n}.json`}),a=await this.#e.send(i);if(!a.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`AdvancedSink`});return}let s=await a.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved advanced log from S3 for actionRunId ${n}`,category:`AdvancedSink`}),JSON.parse(s)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Advanced log not found in S3 for actionRunId ${n}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting advanced log from S3 for actionRunId ${n}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async getStep(e,t,n,r){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get step log from advanced sink`,category:`AdvancedSink`});return}let i=process.env.ADVANCED_LOGS_BUCKET;if(!i)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);try{let a=new o({Bucket:i,Key:`${e}/${t}/${n}/steps/${r}.json`}),s=await this.#e.send(a);if(!s.Body){this.#t?.warning({message:`Received empty body when trying to get step log from S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`});return}let c=await s.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved step log from S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`}),JSON.parse(c)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Step log not found in S3 for actionRunId ${n} stepIndex ${r}`,category:`AdvancedSink`});return}throw this.#t?.error({message:`Error when getting step log from S3 for actionRunId ${n} stepIndex ${r}`,error:e,code:`AdvancedLogReadError`,category:`AdvancedSink`}),e}}async send(e,t){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}let n=process.env.ADVANCED_LOGS_BUCKET;if(!n)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let r=`ttl=30d`;typeof t==`number`&&(t===1?r=`ttl=1d`:t===7&&(r=`ttl=7d`));let i=Math.floor(Date.now()/1e3)+(t??30)*24*60*60;this.#t?.debug({message:`Storing advanced log in S3 bucket ${n} with ttl of ${r}`,category:`AdvancedSink`,context:{bucket:n,key:e.Key,ttlTag:r,expiresAt:new Date(i*1e3).toISOString()}});try{await this.#e.send(new s({Bucket:n,Key:e.Key,Body:e.Body,ContentType:e.ContentType,Expires:new Date(i*1e3),Tagging:r}))}catch(t){throw this.#t?.error({message:`Failed to write advanced logs to S3 at ${e.Key}`,error:t,code:`AdvancedLogWriteError`,category:`AdvancedSink`}),t}}createActionS3(e,t,n){let r=`${t.organizationId}/${t.projectSecureId}/${t.actionRunId}/${t.actionRunId}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeActionResult(e,t,i,g),ContentType:`application/json`,Expires:new Date(i*1e3)}}createStepS3(e,t,n){let r=`${e.organizationId}/${e.projectSecureId}/${e.actionRunId}/steps/${e.stepIndex}.json`,i=Math.floor(Date.now()/1e3)+(n??30)*24*60*60;return{Key:r,Body:this.serializeStepResult(e,t,i,g),ContentType:`application/json`,Expires:new Date(i*1e3)}}serializeActionResult(e,t,n,r){let o=a({req:{...e.headers&&{headers:{...e.headers}}},res:{...t.headers&&{headers:{...t.headers}}}},i.PARTIAL),s=o.req?.headers,c=o.res?.headers,l=a(e.body,i.PARTIAL),u=a(t.body,i.PARTIAL),d=this.isBackgroundLog(e),f={data:{request:{id:t.actionRunId,actionId:t.actionId,method:t.httpMethod,headers:this.filterHeaders(s),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:l},response:{statusCode:t.statusCode??500,headers:this.filterHeaders(c),body:u},...d?{isBackgroundLog:!0}:{}},metadata:{...d?{isBackgroundLog:!0}:{},expirationTime:n}};return this.serializeAndLimit(f,r)}serializeStepResult(e,t,n,r){let i={data:{id:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,input:e.inputs,outputs:t.outputs,errors:t.errors},metadata:{expirationTime:n}};return this.serializeAndLimit(i,r)}serializeAndLimit(e,n){if(t(e,n)){let t={...e,data:{outputs:{error:`Error.TOO_LARGE`}}};return JSON.stringify(t)}return JSON.stringify(e)}filterHeaders(e){if(!e)return;let t={},n=m.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e.mode===`data_sync`}isBackgroundLog(e){return r(e.mode)&&h.includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const buildS3Client=(e,t)=>{try{return new c(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}};var DefenderSink=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, defender sink will not function`,category:`DefenderSink`});return}}async sendAction(e,t,r){if(this.#t?.debug({message:`Defender sink called to send action`,category:`DefenderSink`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,statusCode:t.statusCode,options:{enabled:r?.enabled}}}),n(t.defenderContext)){this.#t?.debug({message:`No defender context in action result, skipping sending action to defender sink`,category:`DefenderSink`,context:{actionRunId:t.actionRunId}});return}if(!r?.enabled){this.#t?.debug({message:`Defender sink is disabled, skipping sending action to defender sink`,category:`DefenderSink`});return}let i=this.createActionS3(t);await this.send(i)}async getAction(e,t,n){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot get action log from defender sink`,category:`DefenderSink`});return}let r=process.env.DEFENDER_LOGS_BUCKET;if(!r)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);try{let i=new o({Bucket:r,Key:this.generateS3Key(e,t,n)}),a=await this.#e.send(i);if(!a.Body){this.#t?.warning({message:`Received empty body when trying to get action log from S3 for actionRunId ${n}`,category:`DefenderSink`});return}let s=await a.Body.transformToString();return this.#t?.debug({message:`Successfully retrieved defender log from S3 for actionRunId ${n}`,category:`DefenderSink`}),JSON.parse(s)}catch(e){let t=e;if(t.name===`NoSuchKey`||t.$metadata?.httpStatusCode===404){this.#t?.debug({message:`Defender log not found in S3 for actionRunId ${n}`,category:`DefenderSink`});return}throw this.#t?.error({message:`Error when getting defender log from S3 for actionRunId ${n}`,error:e,code:`DefenderLogReadError`,category:`DefenderSink`}),e}}async send(e){if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to defender sink`,category:`DefenderSink`});return}let t=process.env.DEFENDER_LOGS_BUCKET;if(!t)throw Error(`DEFENDER_LOGS_BUCKET environment variable is not set`);this.#t?.debug({message:`Storing defender log in S3 bucket ${t}`,category:`DefenderSink`,context:{bucket:t,key:e.Key}});try{await this.#e.send(new s({Bucket:t,Key:e.Key,Body:e.Body,ContentType:e.ContentType}))}catch(t){throw this.#t?.error({message:`Failed to write defender logs to S3 at ${e.Key}`,error:t,code:`DefenderLogWriteError`,category:`DefenderSink`}),t}}generateS3Key(e,t,n){return`${e}/${t}/${n}/defender.json`}createActionS3(e){let t=e.organizationId,n=e.projectSecureId;return{Key:this.generateS3Key(t,n,e.actionRunId),Body:JSON.stringify(e.defenderContext),ContentType:`application/json`}}};const buildHttpClient=e=>{try{return new l({logger:e})}catch(t){let n=t;e?.error({message:`Error building http client: ${n.message}`,error:n,code:`BuildHttpClientError`,category:`buildHttpClient`});return}},buildKafkaClient=(e,t)=>{try{return new d(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}},safeSerialize=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var SchemaValidationError=class extends Error{constructor(e,t){super(e),this.name=`SchemaValidationError`,this.cause=t}},TinybirdClient=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n}async query(e){try{let t=await this.#o(e.endpoint,e.params);return this.#a(t.data,e.schema)}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryWithPagination(e){try{let t=await this.#o(e.endpoint,e.params),n=this.#a(t.data,e.schema);return{data:n,total:t.rows_before_limit_at_least??t.rows??n.length,pageNumber:e.pageNumber??1,pageSize:e.pageSize??n.length}}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}async queryRaw(e){try{let t=await this.#o(e.endpoint,e.params);return t.data=this.#a(t.data,e.schema),t}catch(t){if(t instanceof SchemaValidationError)throw t;let n=t instanceof Error?t.message:String(t);throw this.#n?.error({message:`Error querying Tinybird endpoint ${e.endpoint}: ${n}`,category:`TinybirdClient`,code:`TinybirdQueryError`}),t}}isReady(){return!!(this.#e&&this.#t?.token)}#r(){return new p.Agent({})}#i(){if(!this.#e)throw this.#n?.error({message:`HTTP client not initialized, cannot perform Tinybird query`,category:`TinybirdClient`,code:`HttpClientNotReady`}),Error(`HTTP client is not initialized`);if(!this.#t?.token)throw this.#n?.warning({message:`Missing OLAP token, cannot perform Tinybird query`,category:`TinybirdClient`}),Error(`TinybirdClient - Missing token`)}#a(e,t){if(!t)return e;try{let n=e.map(e=>t.parse(e));return this.#n?.debug({message:`Schema validation passed for ${n.length} rows`,category:`TinybirdClient`}),n}catch(e){let t=e instanceof Error?e.message:String(e);throw this.#n?.error({message:`Schema validation failed: ${t}`,category:`TinybirdClient`,code:`SchemaValidationError`,error:e}),new SchemaValidationError(t,e)}}async#o(e,t){this.#i();let n=this.#e,r=this.#t;if(!n||!r)throw Error(`Client validation failed`);let i=new URL(`/v0/pipes/${e}`,r.baseUrl);this.#n?.debug({message:`Querying Tinybird endpoint: ${e}`,category:`TinybirdClient`,context:{endpoint:e,params:t}});let a=await n.request({method:`post`,url:i.toString(),payload:t,headers:{"Content-Type":`application/json`,Authorization:`Bearer ${r.token}`},...r.allowHttp&&{httpAgent:this.#r()}});return this.#n?.debug({message:`Tinybird query returned ${a.data.data.length} rows`,category:`TinybirdClient`,context:{endpoint:e,rows:a.data.rows,statistics:a.data.statistics}}),a.data}},LogsSink=class{#e;#t;#n;#r;constructor(e,t,n,r){this.#e=e,this.#r=r,this.#n=new TinybirdClient(t,n,r)}get tinybirdClient(){return this.#n}async initialize(){if(!this.#e){this.#r?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(this.#n?.isReady()||this.#r?.warning({message:`Tinybird client is not ready (missing HTTP client or Tinybird config/token), logs sink will not be able to query`,category:`LogsSink`}),!this.#t){try{this.#t=this.#e.producer({createPartitioner:f.DefaultPartitioner,maxInFlightRequests:2,idempotent:!1})}catch(e){this.#r?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#i();try{await this.#t.connect()}catch(e){this.#r?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#r?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async sendAction(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending action to log sink`,category:`KafkaSink`});return}let r=this.buildActionLog(e,t);await this.send(`actions`,r)}async sendStep(e,t,n){if(!n?.enabled){this.#r?.debug({message:`Logs sink is disabled, skipping sending step to log sink`,category:`KafkaSink`});return}let r=this.buildStepLog(e,t);await this.send(`steps`,r)}async getActions(e){try{this.#r?.info({message:`Querying action logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_actions_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} action logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{actionLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#n?.isReady()?this.#r?.warning({message:`Failed to query Tinybird for action logs, returning empty results`,category:`LogsSink`,error:t}):this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{actionLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getAction(e,t,n){return(await this.getActions({organizationId:e,projectSecureId:t,actionRunId:n})).actionLogs[0]}async getSteps(e){try{this.#r?.info({message:`Querying step logs from Tinybird`,category:`LogsSink`,context:{query:e}});let t=await this.#n?.queryWithPagination({endpoint:`project_steps_logs.json`,params:e,pageNumber:e.pageNumber,pageSize:e.pageSize});return t?(this.#r?.info({message:`Successfully retrieved ${t.data.length} step logs from Tinybird (${t.total} total)`,category:`LogsSink`}),{stepLogs:t.data,total:t.total,pageNumber:t.pageNumber,pageSize:t.pageSize}):(this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0})}catch(t){return this.#r?.warning({message:`Tinybird client not available, returning empty results`,category:`LogsSink`,error:t}),{stepLogs:[],total:0,pageNumber:e.pageNumber??1,pageSize:e.pageSize??0}}}async getStep(e,t,n,r){return(await this.getSteps({organizationId:e,projectSecureId:t,actionRunId:n,stepIndex:r})).stepLogs[0]}#i(){this.#t&&(this.#t.on(this.#t.events.CONNECT,()=>{this.#r?.info({message:`Kafka producer connected successfully`,category:`LogsSink`})}),this.#t.on(this.#t.events.DISCONNECT,()=>{this.#r?.warning({message:`Kafka producer disconnected`,category:`LogsSink`})}),this.#t.on(this.#t.events.REQUEST_TIMEOUT,({payload:e})=>{this.#r?.error({message:`Kafka producer request timeout`,category:`LogsSink`,code:`KafkaProducerRequestTimeout`,context:{payload:e}})}))}async send(e,t){if(!this.#t)throw this.#r?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let n=Date.now(),r=safeSerialize(t);this.#r?.debug({message:`Sending to topic ${e}: ${r}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:r}],acks:1,compression:u.GZIP});this.#r?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`,context:{durationMs:Date.now()-n,topic:e}})}catch(t){throw this.#r?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`,context:{durationMs:Date.now()-n,topic:e}}),t}}buildActionLog(e,t){return Object.fromEntries(Object.entries({actionRunId:t.actionRunId,actionId:t.actionId,organizationId:String(t.organizationId),projectSecureId:t.projectSecureId,accountSecureId:t.accountSecureId,mode:e.mode,connectorKey:t.connectorKey,connectorVersion:t.connectorVersion,actionType:t.actionType,category:t.category,originOwnerId:t.originOwnerId,originOwnerName:t.originOwnerName,httpMethod:t.httpMethod,url:t.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,success:t.success,statusCode:t.statusCode,riskLevel:t.defenderContext?.riskLevel,tier2Score:t.defenderContext?.tier2Score,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}buildStepLog(e,t){return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepIndex:e.stepIndex,stepId:e.stepId,organizationId:String(e.organizationId),projectSecureId:String(e.projectSecureId),accountSecureId:String(e.accountSecureId),skipped:t.skipped,success:t.success,message:t.message,startTime:t.startTime,endTime:t.endTime,durationMs:this.calculateDuration(t.startTime,t.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0))}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}};const _={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1,includeBackground:!1},defender:{enabled:!0}};function resolveOlapOptions(e){return{logs:{..._.logs,...e?.logs},advanced:{..._.advanced,...e?.advanced},defender:{..._.defender,...e?.defender}}}var OlapClient=class{#e;#t;#n;#r;#i;#a;#o;constructor({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={}){this.name=`OlapClient`,this.#e=e(r,o),this.#t=t(o),this.#n=n(i,o),this.#r=o,this.#i=new LogsSink(this.#e,this.#t,a,this.#r),this.#a=new AdvancedSink(this.#n,this.#r),this.#o=new DefenderSink(this.#n,this.#r)}async initialize(){await this.#i?.initialize(),await this.#a?.initialize(),await this.#o?.initialize()}get tinybirdClient(){return this.#i?.tinybirdClient}async recordAction(e,t,n){let{logs:r,advanced:i,defender:a}=resolveOlapOptions(n);if(this.#r?.debug({message:`Olap client called to record action`,category:`OlapClient`,context:{mode:e.mode,actionId:e.actionId,actionRunId:t.actionRunId,success:t.success,options:n,resolvedOptions:{logs:r,advanced:i,defender:a}}}),r?.enabled)try{await this.#i?.sendAction(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendAction(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to advanced logs s3: ${e.message}`,category:`OlapClient`,error:e})}if(a?.enabled)try{await this.#o?.sendAction(e,t,a)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to defender logs s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=resolveOlapOptions(n);if(r?.enabled)try{await this.#i?.sendStep(e,t,r)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#a?.sendStep(e,t,i)}catch(e){this.#r?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async getActionLogs(e,t,n){return this.#r?.debug({message:`Querying action logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getActions({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query action logs, returning empty result`,category:`OlapClient`}),{actionLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getActionLog(e,t,r){let i=await this.#i?.getAction(e,t,r);if(n(i)){this.#r?.debug({message:`Failed to retrieve action log for actionRunId ${r}`,category:`OlapClient`});return}return i}async getStepLogs(e,t,n){return this.#r?.debug({message:`Querying step logs`,category:`OlapClient`,context:{query:n}}),await this.#i?.getSteps({...n,organizationId:e,projectSecureId:t})||(this.#r?.warning({message:`Failed to query step logs, returning empty result`,category:`OlapClient`}),{stepLogs:[],total:0,pageNumber:n.pageNumber??1,pageSize:n.pageSize??0})}async getStepLog(e,t,r,i){let a=await this.#i?.getStep(e,t,r,i);if(n(a)){this.#r?.debug({message:`Failed to retrieve step log for actionRunId ${r} stepIndex ${i}`,category:`OlapClient`});return}return a}async getActionAdvancedLog(e,t,r){let i=await this.#a?.getAction(e,t,r);if(n(i)){this.#r?.debug({message:`Failed to retrieve action advanced log for actionRunId ${r}`,category:`OlapClient`});return}return i}async getStepAdvancedLog(e,t,r,i){let a=await this.#a?.getStep(e,t,r,i);if(n(a)){this.#r?.debug({message:`Failed to retrieve step advanced log for actionRunId ${r} stepIndex ${i}`,category:`OlapClient`});return}return a}async getDefenderLog(e,t,r){let i=await this.#o?.getAction(e,t,r);if(n(i)){this.#r?.debug({message:`Failed to retrieve defender log for actionRunId ${r}`,category:`OlapClient`});return}return i}};const buildOlapClientInstance=async({getKafkaClient:e=buildKafkaClient,getHttpClient:t=buildHttpClient,getS3Client:n=buildS3Client,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}={})=>{let s=new OlapClient({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o});return await s.initialize(),s};var OlapClientManager=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o,getOlapClient:s=buildOlapClientInstance}={}){return this.olapClientPromise||=s({getKafkaClient:e,getHttpClient:t,getS3Client:n,kafkaClientConfig:r,s3ClientConfig:i,tinybirdConfig:a,logger:o}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{OlapClient,OlapClientManager};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stackone/olap",
-    "version": "1.21.0",
+    "version": "1.22.0",
     "description": "",
     "main": "dist/index.cjs",
     "module": "dist/index.mjs",