@stackone/olap 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +75 -19
- package/dist/index.d.mts +75 -19
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
let e=require(`@aws-sdk/client-s3`),t=require(`kafkajs`);const n=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var r=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,t,r){if(!r?.enabled){this.#t?.debug({message:`Logs sink is disabled, skipping send to bucket ${e}`,category:`AdvancedSink`});return}let i=n(t);this.#t?.info({message:`[S3] Sent to bucket ${e}: ${i}`,context:{bucket:e,payload:t},category:`S3Sink`}),await Promise.resolve()}};const i=(t,n)=>{try{return new e.S3Client(t)??void 0}catch(e){let t=e;n?.error({message:`Error building s3 client: ${t.message}`,error:t,code:`BuildS3ClientError`,category:`buildS3Client`});return}},a=(e,n)=>{try{return new t.Kafka(e)??void 0}catch(e){let t=e;n?.error({message:`Error building kafka client: ${t.message}`,error:t,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}};var o=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:t.Partitioners.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,r){if(!r?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping send to topic ${e}`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let i=n(this.buildLog(t));this.#n?.debug({message:`Sending to topic ${e}: ${i}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:i}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e){if(this.isActionResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,actionId:e.actionId,organizationId:e.organizationId,projectSecureId:e.projectSecureId,accountSecureId:e.accountSecureId,mode:e.mode,connectorKey:e.connectorKey,connectorVersion:e.connectorVersion,actionType:e.actionType,category:e.category,originOwnerId:e.originOwnerId,originOwnerName:e.originOwnerName,httpMethod:e.httpMethod,url:e.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,status:e.status,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepId:e.stepId,skipped:e.skipped,status:e.status,message:e.message,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const s={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function c(e){return{...s,...e}}var l=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=a,getS3Client:t=i,kafkaClientConfig:n,s3ClientConfig:s,logger:c}={}){this.name=`OlapClient`,this.#e=e(n,c),this.#t=t(s,c),this.#n=c,this.#r=new o(this.#e,this.#n),this.#i=new r(this.#t,this.#n)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t){let{logs:n,advanced:r}=c(t);if(n?.enabled)try{await this.#r?.send(`actions`,e,n)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(r?.enabled)try{await this.#i?.send(`actions`,e,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t){let{logs:n,advanced:r}=c(t);if(n?.enabled)try{await this.#r?.send(`steps`,e,n)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(r?.enabled)try{await this.#i?.send(`steps`,e,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const u=async({getKafkaClient:e=a,getS3Client:t=i,kafkaClientConfig:n,s3ClientConfig:r,logger:o}={})=>{let s=new l({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:o});return await s.initialize(),s};var d=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=u}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=l,exports.OlapClientManager=d;
|
|
1
|
+
let e=require(`@stackone/utils`),t=require(`@aws-sdk/client-s3`),n=require(`date-fns`),r=require(`kafkajs`);const i=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}},a=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],o=[`application/octet-stream`,`application/pdf`,`application/msword`,`application/vnd.openxmlformats-officedocument.wordprocessingml.document`,`application/vnd.ms-excel`,`application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`,`application/zip`,`application/x-zip-compressed`],s=[`image/`,`audio/`,`video/`,`application/vnd.`,`binary`],c=10*1024*1024;var l=class{#e;constructor({logger:e}={}){this.#e=e}async write(e,t,n,r){try{let i=r?.ttl??30,a=await this.generateS3Command(t,n,i);await e.send(a)}catch(e){throw this.#e?.error({message:`Failed to write advanced logs to S3`,category:`AdvancedWriter`,code:`AdvancedWriteError`,error:e}),e}}generateS3Key(e,t,n,r){let i=`${t}/${n}/${e}`;return r?`${i}/steps/${r}.json`:`${i}/${e}.json`}async generateS3Command(e,r,i){let a=process.env.ADVANCED_LOGS_BUCKET;if(!a)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let o=this.generateS3Key(r.actionRunId,r.organizationId,r.projectSecureId),s=(0,n.getUnixTime)((0,n.addDays)(new Date,i)),c=this.serialize(e,s,10485760),l=`ttl=30d`;return i===1?l=`ttl=1d`:i===7&&(l=`ttl=7d`),new t.PutObjectCommand({Bucket:a,Key:o,Body:c,ContentType:`application/json`,Expires:new Date(s*1e3),Tagging:l})}filterHeaders(e){if(!e)return;let t={},n=a.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(t){return(0,e.notMissing)(t.mode)&&[`refresh_authentication`].includes(t.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}serialize(t,n,r){let i=t.response?.body;((0,e.isString)(i)&&i.length>r||Buffer.isBuffer(i)&&i.byteLength>r||i&&(0,e.isObject)(i)&&!Array.isArray(i)&&Object.values(i).some(t=>(0,e.isString)(t)&&t.length>r||Buffer.isBuffer(t)&&t.byteLength>r))&&(t={...t,response:{...t.response,body:{error:`Error.TOO_LARGE`}}});let a=JSON.stringify({data:t,expirationTime:n});if(Buffer.byteLength(a,`utf8`)>r){let e={...t,response:{...t.response,body:{error:`Error.TOO_LARGE`}}};a=JSON.stringify({data:e,expirationTime:n})}return a}processContent(e,t,n){let r=this.getContentType(e);if(!r)return i(t);let a=r.toLowerCase().split(`;`)[0].trim();return o.includes(a)||s.some(e=>a.startsWith(e))?{type:`binary`,action:n,contentType:r}:i(t)}};const u=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],d=10*1024*1024;var f=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n??new l({logger:t})}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,n,r){if(!r?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending to advanced sink`,category:`AdvancedSink`});return}if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}try{let i=await this.createS3Item(e,n,r.ttl);await this.#e.send(new t.PutObjectCommand({Bucket:i.Bucket,Key:i.Key,Body:i.Body,ContentType:i.ContentType,Expires:i.Expires,Tagging:i.Tagging}))}catch(e){throw this.#t?.error({message:`Failed to write advanced logs to S3 account ${n.accountSecureId}`,error:e,code:`AdvancedLogWriteError`,category:`AdvancedSink`,context:{actionRunId:n.actionRunId}}),e}}async createS3Item(e,t,n){let{projectSecureId:r,organizationId:i}=t,a=`ttl=30d`;typeof n==`number`&&(n===1?a=`ttl=1d`:n===7&&(a=`ttl=7d`));let o=Math.floor(Date.now()/1e3)+(n??30)*24*60*60,s=process.env.ADVANCED_LOGS_BUCKET;if(!s)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);return{Bucket:s,Key:this.generateS3Key(t,i,r),Body:this.serialize(e,t,o,10485760),ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:a}}generateS3Key(e,t,n){return`actionId`in e?`${t}/${n}/${e.actionRunId}/${e.actionRunId}.json`:`${t}/${n}/${e.actionRunId}/steps/${e.stepId}.json`}serialize(e,t,n,r){return`actionId`in t?JSON.stringify(this.serializeActionResult(e,t,n,r)):JSON.stringify(this.serializeStepResult(e,t))}serializeActionResult(e,t,n,r){let i=e.headers,a=t.headers,o=e.body,s=t.body,c=this.isBackgroundLog(t);return{data:{request:{id:t.actionRunId,actionId:t.actionId,method:t.httpMethod,headers:this.filterHeaders(i),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:o},response:{statusCode:t.status?parseInt(t.status,10):void 0,headers:this.filterHeaders(a),body:s},...c?{isBackgroundLog:!0}:{}},metadata:{...c?{isBackgroundLog:!0}:{},expirationTime:n},expirationTime:n}}serializeStepResult(e,t){return{}}filterHeaders(e){if(!e)return;let t={},n=u.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(t){return(0,e.notMissing)(t.mode)&&[`refresh_authentication`].includes(t.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const p=(e,n)=>{try{return new t.S3Client(e)??void 0}catch(e){let t=e;n?.error({message:`Error building s3 client: ${t.message}`,error:t,code:`BuildS3ClientError`,category:`buildS3Client`});return}},m=(e,t)=>{try{return new r.Kafka(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}};var h=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:r.Partitioners.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping send to topic ${e}`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let r=i(this.buildLog(t));this.#n?.debug({message:`Sending to topic ${e}: ${r}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:r}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e){if(this.isActionResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,actionId:e.actionId,organizationId:e.organizationId,projectSecureId:e.projectSecureId,accountSecureId:e.accountSecureId,mode:e.mode,connectorKey:e.connectorKey,connectorVersion:e.connectorVersion,actionType:e.actionType,category:e.category,originOwnerId:e.originOwnerId,originOwnerName:e.originOwnerName,httpMethod:e.httpMethod,url:e.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,status:e.status,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepId:e.stepId,skipped:e.skipped,status:e.status,message:e.message,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const g={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function _(e){return{...g,...e}}var v=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=m,getS3Client:t=p,kafkaClientConfig:n,s3ClientConfig:r,logger:i,advancedWriter:a}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new h(this.#e,this.#n),this.#i=new f(this.#t,this.#n,a)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=_(n);if(r?.enabled)try{await this.#r?.send(`actions`,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=_(n);if(r?.enabled)try{await this.#r?.send(`steps`,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const y=async({getKafkaClient:e=m,getS3Client:t=p,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new v({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var b=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=y}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};exports.OlapClient=v,exports.OlapClientManager=b;
|
package/dist/index.d.cts
CHANGED
|
@@ -64,18 +64,6 @@ type LogError = Error & {
|
|
|
64
64
|
url?: string;
|
|
65
65
|
};
|
|
66
66
|
//#endregion
|
|
67
|
-
//#region src/advanced/types.d.ts
|
|
68
|
-
interface S3ClientConfig {
|
|
69
|
-
region?: string;
|
|
70
|
-
}
|
|
71
|
-
type S3ClientBuilder = (config?: S3ClientConfig, logger?: ILogger) => S3Client | undefined;
|
|
72
|
-
//#endregion
|
|
73
|
-
//#region src/logs/types.d.ts
|
|
74
|
-
interface KafkaClientConfig {
|
|
75
|
-
brokers: string[];
|
|
76
|
-
}
|
|
77
|
-
type KafkaClientBuilder = (config?: KafkaClientConfig, logger?: ILogger) => Kafka | undefined;
|
|
78
|
-
//#endregion
|
|
79
67
|
//#region src/olap/olapOptions.d.ts
|
|
80
68
|
type OlapOptions = {
|
|
81
69
|
logs?: LogsOptions;
|
|
@@ -92,9 +80,17 @@ type AdvancedOptions = {
|
|
|
92
80
|
//#endregion
|
|
93
81
|
//#region src/types.d.ts
|
|
94
82
|
interface IOlapClient {
|
|
95
|
-
recordAction(actionResult: ActionResult, options?: OlapOptions): void;
|
|
96
|
-
recordStep(stepResult: StepResult, options?: OlapOptions): void;
|
|
83
|
+
recordAction(actionInput: ActionInput, actionResult: ActionResult, options?: OlapOptions): void;
|
|
84
|
+
recordStep(stepInput: StepInput, stepResult: StepResult, options?: OlapOptions): void;
|
|
97
85
|
}
|
|
86
|
+
type ActionInput = {
|
|
87
|
+
actionId?: string;
|
|
88
|
+
url?: string;
|
|
89
|
+
pathParams?: unknown;
|
|
90
|
+
queryParams?: unknown;
|
|
91
|
+
body?: unknown;
|
|
92
|
+
headers?: Record<string, string>;
|
|
93
|
+
};
|
|
98
94
|
type ActionResult = {
|
|
99
95
|
actionRunId: string;
|
|
100
96
|
actionId: string;
|
|
@@ -119,13 +115,20 @@ type ActionResult = {
|
|
|
119
115
|
childResource?: string;
|
|
120
116
|
status?: string;
|
|
121
117
|
message?: string;
|
|
122
|
-
|
|
118
|
+
headers?: Record<string, string>;
|
|
119
|
+
body?: unknown;
|
|
123
120
|
startTime?: Date;
|
|
124
121
|
endTime?: Date;
|
|
125
122
|
};
|
|
123
|
+
type StepInput = {
|
|
124
|
+
inputs?: unknown;
|
|
125
|
+
};
|
|
126
126
|
type StepResult = {
|
|
127
127
|
actionRunId: string;
|
|
128
128
|
stepId: string;
|
|
129
|
+
organizationId: string;
|
|
130
|
+
projectSecureId: string;
|
|
131
|
+
accountSecureId: string;
|
|
129
132
|
status?: string;
|
|
130
133
|
message?: string;
|
|
131
134
|
outputs?: unknown;
|
|
@@ -135,6 +138,57 @@ type StepResult = {
|
|
|
135
138
|
endTime?: Date;
|
|
136
139
|
};
|
|
137
140
|
//#endregion
|
|
141
|
+
//#region src/advanced/types.d.ts
|
|
142
|
+
interface S3ClientConfig {
|
|
143
|
+
region?: string;
|
|
144
|
+
}
|
|
145
|
+
type S3ClientBuilder = (config?: S3ClientConfig, logger?: ILogger) => S3Client | undefined;
|
|
146
|
+
interface AdvancedLog {
|
|
147
|
+
request: {
|
|
148
|
+
id?: string;
|
|
149
|
+
method?: string;
|
|
150
|
+
headers?: Record<string, unknown>;
|
|
151
|
+
url: {
|
|
152
|
+
url: string;
|
|
153
|
+
hostname: string;
|
|
154
|
+
path: string;
|
|
155
|
+
queryParams: Record<string, string>;
|
|
156
|
+
};
|
|
157
|
+
body?: string | Record<string, unknown>;
|
|
158
|
+
};
|
|
159
|
+
response: {
|
|
160
|
+
statusCode?: number;
|
|
161
|
+
headers?: Record<string, unknown>;
|
|
162
|
+
body?: unknown;
|
|
163
|
+
};
|
|
164
|
+
isBackgroundLog?: boolean;
|
|
165
|
+
}
|
|
166
|
+
//#endregion
|
|
167
|
+
//#region src/advanced/advancedWriter.d.ts
|
|
168
|
+
declare class AdvancedWriter {
|
|
169
|
+
#private;
|
|
170
|
+
constructor({
|
|
171
|
+
logger
|
|
172
|
+
}?: {
|
|
173
|
+
logger?: ILogger;
|
|
174
|
+
});
|
|
175
|
+
write(s3Client: S3Client, logData: AdvancedLog, result: ActionResult, options: AdvancedOptions): Promise<void>;
|
|
176
|
+
private generateS3Key;
|
|
177
|
+
private generateS3Command;
|
|
178
|
+
private filterHeaders;
|
|
179
|
+
private isDataSyncRequest;
|
|
180
|
+
private isBackgroundLog;
|
|
181
|
+
private getContentType;
|
|
182
|
+
private serialize;
|
|
183
|
+
private processContent;
|
|
184
|
+
}
|
|
185
|
+
//#endregion
|
|
186
|
+
//#region src/logs/types.d.ts
|
|
187
|
+
interface KafkaClientConfig {
|
|
188
|
+
brokers: string[];
|
|
189
|
+
}
|
|
190
|
+
type KafkaClientBuilder = (config?: KafkaClientConfig, logger?: ILogger) => Kafka | undefined;
|
|
191
|
+
//#endregion
|
|
138
192
|
//#region src/olap/olapClient.d.ts
|
|
139
193
|
declare class OlapClient implements IOlapClient {
|
|
140
194
|
#private;
|
|
@@ -144,17 +198,19 @@ declare class OlapClient implements IOlapClient {
|
|
|
144
198
|
getS3Client,
|
|
145
199
|
kafkaClientConfig,
|
|
146
200
|
s3ClientConfig,
|
|
147
|
-
logger
|
|
201
|
+
logger,
|
|
202
|
+
advancedWriter
|
|
148
203
|
}?: {
|
|
149
204
|
getKafkaClient?: KafkaClientBuilder;
|
|
150
205
|
getS3Client?: S3ClientBuilder;
|
|
151
206
|
kafkaClientConfig?: KafkaClientConfig;
|
|
152
207
|
s3ClientConfig?: S3ClientConfig;
|
|
153
208
|
logger?: ILogger;
|
|
209
|
+
advancedWriter?: AdvancedWriter;
|
|
154
210
|
});
|
|
155
211
|
initialize(): Promise<void>;
|
|
156
|
-
recordAction(actionResult: ActionResult, options?: OlapOptions): Promise<void>;
|
|
157
|
-
recordStep(stepResult: StepResult, options?: OlapOptions): Promise<void>;
|
|
212
|
+
recordAction(actionInput: ActionInput, actionResult: ActionResult, options?: OlapOptions): Promise<void>;
|
|
213
|
+
recordStep(stepInput: StepInput, stepResult: StepResult, options?: OlapOptions): Promise<void>;
|
|
158
214
|
}
|
|
159
215
|
//#endregion
|
|
160
216
|
//#region src/olap/olapClientManager.d.ts
|
|
@@ -191,4 +247,4 @@ declare class OlapClientManager {
|
|
|
191
247
|
static resetInstance(): void;
|
|
192
248
|
}
|
|
193
249
|
//#endregion
|
|
194
|
-
export { type ActionResult, type IOlapClient, OlapClient, OlapClientManager, type OlapOptions, type StepResult };
|
|
250
|
+
export { type ActionInput, type ActionResult, type IOlapClient, OlapClient, OlapClientManager, type OlapOptions, type StepInput, type StepResult };
|
package/dist/index.d.mts
CHANGED
|
@@ -64,18 +64,6 @@ type LogError = Error & {
|
|
|
64
64
|
url?: string;
|
|
65
65
|
};
|
|
66
66
|
//#endregion
|
|
67
|
-
//#region src/advanced/types.d.ts
|
|
68
|
-
interface S3ClientConfig {
|
|
69
|
-
region?: string;
|
|
70
|
-
}
|
|
71
|
-
type S3ClientBuilder = (config?: S3ClientConfig, logger?: ILogger) => S3Client | undefined;
|
|
72
|
-
//#endregion
|
|
73
|
-
//#region src/logs/types.d.ts
|
|
74
|
-
interface KafkaClientConfig {
|
|
75
|
-
brokers: string[];
|
|
76
|
-
}
|
|
77
|
-
type KafkaClientBuilder = (config?: KafkaClientConfig, logger?: ILogger) => Kafka | undefined;
|
|
78
|
-
//#endregion
|
|
79
67
|
//#region src/olap/olapOptions.d.ts
|
|
80
68
|
type OlapOptions = {
|
|
81
69
|
logs?: LogsOptions;
|
|
@@ -92,9 +80,17 @@ type AdvancedOptions = {
|
|
|
92
80
|
//#endregion
|
|
93
81
|
//#region src/types.d.ts
|
|
94
82
|
interface IOlapClient {
|
|
95
|
-
recordAction(actionResult: ActionResult, options?: OlapOptions): void;
|
|
96
|
-
recordStep(stepResult: StepResult, options?: OlapOptions): void;
|
|
83
|
+
recordAction(actionInput: ActionInput, actionResult: ActionResult, options?: OlapOptions): void;
|
|
84
|
+
recordStep(stepInput: StepInput, stepResult: StepResult, options?: OlapOptions): void;
|
|
97
85
|
}
|
|
86
|
+
type ActionInput = {
|
|
87
|
+
actionId?: string;
|
|
88
|
+
url?: string;
|
|
89
|
+
pathParams?: unknown;
|
|
90
|
+
queryParams?: unknown;
|
|
91
|
+
body?: unknown;
|
|
92
|
+
headers?: Record<string, string>;
|
|
93
|
+
};
|
|
98
94
|
type ActionResult = {
|
|
99
95
|
actionRunId: string;
|
|
100
96
|
actionId: string;
|
|
@@ -119,13 +115,20 @@ type ActionResult = {
|
|
|
119
115
|
childResource?: string;
|
|
120
116
|
status?: string;
|
|
121
117
|
message?: string;
|
|
122
|
-
|
|
118
|
+
headers?: Record<string, string>;
|
|
119
|
+
body?: unknown;
|
|
123
120
|
startTime?: Date;
|
|
124
121
|
endTime?: Date;
|
|
125
122
|
};
|
|
123
|
+
type StepInput = {
|
|
124
|
+
inputs?: unknown;
|
|
125
|
+
};
|
|
126
126
|
type StepResult = {
|
|
127
127
|
actionRunId: string;
|
|
128
128
|
stepId: string;
|
|
129
|
+
organizationId: string;
|
|
130
|
+
projectSecureId: string;
|
|
131
|
+
accountSecureId: string;
|
|
129
132
|
status?: string;
|
|
130
133
|
message?: string;
|
|
131
134
|
outputs?: unknown;
|
|
@@ -135,6 +138,57 @@ type StepResult = {
|
|
|
135
138
|
endTime?: Date;
|
|
136
139
|
};
|
|
137
140
|
//#endregion
|
|
141
|
+
//#region src/advanced/types.d.ts
|
|
142
|
+
interface S3ClientConfig {
|
|
143
|
+
region?: string;
|
|
144
|
+
}
|
|
145
|
+
type S3ClientBuilder = (config?: S3ClientConfig, logger?: ILogger) => S3Client | undefined;
|
|
146
|
+
interface AdvancedLog {
|
|
147
|
+
request: {
|
|
148
|
+
id?: string;
|
|
149
|
+
method?: string;
|
|
150
|
+
headers?: Record<string, unknown>;
|
|
151
|
+
url: {
|
|
152
|
+
url: string;
|
|
153
|
+
hostname: string;
|
|
154
|
+
path: string;
|
|
155
|
+
queryParams: Record<string, string>;
|
|
156
|
+
};
|
|
157
|
+
body?: string | Record<string, unknown>;
|
|
158
|
+
};
|
|
159
|
+
response: {
|
|
160
|
+
statusCode?: number;
|
|
161
|
+
headers?: Record<string, unknown>;
|
|
162
|
+
body?: unknown;
|
|
163
|
+
};
|
|
164
|
+
isBackgroundLog?: boolean;
|
|
165
|
+
}
|
|
166
|
+
//#endregion
|
|
167
|
+
//#region src/advanced/advancedWriter.d.ts
|
|
168
|
+
declare class AdvancedWriter {
|
|
169
|
+
#private;
|
|
170
|
+
constructor({
|
|
171
|
+
logger
|
|
172
|
+
}?: {
|
|
173
|
+
logger?: ILogger;
|
|
174
|
+
});
|
|
175
|
+
write(s3Client: S3Client, logData: AdvancedLog, result: ActionResult, options: AdvancedOptions): Promise<void>;
|
|
176
|
+
private generateS3Key;
|
|
177
|
+
private generateS3Command;
|
|
178
|
+
private filterHeaders;
|
|
179
|
+
private isDataSyncRequest;
|
|
180
|
+
private isBackgroundLog;
|
|
181
|
+
private getContentType;
|
|
182
|
+
private serialize;
|
|
183
|
+
private processContent;
|
|
184
|
+
}
|
|
185
|
+
//#endregion
|
|
186
|
+
//#region src/logs/types.d.ts
|
|
187
|
+
interface KafkaClientConfig {
|
|
188
|
+
brokers: string[];
|
|
189
|
+
}
|
|
190
|
+
type KafkaClientBuilder = (config?: KafkaClientConfig, logger?: ILogger) => Kafka | undefined;
|
|
191
|
+
//#endregion
|
|
138
192
|
//#region src/olap/olapClient.d.ts
|
|
139
193
|
declare class OlapClient implements IOlapClient {
|
|
140
194
|
#private;
|
|
@@ -144,17 +198,19 @@ declare class OlapClient implements IOlapClient {
|
|
|
144
198
|
getS3Client,
|
|
145
199
|
kafkaClientConfig,
|
|
146
200
|
s3ClientConfig,
|
|
147
|
-
logger
|
|
201
|
+
logger,
|
|
202
|
+
advancedWriter
|
|
148
203
|
}?: {
|
|
149
204
|
getKafkaClient?: KafkaClientBuilder;
|
|
150
205
|
getS3Client?: S3ClientBuilder;
|
|
151
206
|
kafkaClientConfig?: KafkaClientConfig;
|
|
152
207
|
s3ClientConfig?: S3ClientConfig;
|
|
153
208
|
logger?: ILogger;
|
|
209
|
+
advancedWriter?: AdvancedWriter;
|
|
154
210
|
});
|
|
155
211
|
initialize(): Promise<void>;
|
|
156
|
-
recordAction(actionResult: ActionResult, options?: OlapOptions): Promise<void>;
|
|
157
|
-
recordStep(stepResult: StepResult, options?: OlapOptions): Promise<void>;
|
|
212
|
+
recordAction(actionInput: ActionInput, actionResult: ActionResult, options?: OlapOptions): Promise<void>;
|
|
213
|
+
recordStep(stepInput: StepInput, stepResult: StepResult, options?: OlapOptions): Promise<void>;
|
|
158
214
|
}
|
|
159
215
|
//#endregion
|
|
160
216
|
//#region src/olap/olapClientManager.d.ts
|
|
@@ -191,4 +247,4 @@ declare class OlapClientManager {
|
|
|
191
247
|
static resetInstance(): void;
|
|
192
248
|
}
|
|
193
249
|
//#endregion
|
|
194
|
-
export { type ActionResult, type IOlapClient, OlapClient, OlapClientManager, type OlapOptions, type StepResult };
|
|
250
|
+
export { type ActionInput, type ActionResult, type IOlapClient, OlapClient, OlapClientManager, type OlapOptions, type StepInput, type StepResult };
|
package/dist/index.mjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{S3Client as e}from"@aws-sdk/client-s3";import{Kafka as t,Partitioners as n}from"kafkajs";const r=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}};var i=class{#e;#t;constructor(e,t){this.#e=e,this.#t=t}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,t,n){if(!n?.enabled){this.#t?.debug({message:`Logs sink is disabled, skipping send to bucket ${e}`,category:`AdvancedSink`});return}let i=r(t);this.#t?.info({message:`[S3] Sent to bucket ${e}: ${i}`,context:{bucket:e,payload:t},category:`S3Sink`}),await Promise.resolve()}};const a=(t,n)=>{try{return new e(t)??void 0}catch(e){let t=e;n?.error({message:`Error building s3 client: ${t.message}`,error:t,code:`BuildS3ClientError`,category:`buildS3Client`});return}},o=(e,n)=>{try{return new t(e)??void 0}catch(e){let t=e;n?.error({message:`Error building kafka client: ${t.message}`,error:t,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}};var s=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:n.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping send to topic ${e}`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let i=r(this.buildLog(t));this.#n?.debug({message:`Sending to topic ${e}: ${i}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:i}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e){if(this.isActionResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,actionId:e.actionId,organizationId:e.organizationId,projectSecureId:e.projectSecureId,accountSecureId:e.accountSecureId,mode:e.mode,connectorKey:e.connectorKey,connectorVersion:e.connectorVersion,actionType:e.actionType,category:e.category,originOwnerId:e.originOwnerId,originOwnerName:e.originOwnerName,httpMethod:e.httpMethod,url:e.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,status:e.status,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepId:e.stepId,skipped:e.skipped,status:e.status,message:e.message,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const c={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function l(e){return{...c,...e}}var u=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=o,getS3Client:t=a,kafkaClientConfig:n,s3ClientConfig:r,logger:c}={}){this.name=`OlapClient`,this.#e=e(n,c),this.#t=t(r,c),this.#n=c,this.#r=new s(this.#e,this.#n),this.#i=new i(this.#t,this.#n)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t){let{logs:n,advanced:r}=l(t);if(n?.enabled)try{await this.#r?.send(`actions`,e,n)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(r?.enabled)try{await this.#i?.send(`actions`,e,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t){let{logs:n,advanced:r}=l(t);if(n?.enabled)try{await this.#r?.send(`steps`,e,n)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(r?.enabled)try{await this.#i?.send(`steps`,e,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const d=async({getKafkaClient:e=o,getS3Client:t=a,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let s=new u({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await s.initialize(),s};var f=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=d}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{u as OlapClient,f as OlapClientManager};
|
|
1
|
+
import{isObject as e,isString as t,notMissing as n}from"@stackone/utils";import{PutObjectCommand as r,S3Client as i}from"@aws-sdk/client-s3";import{addDays as a,getUnixTime as o}from"date-fns";import{Kafka as s,Partitioners as c}from"kafkajs";const l=e=>{try{return JSON.stringify(e)}catch{return`[Unserializable payload]`}},u=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`],d=[`application/octet-stream`,`application/pdf`,`application/msword`,`application/vnd.openxmlformats-officedocument.wordprocessingml.document`,`application/vnd.ms-excel`,`application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`,`application/zip`,`application/x-zip-compressed`],f=[`image/`,`audio/`,`video/`,`application/vnd.`,`binary`];var p=class{#e;constructor({logger:e}={}){this.#e=e}async write(e,t,n,r){try{let i=r?.ttl??30,a=await this.generateS3Command(t,n,i);await e.send(a)}catch(e){throw this.#e?.error({message:`Failed to write advanced logs to S3`,category:`AdvancedWriter`,code:`AdvancedWriteError`,error:e}),e}}generateS3Key(e,t,n,r){let i=`${t}/${n}/${e}`;return r?`${i}/steps/${r}.json`:`${i}/${e}.json`}async generateS3Command(e,t,n){let i=process.env.ADVANCED_LOGS_BUCKET;if(!i)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);let s=this.generateS3Key(t.actionRunId,t.organizationId,t.projectSecureId),c=o(a(new Date,n)),l=this.serialize(e,c,10485760),u=`ttl=30d`;return n===1?u=`ttl=1d`:n===7&&(u=`ttl=7d`),new r({Bucket:i,Key:s,Body:l,ContentType:`application/json`,Expires:new Date(c*1e3),Tagging:u})}filterHeaders(e){if(!e)return;let t={},n=u.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return n(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}serialize(n,r,i){let a=n.response?.body;(t(a)&&a.length>i||Buffer.isBuffer(a)&&a.byteLength>i||a&&e(a)&&!Array.isArray(a)&&Object.values(a).some(e=>t(e)&&e.length>i||Buffer.isBuffer(e)&&e.byteLength>i))&&(n={...n,response:{...n.response,body:{error:`Error.TOO_LARGE`}}});let o=JSON.stringify({data:n,expirationTime:r});if(Buffer.byteLength(o,`utf8`)>i){let e={...n,response:{...n.response,body:{error:`Error.TOO_LARGE`}}};o=JSON.stringify({data:e,expirationTime:r})}return o}processContent(e,t,n){let r=this.getContentType(e);if(!r)return l(t);let i=r.toLowerCase().split(`;`)[0].trim();return d.includes(i)||f.some(e=>i.startsWith(e))?{type:`binary`,action:n,contentType:r}:l(t)}};const m=[`x-datadog-parent-id`,`x-datadog-sampling-priority`,`x-datadog-tags`,`x-datadog-trace-id`,`x-forwarded-proto`,`x-forwarded-port`,`x-forwarded-for`,`x-amzn-trace-id`,`traceparent`,`tracestate`,`x-request-nonce`,`x-signing-method`,`x-signature`,`host`,`via`];var h=class{#e;#t;#n;constructor(e,t,n){this.#e=e,this.#t=t,this.#n=n??new p({logger:t})}async initialize(){if(!this.#e){this.#t?.warning({message:`No s3 client provided, advanced sink will not function`,category:`AdvancedSink`});return}}async send(e,t,n){if(!n?.enabled){this.#t?.debug({message:`Advanced sink is disabled, skipping sending to advanced sink`,category:`AdvancedSink`});return}if(!this.#e){this.#t?.warning({message:`No s3 client available, cannot send to advanced sink`,category:`AdvancedSink`});return}try{let i=await this.createS3Item(e,t,n.ttl);await this.#e.send(new r({Bucket:i.Bucket,Key:i.Key,Body:i.Body,ContentType:i.ContentType,Expires:i.Expires,Tagging:i.Tagging}))}catch(e){throw this.#t?.error({message:`Failed to write advanced logs to S3 account ${t.accountSecureId}`,error:e,code:`AdvancedLogWriteError`,category:`AdvancedSink`,context:{actionRunId:t.actionRunId}}),e}}async createS3Item(e,t,n){let{projectSecureId:r,organizationId:i}=t,a=`ttl=30d`;typeof n==`number`&&(n===1?a=`ttl=1d`:n===7&&(a=`ttl=7d`));let o=Math.floor(Date.now()/1e3)+(n??30)*24*60*60,s=process.env.ADVANCED_LOGS_BUCKET;if(!s)throw Error(`ADVANCED_LOGS_BUCKET environment variable is not set`);return{Bucket:s,Key:this.generateS3Key(t,i,r),Body:this.serialize(e,t,o,10485760),ContentType:`application/json`,Expires:new Date(o*1e3),Tagging:a}}generateS3Key(e,t,n){return`actionId`in e?`${t}/${n}/${e.actionRunId}/${e.actionRunId}.json`:`${t}/${n}/${e.actionRunId}/steps/${e.stepId}.json`}serialize(e,t,n,r){return`actionId`in t?JSON.stringify(this.serializeActionResult(e,t,n,r)):JSON.stringify(this.serializeStepResult(e,t))}serializeActionResult(e,t,n,r){let i=e.headers,a=t.headers,o=e.body,s=t.body,c=this.isBackgroundLog(t);return{data:{request:{id:t.actionRunId,actionId:t.actionId,method:t.httpMethod,headers:this.filterHeaders(i),url:{url:e.url,path:e.pathParams,queryParams:e.queryParams},body:o},response:{statusCode:t.status?parseInt(t.status,10):void 0,headers:this.filterHeaders(a),body:s},...c?{isBackgroundLog:!0}:{}},metadata:{...c?{isBackgroundLog:!0}:{},expirationTime:n},expirationTime:n}}serializeStepResult(e,t){return{}}filterHeaders(e){if(!e)return;let t={},n=m.map(e=>e.toLowerCase());for(let[r,i]of Object.entries(e))n.includes(r.toLowerCase())||(t[r]=i);return t}isDataSyncRequest(e){return e?.mode===`data_sync`}isBackgroundLog(e){return n(e.mode)&&[`refresh_authentication`].includes(e.mode)}getContentType(e){if(!e)return;let t=[`content-type`,`contenttype`];return Object.entries(e).find(([e])=>t.includes(e.toLowerCase()))?.[1]?.toString()}};const g=(e,t)=>{try{return new i(e)??void 0}catch(e){let n=e;t?.error({message:`Error building s3 client: ${n.message}`,error:n,code:`BuildS3ClientError`,category:`buildS3Client`});return}},_=(e,t)=>{try{return new s(e)??void 0}catch(e){let n=e;t?.error({message:`Error building kafka client: ${n.message}`,error:n,code:`BuildKafkaClientError`,category:`buildKafkaClient`});return}};var v=class{#e;#t;#n;constructor(e,t){this.#e=e,this.#n=t}async initialize(){if(!this.#e){this.#n?.warning({message:`No kafka client provided, logs sink cannot be initialized`,category:`LogsSink`});return}if(!this.#t){try{this.#t=this.#e.producer({createPartitioner:c.DefaultPartitioner})}catch(e){this.#n?.error({message:`Failed to create kafka producer for logs sink`,code:`KafkaProducerCreationError`,category:`LogsSink`,error:e}),this.#t=void 0;return}try{await this.#t.connect()}catch(e){this.#n?.error?.({message:`Failed to connect kafka producer for logs sink`,code:`KafkaProducerConnectionError`,category:`LogsSink`,error:e}),this.#t=void 0;return}this.#n?.info({message:`Logs sink kafka producer initialized`,category:`LogsSink`})}}async send(e,t,n){if(!n?.enabled){this.#n?.debug({message:`Logs sink is disabled, skipping send to topic ${e}`,category:`KafkaSink`});return}if(!this.#t)throw this.#n?.error({message:`Kafka not initialized, dropping message for topic ${e}`,category:`KafkaSink`,code:`KafkaNotReady`}),Error(`Kafka client is not initialized`);let r=l(this.buildLog(t));this.#n?.debug({message:`Sending to topic ${e}: ${r}`,category:`KafkaSink`});try{let t=await this.#t.send({topic:e,messages:[{value:r}]});this.#n?.debug({message:`Kafka producer response: ${JSON.stringify(t)}`,category:`KafkaSink`})}catch(t){throw this.#n?.error({message:`Error sending to topic ${e}: ${t.message}`,category:`KafkaSink`,error:t,code:`KafkaSendError`}),t}}buildLog(e){if(this.isActionResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,actionId:e.actionId,organizationId:e.organizationId,projectSecureId:e.projectSecureId,accountSecureId:e.accountSecureId,mode:e.mode,connectorKey:e.connectorKey,connectorVersion:e.connectorVersion,actionType:e.actionType,category:e.category,originOwnerId:e.originOwnerId,originOwnerName:e.originOwnerName,httpMethod:e.httpMethod,url:e.url,sourceId:e.sourceId,sourceType:e.sourceType,sourceValue:e.sourceValue,status:e.status,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));if(this.isStepResult(e))return Object.fromEntries(Object.entries({actionRunId:e.actionRunId,stepId:e.stepId,skipped:e.skipped,status:e.status,message:e.message,startTime:e.startTime,endTime:e.endTime,durationMs:this.calculateDuration(e.startTime,e.endTime),eventTime:new Date}).filter(([,e])=>e!==void 0));throw this.#n?.error({message:`Unknown result type, cannot build log`,category:`KafkaSink`,code:`UnknownResultType`}),Error(`Unknown result type`)}calculateDuration(e,t){if(!(e instanceof Date)||!(t instanceof Date))return;let n=e.getTime(),r=t.getTime();if(!(Number.isNaN(n)||Number.isNaN(r)||r<n))return r-n}isActionResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`actionId`in e}isStepResult(e){return typeof e==`object`&&!!e&&`actionRunId`in e&&`stepId`in e}};const y={logs:{enabled:!0},advanced:{enabled:!1,ttl:7,errorsOnly:!1}};function b(e){return{...y,...e}}var x=class{#e;#t;#n;#r;#i;constructor({getKafkaClient:e=_,getS3Client:t=g,kafkaClientConfig:n,s3ClientConfig:r,logger:i,advancedWriter:a}={}){this.name=`OlapClient`,this.#e=e(n,i),this.#t=t(r,i),this.#n=i,this.#r=new v(this.#e,this.#n),this.#i=new h(this.#t,this.#n,a)}async initialize(){await this.#r?.initialize(),await this.#i?.initialize()}async recordAction(e,t,n){let{logs:r,advanced:i}=b(n);if(r?.enabled)try{await this.#r?.send(`actions`,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}async recordStep(e,t,n){let{logs:r,advanced:i}=b(n);if(r?.enabled)try{await this.#r?.send(`steps`,t,r)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to kafka: ${e.message}`,category:`OlapClient`,error:e})}if(i?.enabled)try{await this.#i?.send(e,t,i)}catch(e){this.#n?.warning({message:`[OlapClient] Error sending to s3: ${e.message}`,category:`OlapClient`,error:e})}}};const S=async({getKafkaClient:e=_,getS3Client:t=g,kafkaClientConfig:n,s3ClientConfig:r,logger:i}={})=>{let a=new x({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i});return await a.initialize(),a};var C=class{static{this.olapClientPromise=null}static getInstance({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i,getOlapClient:a=S}={}){return this.olapClientPromise||=a({getKafkaClient:e,getS3Client:t,kafkaClientConfig:n,s3ClientConfig:r,logger:i}),this.olapClientPromise}static resetInstance(){this.olapClientPromise=null}};export{x as OlapClient,C as OlapClientManager};
|