@uploadista/data-store-s3 0.1.4-beta.1 → 1.0.0-beta.2

package/dist/index.cjs

@@ -1 +1 @@
-let e=require(`@uploadista/core/errors`),t=require(`@uploadista/core/types`),n=require(`@uploadista/observability`),r=require(`effect`),i=require(`@aws-sdk/client-s3`);const a=e=>e&&e.length>0?e.reduce((e,t)=>e+(t?.Size??0),0):0,o=(e,t,n,r,i=5497558138880)=>{let a=e??i,o;o=a<=t?a:a<=t*r?t:Math.ceil(a/r);let s=e&&e<n?o:Math.max(o,n),c=1024;return Math.ceil(s/c)*c},s=e=>`${e}.part`,c=(e,t)=>{let n=new Date(e);return new Date(n.getTime()+t)},l=(t,i,a={})=>(r.Effect.runSync((0,n.trackS3Error)(t,i,a)),e.UploadistaError.fromCode(`FILE_WRITE_ERROR`,i)),u=e=>[`NotFound`,`NoSuchKey`,`NoSuchUpload`].includes(e),d=e=>{if(typeof e!=`object`||!e)return null;let t={};return`code`in e&&typeof e.code==`string`&&(t.code=e.code),`name`in e&&typeof e.name==`string`&&(t.name=e.name),Object.keys(t).length>0?t:null},f=(t,n,i={})=>{let a=d(n);return a&&(a.code&&u(a.code)||a.name&&u(a.name))?(r.Effect.runSync(r.Effect.logWarning(`File not found during ${t} operation`).pipe(r.Effect.annotateLogs({error_code:a.code,error_name:a.name,...i}))),e.UploadistaError.fromCode(`FILE_NOT_FOUND`)):l(t,n,i)},p=e=>e===`NoSuchUpload`||e===`NoSuchKey`,m=t=>{let n=d(t);if(n&&(n.code&&p(n.code)||n.name&&p(n.name)))return!0;if(t instanceof e.UploadistaError&&t.cause){let e=d(t.cause);if(e&&(e.code&&p(e.code)||e.name&&p(e.name)))return!0}return!1};function h(e){if(e instanceof ReadableStream||e&&typeof e==`object`&&`getReader`in e)return e;if(e&&typeof e==`object`&&`pipe`in e&&`on`in e){let t=e;return new ReadableStream({start(e){t.on(`data`,t=>{e.enqueue(new Uint8Array(t))}),t.on(`end`,()=>{e.close()}),t.on(`error`,t=>{e.error(t)})}})}throw Error(`Unsupported body type: ${typeof e}. Expected ReadableStream or Node.js Readable.`)}var g=class extends r.Context.Tag(`S3ClientService`)(){};const _=(e,t)=>{let a=new i.S3(e),o=e=>r.Effect.tryPromise({try:async()=>h((await a.getObject({Bucket:t,Key:e})).Body),catch:n=>l(`getObject`,n,{key:e,bucket:t})}),c=e=>r.Effect.tryPromise({try:async()=>{try{return(await a.headObject({Bucket:t,Key:e})).ContentLength}catch(e){if(e instanceof i.NotFound)return;throw e}},catch:n=>l(`headObject`,n,{key:e,bucket:t})}),u=(e,n)=>r.Effect.tryPromise({try:async()=>(await a.putObject({Bucket:t,Key:e,Body:n})).ETag||``,catch:r=>l(`putObject`,r,{key:e,bucket:t,size:n.length})}),d=e=>r.Effect.tryPromise({try:async()=>{await a.deleteObject({Bucket:t,Key:e})},catch:n=>l(`deleteObject`,n,{key:e,bucket:t})});return{bucket:t,getObject:o,headObject:c,putObject:u,deleteObject:d,deleteObjects:e=>r.Effect.tryPromise({try:()=>a.deleteObjects({Bucket:t,Delete:{Objects:e.map(e=>({Key:e}))}}),catch:n=>l(`deleteObjects`,n,{keys:e.length,bucket:t})}),createMultipartUpload:e=>(0,n.withS3ApiMetrics)(`createMultipartUpload`,r.Effect.tryPromise({try:async()=>{let t={Bucket:e.bucket,Key:e.key};e.contentType&&(t.ContentType=e.contentType),e.cacheControl&&(t.CacheControl=e.cacheControl);let n=await a.createMultipartUpload(t);if(!n.UploadId)throw Error(`Upload ID is undefined`);if(!n.Key)throw Error(`Key is undefined`);return{uploadId:n.UploadId,bucket:e.bucket,key:n.Key}},catch:t=>l(`createMultipartUpload`,t,e)})),uploadPart:e=>(0,n.withS3ApiMetrics)(`uploadPart`,r.Effect.tryPromise({try:()=>a.uploadPart({Bucket:e.bucket,Key:e.key,UploadId:e.uploadId,PartNumber:e.partNumber,Body:e.data}),catch:t=>l(`uploadPart`,t,{upload_id:e.key,part_number:e.partNumber,part_size:e.data.length,s3_bucket:e.bucket})}).pipe(r.Effect.map(e=>e.ETag))),completeMultipartUpload:(e,t)=>(0,n.withS3ApiMetrics)(`completeMultipartUpload`,r.Effect.tryPromise({try:()=>a.completeMultipartUpload({Bucket:e.bucket,Key:e.key,UploadId:e.uploadId,MultipartUpload:{Parts:t.map(e=>({ETag:e.ETag,PartNumber:e.PartNumber}))}}).then(e=>e.Location),catch:n=>l(`completeMultipartUpload`,n,{upload_id:e.key,parts_count:t.length,s3_bucket:e.bucket})})),abortMultipartUpload:e=>r.Effect.tryPromise({try:async()=>{await a.abortMultipartUpload({Bucket:e.bucket,Key:e.key,UploadId:e.uploadId})},catch:t=>f(`abortMultipartUpload`,t,{upload_id:e.key,s3_bucket:e.bucket})}),listParts:e=>r.Effect.tryPromise({try:async()=>{let t={Bucket:e.bucket,Key:e.key,UploadId:e.uploadId,PartNumberMarker:e.partNumberMarker},n=await a.listParts(t);return{parts:n.Parts??[],isTruncated:n.IsTruncated??!1,nextPartNumberMarker:n.NextPartNumberMarker}},catch:t=>l(`listParts`,t,{upload_id:e.key,s3_bucket:e.bucket})}),listMultipartUploads:(e,n)=>r.Effect.tryPromise({try:()=>a.listMultipartUploads({Bucket:t,KeyMarker:e,UploadIdMarker:n}),catch:e=>l(`listMultipartUploads`,e,{bucket:t})}),getIncompletePart:e=>r.Effect.tryPromise({try:async()=>{try{return h((await a.getObject({Bucket:t,Key:s(e)})).Body)}catch(e){if(e instanceof i.NoSuchKey)return;throw e}},catch:n=>l(`getIncompletePart`,n,{upload_id:e,bucket:t})}),getIncompletePartSize:e=>c(s(e)),putIncompletePart:(e,t)=>u(s(e),t).pipe(r.Effect.tap(()=>r.Effect.logInfo(`Incomplete part uploaded`).pipe(r.Effect.annotateLogs({upload_id:e})))),deleteIncompletePart:e=>d(s(e))}},v=(e,t)=>r.Layer.succeed(g,_(e,t)),y=e=>{let{id:t,metadata:n}=e;if(!n)return t;let r=n.filename||n.fileName||n.name;return typeof r==`string`&&r.includes(`.`)?`${t}${r.substring(r.lastIndexOf(`.`))}`:t};function b(i){let{deliveryUrl:s,partSize:l,minPartSize:u=5242880,useTags:d=!0,maxMultipartParts:f=1e4,maxConcurrentPartUploads:p=60,expirationPeriodInMilliseconds:h=1e3*60*60*24*7,s3ClientConfig:{bucket:_}}=i;return r.Effect.gen(function*(){let i=yield*g,v=yield*t.UploadFileKVStore,b=l||8*1024*1024,x=t=>{let n=t.storage.uploadId;return n?r.Effect.succeed(n):r.Effect.fail(e.UploadistaError.fromCode(`FILE_WRITE_ERROR`,Error(`Upload ID is undefined`)))},S=(t,a,o)=>{let s=y(t);return(0,n.withS3TimingMetrics)(n.s3PartUploadDurationHistogram,r.Effect.gen(function*(){let c=yield*x(t),l=yield*i.uploadPart({bucket:i.bucket,key:s,uploadId:c,partNumber:o,data:a}).pipe(r.Effect.retry({schedule:r.Schedule.exponential(`1 second`,2).pipe(r.Schedule.intersect(r.Schedule.recurs(3))),while:e=>!m(e)}),r.Effect.tapError(e=>r.Effect.logWarning(`Retrying part upload`).pipe(r.Effect.annotateLogs({upload_id:t.id,part_number:o,error_message:e.message,retry_attempt:`unknown`,part_size:a.length,s3_bucket:i.bucket}))),r.Effect.catchAll(n=>m(n)?r.Effect.fail(e.UploadistaError.fromCode(`UPLOAD_CANCELLED`,{cause:n,body:`Upload ${t.id} was cancelled`})):r.Effect.fail(n)));return yield*(0,n.s3UploadPartsTotal)(r.Effect.succeed(1)),yield*r.Effect.logInfo(`Part uploaded successfully`).pipe(r.Effect.annotateLogs({upload_id:t.id,part_number:o,part_size:a.length,etag:l})),l})).pipe(r.Effect.withSpan(`s3-upload-part-${o}`,{attributes:{"upload.id":t.id,"upload.part_number":o,"upload.part_size":a.length,"s3.bucket":i.bucket,"s3.key":s}}))},C=(e,t)=>i.putIncompletePart(e,t),w=e=>r.Effect.gen(function*(){let t=yield*i.getIncompletePart(e);if(!t)return;let n=t.getReader(),a=[],o=0;try{for(;;){let{done:e,value:t}=yield*r.Effect.promise(()=>n.read());if(e)break;a.push(t),o+=t.length}}finally{n.releaseLock()}let s=r.Stream.fromIterable(a);return{size:o,stream:s}}),T=e=>i.deleteIncompletePart(e),E=e=>i.getIncompletePartSize(e),D=(e,t)=>{let a=y(e);return r.Effect.gen(function*(){let n=yield*x(e);return yield*i.completeMultipartUpload({bucket:i.bucket,key:a,uploadId:n},t)}).pipe(r.Effect.tap(()=>(0,n.s3UploadSuccessTotal)(r.Effect.succeed(1))),r.Effect.withSpan(`s3-complete-multipart-upload`,{attributes:{"upload.id":e.id,"upload.parts_count":t.length,"s3.bucket":i.bucket,"s3.key":a}}))},O=e=>{let t=y(e);return r.Effect.gen(function*(){let n=yield*x(e);yield*i.abortMultipartUpload({bucket:i.bucket,key:t,uploadId:n}),yield*i.deleteObjects([t])})},k=(e,t,n,a)=>r.Effect.gen(function*(){let r=yield*i.listParts({bucket:i.bucket,key:e,uploadId:t,partNumberMarker:a}),o=r.parts;if(r.isTruncated){let i=yield*k(e,t,n,r.nextPartNumberMarker);o=[...o,...i.parts]}return a||o.sort((e,t)=>(e.PartNumber??0)-(t.PartNumber??0)),{uploadFound:!0,parts:o}}).pipe(r.Effect.catchAll(e=>m(e)?r.Effect.logWarning(`S3 upload not found during listParts`).pipe(r.Effect.annotateLogs({upload_id:n,error_code:e.code}),r.Effect.as({uploadFound:!1,parts:[]})):r.Effect.fail(e))),A=(e,t)=>r.Effect.gen(function*(){let n=yield*v.get(e),r=yield*x(n);return yield*k(y(n),r,e,t)}),j=(e,t)=>r.Effect.gen(function*(){if(!t)return 0;let n=yield*v.get(e.id),r=n.storage.uploadId;return r&&(yield*v.set(e.id,{...n,storage:{...n.storage,uploadId:r}})),0}),M=e=>r.Effect.gen(function*(){yield*r.Effect.logInfo(`Clearing cache`).pipe(r.Effect.annotateLogs({upload_id:e})),yield*v.delete(e)}),N=e=>{let t=y(e);return r.Effect.gen(function*(){yield*r.Effect.logInfo(`Initializing multipart upload`).pipe(r.Effect.annotateLogs({upload_id:e.id}));let a=yield*i.createMultipartUpload({bucket:i.bucket,key:t,uploadId:``,contentType:e.metadata?.contentType?.toString(),cacheControl:e.metadata?.cacheControl?.toString()}),o={...e,storage:{...e.storage,path:a.key,uploadId:a.uploadId,bucket:a.bucket},url:`${s}/${t}`};return yield*v.set(e.id,o),yield*r.Effect.logInfo(`Multipart upload created`).pipe(r.Effect.annotateLogs({upload_id:e.id,s3_upload_id:o.storage.uploadId,s3_key:t})),yield*(0,n.s3UploadRequestsTotal)(r.Effect.succeed(1)),yield*(0,n.s3FileSizeHistogram)(r.Effect.succeed(e.size||0)),o}).pipe(r.Effect.withSpan(`s3-create-upload`,{attributes:{"upload.id":e.id,"upload.size":e.size||0,"s3.bucket":i.bucket,"s3.key":t}}))},P=e=>r.Effect.gen(function*(){yield*r.Effect.logInfo(`Initializing multipart upload`).pipe(r.Effect.annotateLogs({upload_id:e.id}));let t=yield*N(e);return yield*v.set(e.id,t),yield*r.Effect.logInfo(`Multipart upload created`).pipe(r.Effect.annotateLogs({upload_id:e.id,s3_upload_id:t.storage.uploadId})),yield*(0,n.s3UploadRequestsTotal)(r.Effect.succeed(1)),t}).pipe(r.Effect.withSpan(`s3-create-upload`,{attributes:{"upload.id":e.id,"upload.size":e.size||0,"s3.bucket":_}})),F=e=>r.Effect.gen(function*(){yield*O(yield*v.get(e)),yield*M(e)}),I=(e,t)=>(0,n.withS3UploadMetrics)(e.file_id,(0,n.withS3TimingMetrics)(n.s3UploadDurationHistogram,r.Effect.gen(function*(){let{stream:i,file_id:a,offset:s}=e,{onProgress:c}=t,l=Date.now();yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(1));let{uploadFile:d,nextPartNumber:m,offset:h,data:g,existingPartSize:_}=yield*R(a,s,i),v=_||o(d.size,b,u,f);yield*r.Effect.logInfo(`Part size decision`).pipe(r.Effect.annotateLogs({upload_id:a,existing_part_size:_,calculated_part_size:o(d.size,b,u,f),final_part_size:v,next_part_number:m}));let y=h+(yield*U(d,g,m,h,v,u,p,c));return d.size===y&&(yield*z(a,d,l)),y}).pipe(r.Effect.ensuring((0,n.s3ActiveUploadsGauge)(r.Effect.succeed(0)))))),L=e=>r.Effect.gen(function*(){let t=yield*v.get(e),{parts:n,uploadFound:r}=yield*A(e);if(!r)return{...t,offset:t.size,size:t.size};let i=a(n),o=yield*E(e);return{...t,offset:i+(o??0),size:t.size,storage:t.storage}}),R=(e,t,n)=>r.Effect.gen(function*(){let i=yield*v.get(e),{parts:a}=yield*A(e),o=(a.length>0&&a[a.length-1].PartNumber?a[a.length-1].PartNumber??0:0)+1,s=a.length>0&&a[0].Size?a[0].Size:null;if(s&&a.length>1){let t=a.slice(0,-1).find(e=>e.Size!==s);t&&(yield*r.Effect.logWarning(`Inconsistent part sizes detected in existing upload`).pipe(r.Effect.annotateLogs({upload_id:e,expected_size:s,inconsistent_part:t.PartNumber,inconsistent_size:t.Size})))}let c=yield*w(e);if(c){yield*T(e);let a=t-c.size,l=c.stream.pipe(r.Stream.concat(n));return{uploadFile:i,nextPartNumber:o,offset:a,incompletePartSize:c.size,data:l,existingPartSize:s}}else return{uploadFile:i,nextPartNumber:o,offset:t,incompletePartSize:0,data:n,existingPartSize:s}}),z=(e,t,i)=>r.Effect.gen(function*(){let{parts:a}=yield*A(e);yield*r.Effect.logInfo(`Attempting to complete multipart upload`).pipe(r.Effect.annotateLogs({upload_id:e,parts_count:a.length,parts_info:a.map((e,t)=>({part_number:e.PartNumber,size:e.Size,etag:e.ETag,is_final_part:t===a.length-1}))})),yield*D(t,a),yield*j(t,d);let o=Date.now()-i,s=t.size||0,c=o>0?s*1e3/o:0,l=a.length>0?a.reduce((e,t)=>e+(t.Size||0),0)/a.length:void 0;yield*(0,n.logS3UploadCompletion)(e,{fileSize:s,totalDurationMs:o,partsCount:a.length,averagePartSize:l,throughputBps:c})}).pipe(r.Effect.tapError(t=>r.Effect.gen(function*(){yield*(0,n.s3UploadErrorsTotal)(r.Effect.succeed(1)),yield*r.Effect.logError(`Failed to finish upload`).pipe(r.Effect.annotateLogs({upload_id:e,error:String(t)}))}))),B=()=>r.Effect.gen(function*(){if(h===0)return 0;let e,t,n=!0,a=0;for(;n;){let o=yield*i.listMultipartUploads(e,t),s=o.Uploads?.filter(e=>{let t=e.Initiated;return t&&Date.now()>c(t.toISOString(),h).getTime()})||[],l=s.filter(e=>!!e.Key).map(e=>e.Key);l.length>0&&(yield*i.deleteObjects(l),yield*r.Effect.forEach(s,e=>r.Effect.gen(function*(){!e.Key||!e.UploadId||(yield*i.abortMultipartUpload({bucket:_,key:e.Key,uploadId:e.UploadId}))})),a+=l.length),n=o.IsTruncated??!1,n&&(e=o.NextKeyMarker,t=o.NextUploadIdMarker)}return a}),V=e=>t=>r.Stream.async(n=>{let i=new Uint8Array,a=1,o=0,s=(t,i=!1)=>{r.Effect.runSync(r.Effect.logInfo(`Creating chunk`).pipe(r.Effect.annotateLogs({part_number:a,chunk_size:t.length,expected_size:e,is_final_chunk:i,total_bytes_processed:o+t.length}))),n.single({partNumber:a++,data:t,size:t.length})},c=t=>{let n=new Uint8Array(i.length+t.length);for(n.set(i),n.set(t,i.length),i=n,o+=t.length;i.length>=e;){let t=i.slice(0,e);i=i.slice(e),s(t,!1)}};r.Effect.runFork(t.pipe(r.Stream.runForEach(e=>r.Effect.sync(()=>c(e))),r.Effect.andThen(()=>r.Effect.sync(()=>{i.length>0&&s(i,!0),n.end()})),r.Effect.catchAll(e=>r.Effect.sync(()=>n.fail(e)))))}),H=(e,t=0)=>n=>e?r.Effect.gen(function*(){let i=yield*r.Ref.make(t);return n.pipe(r.Stream.tap(t=>r.Effect.gen(function*(){e(yield*r.Ref.updateAndGet(i,e=>e+t.length))})))}).pipe(r.Stream.unwrap):n,U=(t,i,a,o,s,c,l,u)=>r.Effect.gen(function*(){yield*r.Effect.logInfo(`Starting part uploads`).pipe(r.Effect.annotateLogs({upload_id:t.id,init_offset:o,file_size:t.size,part_size:s,min_part_size:c}));let d=i.pipe(H(u,o),V(s)),f=yield*r.Ref.make(o),p=yield*r.Ref.make(0),m=i=>r.Effect.gen(function*(){let o=yield*r.Ref.updateAndGet(f,e=>e+i.size),l=o>=(t.size||0);yield*r.Effect.logDebug(`Processing chunk`).pipe(r.Effect.annotateLogs({upload_id:t.id,cumulative_offset:o,file_size:t.size,chunk_size:i.size,is_final_part:l}));let u=a+i.partNumber-1;i.size>s&&(yield*r.Effect.fail(e.UploadistaError.fromCode(`FILE_WRITE_ERROR`,Error(`Part size ${i.size} exceeds upload part size ${s}`)))),i.size>=c||l?(yield*r.Effect.logDebug(`Uploading multipart chunk`).pipe(r.Effect.annotateLogs({upload_id:t.id,part_number:u,chunk_size:i.size,min_part_size:c,is_final_part:l})),yield*S(t,i.data,u),yield*(0,n.s3PartSizeHistogram)(r.Effect.succeed(i.size))):yield*C(t.id,i.data),yield*r.Ref.update(p,e=>e+i.size)});return yield*d.pipe(r.Stream.runForEach(e=>m(e)),r.Effect.withConcurrency(l)),yield*r.Ref.get(p)}),W=()=>({supportsParallelUploads:!0,supportsConcatenation:!0,supportsDeferredLength:!0,supportsResumableUploads:!0,supportsTransactionalUploads:!0,supportsStreamingRead:!0,supportsStreamingWrite:!0,maxConcurrentUploads:p,minChunkSize:u,maxChunkSize:5368709120,maxParts:f,optimalChunkSize:b,requiresOrderedChunks:!1,requiresMimeTypeValidation:!0,maxValidationSize:void 0}),G=()=>({minChunkSize:u,maxChunkSize:5368709120,optimalChunkSize:b,requiresOrderedChunks:!1}),K=e=>{let t=W(),n=(()=>{switch(e){case`parallel`:return t.supportsParallelUploads;case`single`:return!0;default:return!1}})();return r.Effect.succeed(n)},q=e=>{let t=new Uint8Array(e.reduce((e,t)=>e+t.length,0)),n=0;for(let r of e)t.set(r,n),n+=r.length;return t},J=async e=>{let t=e.getReader(),n=[];for(;;){let{done:e,value:r}=await t.read();if(e)break;n.push(r)}return q(n)};return{bucket:_,create:P,remove:F,write:I,getUpload:L,read:t=>r.Effect.gen(function*(){let n=yield*v.get(t);if(console.log(n),!n.id)return yield*r.Effect.fail(e.UploadistaError.fromCode(`FILE_READ_ERROR`,Error(`Upload Key is undefined`)));let a=y(n),o=yield*i.getObject(a);return yield*r.Effect.promise(()=>J(o))}),readStream:(n,a)=>r.Effect.gen(function*(){let o=yield*v.get(n);if(!o.id)return yield*r.Effect.fail(e.UploadistaError.fromCode(`FILE_READ_ERROR`,Error(`Upload Key is undefined`)));let s={...t.DEFAULT_STREAMING_CONFIG,...a},c=y(o),l=yield*i.getObject(c);return r.Stream.async(t=>{let n=l.getReader(),i=s.chunkSize,a=new Uint8Array;return(async()=>{try{for(;;){let{done:e,value:r}=await n.read();if(e){a.length>0&&t.single(a),t.end();return}if(r){let e=new Uint8Array(a.length+r.length);for(e.set(a),e.set(r,a.length),a=e;a.length>=i;){let e=a.slice(0,i);a=a.slice(i),t.single(e)}}}}catch(n){t.fail(new e.UploadistaError({code:`FILE_READ_ERROR`,status:500,body:`Failed to read S3 object stream`,details:`S3 stream read failed: ${String(n)}`}))}})(),r.Effect.sync(()=>{n.releaseLock()})})}),writeStream:(t,a)=>(0,n.withS3TimingMetrics)(n.s3UploadDurationHistogram,r.Effect.gen(function*(){let c=Date.now(),l=t;yield*r.Effect.logInfo(`Starting streaming write to S3`).pipe(r.Effect.annotateLogs({upload_id:t,s3_key:l,size_hint:a.sizeHint})),yield*(0,n.s3UploadRequestsTotal)(r.Effect.succeed(1)),yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(1));let d=a.sizeHint?o(a.sizeHint,b,u,f):b,p=(yield*i.createMultipartUpload({bucket:i.bucket,key:l,uploadId:``,contentType:a.contentType})).uploadId;yield*r.Effect.logInfo(`Multipart upload created for streaming write`).pipe(r.Effect.annotateLogs({upload_id:t,s3_upload_id:p,s3_key:l,part_size:d}));let h=yield*r.Ref.make([]),g=yield*r.Ref.make(0),_=yield*r.Ref.make(1),v=yield*r.Ref.make(new Uint8Array),y=(e,a)=>r.Effect.gen(function*(){if(e.length===0||e.length<u&&!a)return;let o=yield*r.Ref.getAndUpdate(_,e=>e+1);yield*r.Effect.logDebug(`Uploading part from stream`).pipe(r.Effect.annotateLogs({upload_id:t,part_number:o,part_size:e.length,is_final_part:a}));let s=yield*i.uploadPart({bucket:i.bucket,key:l,uploadId:p,partNumber:o,data:e}).pipe(r.Effect.retry({schedule:r.Schedule.exponential(`1 second`,2).pipe(r.Schedule.intersect(r.Schedule.recurs(3))),while:e=>!m(e)}));yield*r.Ref.update(h,e=>[...e,{PartNumber:o,ETag:s}]),yield*(0,n.s3UploadPartsTotal)(r.Effect.succeed(1)),yield*(0,n.s3PartSizeHistogram)(r.Effect.succeed(e.length))});yield*a.stream.pipe(r.Stream.runForEach(e=>r.Effect.gen(function*(){yield*r.Ref.update(g,t=>t+e.length);let t=yield*r.Ref.get(v),n=new Uint8Array(t.length+e.length);n.set(t),n.set(e,t.length);let i=0;for(;n.length-i>=d;)yield*y(n.slice(i,i+d),!1),i+=d;yield*r.Ref.set(v,n.slice(i))})));let x=yield*r.Ref.get(v);x.length>0&&(yield*y(x,!0));let S=yield*r.Ref.get(h),C=yield*r.Ref.get(g);if(S.length===0)return yield*i.abortMultipartUpload({bucket:i.bucket,key:l,uploadId:p}),yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(-1)),yield*r.Effect.fail(new e.UploadistaError({code:`FILE_WRITE_ERROR`,status:400,body:`Cannot complete upload with no data`,details:`The stream provided no data to upload`}));S.sort((e,t)=>(e.PartNumber??0)-(t.PartNumber??0)),yield*i.completeMultipartUpload({bucket:i.bucket,key:l,uploadId:p},S);let w=Date.now()-c,T=w>0?C*1e3/w:0,E=S.length>0?C/S.length:void 0;return yield*(0,n.logS3UploadCompletion)(t,{fileSize:C,totalDurationMs:w,partsCount:S.length,averagePartSize:E,throughputBps:T}),yield*(0,n.s3UploadSuccessTotal)(r.Effect.succeed(1)),yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(-1)),yield*(0,n.s3FileSizeHistogram)(r.Effect.succeed(C)),yield*r.Effect.logInfo(`Streaming write to S3 completed`).pipe(r.Effect.annotateLogs({upload_id:t,total_bytes:C,parts_count:S.length,duration_ms:w})),{id:l,size:C,path:l,bucket:i.bucket,url:`${s}/${l}`}}).pipe(r.Effect.catchAll(e=>r.Effect.gen(function*(){return yield*(0,n.s3UploadErrorsTotal)(r.Effect.succeed(1)),yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(-1)),yield*r.Effect.fail(e)})))),deleteExpired:B,getCapabilities:W,getChunkerConstraints:G,validateUploadStrategy:K}})}const x=e=>{let{s3ClientConfig:{bucket:t,...n}}=e;return b(e).pipe(r.Effect.provide(v(n,t)))};exports.createS3Store=b,exports.s3Store=x;
+Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});let e=require(`@uploadista/core/errors`),t=require(`@uploadista/core/types`),n=require(`@uploadista/observability`),r=require(`effect`),i=require(`@aws-sdk/client-s3`);const a=e=>e&&e.length>0?e.reduce((e,t)=>e+(t?.Size??0),0):0,o=(e,t,n,r,i=5497558138880)=>{let a=e??i,o;o=a<=t?a:a<=t*r?t:Math.ceil(a/r);let s=e&&e<n?o:Math.max(o,n),c=1024;return Math.ceil(s/c)*c},s=e=>`${e}.part`,c=(e,t)=>{let n=new Date(e);return new Date(n.getTime()+t)},l=(t,i,a={})=>(r.Effect.runSync((0,n.trackS3Error)(t,i,a)),e.UploadistaError.fromCode(`FILE_WRITE_ERROR`,i)),u=e=>[`NotFound`,`NoSuchKey`,`NoSuchUpload`].includes(e),d=e=>{if(typeof e!=`object`||!e)return null;let t={};return`code`in e&&typeof e.code==`string`&&(t.code=e.code),`name`in e&&typeof e.name==`string`&&(t.name=e.name),Object.keys(t).length>0?t:null},f=(t,n,i={})=>{let a=d(n);return a&&(a.code&&u(a.code)||a.name&&u(a.name))?(r.Effect.runSync(r.Effect.logWarning(`File not found during ${t} operation`).pipe(r.Effect.annotateLogs({error_code:a.code,error_name:a.name,...i}))),e.UploadistaError.fromCode(`FILE_NOT_FOUND`)):l(t,n,i)},p=e=>e===`NoSuchUpload`||e===`NoSuchKey`,m=t=>{let n=d(t);if(n&&(n.code&&p(n.code)||n.name&&p(n.name)))return!0;if(t instanceof e.UploadistaError&&t.cause){let e=d(t.cause);if(e&&(e.code&&p(e.code)||e.name&&p(e.name)))return!0}return!1};function h(e){if(e instanceof ReadableStream||e&&typeof e==`object`&&`getReader`in e)return e;if(e&&typeof e==`object`&&`pipe`in e&&`on`in e){let t=e;return new ReadableStream({start(e){t.on(`data`,t=>{e.enqueue(new Uint8Array(t))}),t.on(`end`,()=>{e.close()}),t.on(`error`,t=>{e.error(t)})}})}throw Error(`Unsupported body type: ${typeof e}. Expected ReadableStream or Node.js Readable.`)}var g=class extends r.Context.Tag(`S3ClientService`)(){};const _=(e,t)=>{let a=new i.S3(e),o=e=>r.Effect.tryPromise({try:async()=>h((await a.getObject({Bucket:t,Key:e})).Body),catch:n=>l(`getObject`,n,{key:e,bucket:t})}),c=e=>r.Effect.tryPromise({try:async()=>{try{return(await a.headObject({Bucket:t,Key:e})).ContentLength}catch(e){if(e instanceof i.NotFound)return;throw e}},catch:n=>l(`headObject`,n,{key:e,bucket:t})}),u=(e,n)=>r.Effect.tryPromise({try:async()=>(await a.putObject({Bucket:t,Key:e,Body:n})).ETag||``,catch:r=>l(`putObject`,r,{key:e,bucket:t,size:n.length})}),d=e=>r.Effect.tryPromise({try:async()=>{await a.deleteObject({Bucket:t,Key:e})},catch:n=>l(`deleteObject`,n,{key:e,bucket:t})});return{bucket:t,getObject:o,headObject:c,putObject:u,deleteObject:d,deleteObjects:e=>r.Effect.tryPromise({try:()=>a.deleteObjects({Bucket:t,Delete:{Objects:e.map(e=>({Key:e}))}}),catch:n=>l(`deleteObjects`,n,{keys:e.length,bucket:t})}),createMultipartUpload:e=>(0,n.withS3ApiMetrics)(`createMultipartUpload`,r.Effect.tryPromise({try:async()=>{let t={Bucket:e.bucket,Key:e.key};e.contentType&&(t.ContentType=e.contentType),e.cacheControl&&(t.CacheControl=e.cacheControl);let n=await a.createMultipartUpload(t);if(!n.UploadId)throw Error(`Upload ID is undefined`);if(!n.Key)throw Error(`Key is undefined`);return{uploadId:n.UploadId,bucket:e.bucket,key:n.Key}},catch:t=>l(`createMultipartUpload`,t,e)})),uploadPart:e=>(0,n.withS3ApiMetrics)(`uploadPart`,r.Effect.tryPromise({try:()=>a.uploadPart({Bucket:e.bucket,Key:e.key,UploadId:e.uploadId,PartNumber:e.partNumber,Body:e.data}),catch:t=>l(`uploadPart`,t,{upload_id:e.key,part_number:e.partNumber,part_size:e.data.length,s3_bucket:e.bucket})}).pipe(r.Effect.map(e=>e.ETag))),completeMultipartUpload:(e,t)=>(0,n.withS3ApiMetrics)(`completeMultipartUpload`,r.Effect.tryPromise({try:()=>a.completeMultipartUpload({Bucket:e.bucket,Key:e.key,UploadId:e.uploadId,MultipartUpload:{Parts:t.map(e=>({ETag:e.ETag,PartNumber:e.PartNumber}))}}).then(e=>e.Location),catch:n=>l(`completeMultipartUpload`,n,{upload_id:e.key,parts_count:t.length,s3_bucket:e.bucket})})),abortMultipartUpload:e=>r.Effect.tryPromise({try:async()=>{await a.abortMultipartUpload({Bucket:e.bucket,Key:e.key,UploadId:e.uploadId})},catch:t=>f(`abortMultipartUpload`,t,{upload_id:e.key,s3_bucket:e.bucket})}),listParts:e=>r.Effect.tryPromise({try:async()=>{let t={Bucket:e.bucket,Key:e.key,UploadId:e.uploadId,PartNumberMarker:e.partNumberMarker},n=await a.listParts(t);return{parts:n.Parts??[],isTruncated:n.IsTruncated??!1,nextPartNumberMarker:n.NextPartNumberMarker}},catch:t=>l(`listParts`,t,{upload_id:e.key,s3_bucket:e.bucket})}),listMultipartUploads:(e,n)=>r.Effect.tryPromise({try:()=>a.listMultipartUploads({Bucket:t,KeyMarker:e,UploadIdMarker:n}),catch:e=>l(`listMultipartUploads`,e,{bucket:t})}),getIncompletePart:e=>r.Effect.tryPromise({try:async()=>{try{return h((await a.getObject({Bucket:t,Key:s(e)})).Body)}catch(e){if(e instanceof i.NoSuchKey)return;throw e}},catch:n=>l(`getIncompletePart`,n,{upload_id:e,bucket:t})}),getIncompletePartSize:e=>c(s(e)),putIncompletePart:(e,t)=>u(s(e),t).pipe(r.Effect.tap(()=>r.Effect.logInfo(`Incomplete part uploaded`).pipe(r.Effect.annotateLogs({upload_id:e})))),deleteIncompletePart:e=>d(s(e))}},v=(e,t)=>r.Layer.succeed(g,_(e,t)),y=e=>{let{id:t,metadata:n}=e;if(!n)return t;let r=n.filename||n.fileName||n.name;return typeof r==`string`&&r.includes(`.`)?`${t}${r.substring(r.lastIndexOf(`.`))}`:t};function b(i){let{deliveryUrl:s,partSize:l,minPartSize:u=5242880,useTags:d=!0,maxMultipartParts:f=1e4,maxConcurrentPartUploads:p=60,expirationPeriodInMilliseconds:h=1e3*60*60*24*7,s3ClientConfig:{bucket:_}}=i;return r.Effect.gen(function*(){let i=yield*g,v=yield*t.UploadFileKVStore,b=l||8*1024*1024,x=t=>{let n=t.storage.uploadId;return n?r.Effect.succeed(n):r.Effect.fail(e.UploadistaError.fromCode(`FILE_WRITE_ERROR`,Error(`Upload ID is undefined`)))},S=(t,a,o)=>{let s=y(t);return(0,n.withS3TimingMetrics)(n.s3PartUploadDurationHistogram,r.Effect.gen(function*(){let c=yield*x(t),l=yield*i.uploadPart({bucket:i.bucket,key:s,uploadId:c,partNumber:o,data:a}).pipe(r.Effect.retry({schedule:r.Schedule.exponential(`1 second`,2).pipe(r.Schedule.intersect(r.Schedule.recurs(3))),while:e=>!m(e)}),r.Effect.tapError(e=>r.Effect.logWarning(`Retrying part upload`).pipe(r.Effect.annotateLogs({upload_id:t.id,part_number:o,error_message:e.message,retry_attempt:`unknown`,part_size:a.length,s3_bucket:i.bucket}))),r.Effect.catchAll(n=>m(n)?r.Effect.fail(e.UploadistaError.fromCode(`UPLOAD_CANCELLED`,{cause:n,body:`Upload ${t.id} was cancelled`})):r.Effect.fail(n)));return yield*(0,n.s3UploadPartsTotal)(r.Effect.succeed(1)),yield*r.Effect.logInfo(`Part uploaded successfully`).pipe(r.Effect.annotateLogs({upload_id:t.id,part_number:o,part_size:a.length,etag:l})),l})).pipe(r.Effect.withSpan(`s3-upload-part-${o}`,{attributes:{"upload.id":t.id,"upload.part_number":o,"upload.part_size":a.length,"s3.bucket":i.bucket,"s3.key":s}}))},C=(e,t)=>i.putIncompletePart(e,t),w=e=>r.Effect.gen(function*(){let t=yield*i.getIncompletePart(e);if(!t)return;let n=t.getReader(),a=[],o=0;try{for(;;){let{done:e,value:t}=yield*r.Effect.promise(()=>n.read());if(e)break;a.push(t),o+=t.length}}finally{n.releaseLock()}let s=r.Stream.fromIterable(a);return{size:o,stream:s}}),T=e=>i.deleteIncompletePart(e),E=e=>i.getIncompletePartSize(e),D=(e,t)=>{let a=y(e);return r.Effect.gen(function*(){let n=yield*x(e);return yield*i.completeMultipartUpload({bucket:i.bucket,key:a,uploadId:n},t)}).pipe(r.Effect.tap(()=>(0,n.s3UploadSuccessTotal)(r.Effect.succeed(1))),r.Effect.withSpan(`s3-complete-multipart-upload`,{attributes:{"upload.id":e.id,"upload.parts_count":t.length,"s3.bucket":i.bucket,"s3.key":a}}))},O=e=>{let t=y(e);return r.Effect.gen(function*(){let n=yield*x(e);yield*i.abortMultipartUpload({bucket:i.bucket,key:t,uploadId:n}),yield*i.deleteObjects([t])})},k=(e,t,n,a)=>r.Effect.gen(function*(){let r=yield*i.listParts({bucket:i.bucket,key:e,uploadId:t,partNumberMarker:a}),o=r.parts;if(r.isTruncated){let i=yield*k(e,t,n,r.nextPartNumberMarker);o=[...o,...i.parts]}return a||o.sort((e,t)=>(e.PartNumber??0)-(t.PartNumber??0)),{uploadFound:!0,parts:o}}).pipe(r.Effect.catchAll(e=>m(e)?r.Effect.logWarning(`S3 upload not found during listParts`).pipe(r.Effect.annotateLogs({upload_id:n,error_code:e.code}),r.Effect.as({uploadFound:!1,parts:[]})):r.Effect.fail(e))),A=(e,t)=>r.Effect.gen(function*(){let n=yield*v.get(e),r=yield*x(n);return yield*k(y(n),r,e,t)}),j=(e,t)=>r.Effect.gen(function*(){if(!t)return 0;let n=yield*v.get(e.id),r=n.storage.uploadId;return r&&(yield*v.set(e.id,{...n,storage:{...n.storage,uploadId:r}})),0}),M=e=>r.Effect.gen(function*(){yield*r.Effect.logInfo(`Clearing cache`).pipe(r.Effect.annotateLogs({upload_id:e})),yield*v.delete(e)}),N=e=>{let t=y(e);return r.Effect.gen(function*(){yield*r.Effect.logInfo(`Initializing multipart upload`).pipe(r.Effect.annotateLogs({upload_id:e.id}));let a=yield*i.createMultipartUpload({bucket:i.bucket,key:t,uploadId:``,contentType:e.metadata?.contentType?.toString(),cacheControl:e.metadata?.cacheControl?.toString()}),o={...e,storage:{...e.storage,path:a.key,uploadId:a.uploadId,bucket:a.bucket},url:`${s}/${t}`};return yield*v.set(e.id,o),yield*r.Effect.logInfo(`Multipart upload created`).pipe(r.Effect.annotateLogs({upload_id:e.id,s3_upload_id:o.storage.uploadId,s3_key:t})),yield*(0,n.s3UploadRequestsTotal)(r.Effect.succeed(1)),yield*(0,n.s3FileSizeHistogram)(r.Effect.succeed(e.size||0)),o}).pipe(r.Effect.withSpan(`s3-create-upload`,{attributes:{"upload.id":e.id,"upload.size":e.size||0,"s3.bucket":i.bucket,"s3.key":t}}))},P=e=>r.Effect.gen(function*(){yield*r.Effect.logInfo(`Initializing multipart upload`).pipe(r.Effect.annotateLogs({upload_id:e.id}));let t=yield*N(e);return yield*v.set(e.id,t),yield*r.Effect.logInfo(`Multipart upload created`).pipe(r.Effect.annotateLogs({upload_id:e.id,s3_upload_id:t.storage.uploadId})),yield*(0,n.s3UploadRequestsTotal)(r.Effect.succeed(1)),t}).pipe(r.Effect.withSpan(`s3-create-upload`,{attributes:{"upload.id":e.id,"upload.size":e.size||0,"s3.bucket":_}})),F=e=>r.Effect.gen(function*(){yield*O(yield*v.get(e)),yield*M(e)}),I=(e,t)=>(0,n.withS3UploadMetrics)(e.file_id,(0,n.withS3TimingMetrics)(n.s3UploadDurationHistogram,r.Effect.gen(function*(){let{stream:i,file_id:a,offset:s}=e,{onProgress:c}=t,l=Date.now();yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(1));let{uploadFile:d,nextPartNumber:m,offset:h,data:g,existingPartSize:_}=yield*R(a,s,i),v=_||o(d.size,b,u,f);yield*r.Effect.logInfo(`Part size decision`).pipe(r.Effect.annotateLogs({upload_id:a,existing_part_size:_,calculated_part_size:o(d.size,b,u,f),final_part_size:v,next_part_number:m}));let y=h+(yield*U(d,g,m,h,v,u,p,c));return d.size===y&&(yield*z(a,d,l)),y}).pipe(r.Effect.ensuring((0,n.s3ActiveUploadsGauge)(r.Effect.succeed(0)))))),L=e=>r.Effect.gen(function*(){let t=yield*v.get(e),{parts:n,uploadFound:r}=yield*A(e);if(!r)return{...t,offset:t.size,size:t.size};let i=a(n),o=yield*E(e);return{...t,offset:i+(o??0),size:t.size,storage:t.storage}}),R=(e,t,n)=>r.Effect.gen(function*(){let i=yield*v.get(e),{parts:a}=yield*A(e),o=(a.length>0&&a[a.length-1].PartNumber?a[a.length-1].PartNumber??0:0)+1,s=a.length>0&&a[0].Size?a[0].Size:null;if(s&&a.length>1){let t=a.slice(0,-1).find(e=>e.Size!==s);t&&(yield*r.Effect.logWarning(`Inconsistent part sizes detected in existing upload`).pipe(r.Effect.annotateLogs({upload_id:e,expected_size:s,inconsistent_part:t.PartNumber,inconsistent_size:t.Size})))}let c=yield*w(e);if(c){yield*T(e);let a=t-c.size,l=c.stream.pipe(r.Stream.concat(n));return{uploadFile:i,nextPartNumber:o,offset:a,incompletePartSize:c.size,data:l,existingPartSize:s}}else return{uploadFile:i,nextPartNumber:o,offset:t,incompletePartSize:0,data:n,existingPartSize:s}}),z=(e,t,i)=>r.Effect.gen(function*(){let{parts:a}=yield*A(e);yield*r.Effect.logInfo(`Attempting to complete multipart upload`).pipe(r.Effect.annotateLogs({upload_id:e,parts_count:a.length,parts_info:a.map((e,t)=>({part_number:e.PartNumber,size:e.Size,etag:e.ETag,is_final_part:t===a.length-1}))})),yield*D(t,a),yield*j(t,d);let o=Date.now()-i,s=t.size||0,c=o>0?s*1e3/o:0,l=a.length>0?a.reduce((e,t)=>e+(t.Size||0),0)/a.length:void 0;yield*(0,n.logS3UploadCompletion)(e,{fileSize:s,totalDurationMs:o,partsCount:a.length,averagePartSize:l,throughputBps:c})}).pipe(r.Effect.tapError(t=>r.Effect.gen(function*(){yield*(0,n.s3UploadErrorsTotal)(r.Effect.succeed(1)),yield*r.Effect.logError(`Failed to finish upload`).pipe(r.Effect.annotateLogs({upload_id:e,error:String(t)}))}))),B=()=>r.Effect.gen(function*(){if(h===0)return 0;let e,t,n=!0,a=0;for(;n;){let o=yield*i.listMultipartUploads(e,t),s=o.Uploads?.filter(e=>{let t=e.Initiated;return t&&Date.now()>c(t.toISOString(),h).getTime()})||[],l=s.filter(e=>!!e.Key).map(e=>e.Key);l.length>0&&(yield*i.deleteObjects(l),yield*r.Effect.forEach(s,e=>r.Effect.gen(function*(){!e.Key||!e.UploadId||(yield*i.abortMultipartUpload({bucket:_,key:e.Key,uploadId:e.UploadId}))})),a+=l.length),n=o.IsTruncated??!1,n&&(e=o.NextKeyMarker,t=o.NextUploadIdMarker)}return a}),V=e=>t=>r.Stream.async(n=>{let i=new Uint8Array,a=1,o=0,s=(t,i=!1)=>{r.Effect.runSync(r.Effect.logInfo(`Creating chunk`).pipe(r.Effect.annotateLogs({part_number:a,chunk_size:t.length,expected_size:e,is_final_chunk:i,total_bytes_processed:o+t.length}))),n.single({partNumber:a++,data:t,size:t.length})},c=t=>{let n=new Uint8Array(i.length+t.length);for(n.set(i),n.set(t,i.length),i=n,o+=t.length;i.length>=e;){let t=i.slice(0,e);i=i.slice(e),s(t,!1)}};r.Effect.runFork(t.pipe(r.Stream.runForEach(e=>r.Effect.sync(()=>c(e))),r.Effect.andThen(()=>r.Effect.sync(()=>{i.length>0&&s(i,!0),n.end()})),r.Effect.catchAll(e=>r.Effect.sync(()=>n.fail(e)))))}),H=(e,t=0)=>n=>e?r.Effect.gen(function*(){let i=yield*r.Ref.make(t);return n.pipe(r.Stream.tap(t=>r.Effect.gen(function*(){e(yield*r.Ref.updateAndGet(i,e=>e+t.length))})))}).pipe(r.Stream.unwrap):n,U=(t,i,a,o,s,c,l,u)=>r.Effect.gen(function*(){yield*r.Effect.logInfo(`Starting part uploads`).pipe(r.Effect.annotateLogs({upload_id:t.id,init_offset:o,file_size:t.size,part_size:s,min_part_size:c}));let d=i.pipe(H(u,o),V(s)),f=yield*r.Ref.make(o),p=yield*r.Ref.make(0),m=i=>r.Effect.gen(function*(){let o=yield*r.Ref.updateAndGet(f,e=>e+i.size),l=o>=(t.size||0);yield*r.Effect.logDebug(`Processing chunk`).pipe(r.Effect.annotateLogs({upload_id:t.id,cumulative_offset:o,file_size:t.size,chunk_size:i.size,is_final_part:l}));let u=a+i.partNumber-1;i.size>s&&(yield*r.Effect.fail(e.UploadistaError.fromCode(`FILE_WRITE_ERROR`,Error(`Part size ${i.size} exceeds upload part size ${s}`)))),i.size>=c||l?(yield*r.Effect.logDebug(`Uploading multipart chunk`).pipe(r.Effect.annotateLogs({upload_id:t.id,part_number:u,chunk_size:i.size,min_part_size:c,is_final_part:l})),yield*S(t,i.data,u),yield*(0,n.s3PartSizeHistogram)(r.Effect.succeed(i.size))):yield*C(t.id,i.data),yield*r.Ref.update(p,e=>e+i.size)});return yield*d.pipe(r.Stream.runForEach(e=>m(e)),r.Effect.withConcurrency(l)),yield*r.Ref.get(p)}),W=()=>({supportsParallelUploads:!0,supportsConcatenation:!0,supportsDeferredLength:!0,supportsResumableUploads:!0,supportsTransactionalUploads:!0,supportsStreamingRead:!0,supportsStreamingWrite:!0,maxConcurrentUploads:p,minChunkSize:u,maxChunkSize:5368709120,maxParts:f,optimalChunkSize:b,requiresOrderedChunks:!1,requiresMimeTypeValidation:!0,maxValidationSize:void 0}),G=()=>({minChunkSize:u,maxChunkSize:5368709120,optimalChunkSize:b,requiresOrderedChunks:!1}),K=e=>{let t=W(),n=(()=>{switch(e){case`parallel`:return t.supportsParallelUploads;case`single`:return!0;default:return!1}})();return r.Effect.succeed(n)},q=e=>{let t=new Uint8Array(e.reduce((e,t)=>e+t.length,0)),n=0;for(let r of e)t.set(r,n),n+=r.length;return t},J=async e=>{let t=e.getReader(),n=[];for(;;){let{done:e,value:r}=await t.read();if(e)break;n.push(r)}return q(n)};return{bucket:_,create:P,remove:F,write:I,getUpload:L,read:t=>r.Effect.gen(function*(){let n=yield*v.get(t);if(console.log(n),!n.id)return yield*r.Effect.fail(e.UploadistaError.fromCode(`FILE_READ_ERROR`,Error(`Upload Key is undefined`)));let a=y(n),o=yield*i.getObject(a);return yield*r.Effect.promise(()=>J(o))}),readStream:(n,a)=>r.Effect.gen(function*(){let o=yield*v.get(n);if(!o.id)return yield*r.Effect.fail(e.UploadistaError.fromCode(`FILE_READ_ERROR`,Error(`Upload Key is undefined`)));let s={...t.DEFAULT_STREAMING_CONFIG,...a},c=y(o),l=yield*i.getObject(c);return r.Stream.async(t=>{let n=l.getReader(),i=s.chunkSize,a=new Uint8Array;return(async()=>{try{for(;;){let{done:e,value:r}=await n.read();if(e){a.length>0&&t.single(a),t.end();return}if(r){let e=new Uint8Array(a.length+r.length);for(e.set(a),e.set(r,a.length),a=e;a.length>=i;){let e=a.slice(0,i);a=a.slice(i),t.single(e)}}}}catch(n){t.fail(new e.UploadistaError({code:`FILE_READ_ERROR`,status:500,body:`Failed to read S3 object stream`,details:`S3 stream read failed: ${String(n)}`}))}})(),r.Effect.sync(()=>{n.releaseLock()})})}),writeStream:(t,a)=>(0,n.withS3TimingMetrics)(n.s3UploadDurationHistogram,r.Effect.gen(function*(){let c=Date.now(),l=t;yield*r.Effect.logInfo(`Starting streaming write to S3`).pipe(r.Effect.annotateLogs({upload_id:t,s3_key:l,size_hint:a.sizeHint})),yield*(0,n.s3UploadRequestsTotal)(r.Effect.succeed(1)),yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(1));let d=a.sizeHint?o(a.sizeHint,b,u,f):b,p=(yield*i.createMultipartUpload({bucket:i.bucket,key:l,uploadId:``,contentType:a.contentType})).uploadId;yield*r.Effect.logInfo(`Multipart upload created for streaming write`).pipe(r.Effect.annotateLogs({upload_id:t,s3_upload_id:p,s3_key:l,part_size:d}));let h=yield*r.Ref.make([]),g=yield*r.Ref.make(0),_=yield*r.Ref.make(1),v=yield*r.Ref.make(new Uint8Array),y=(e,a)=>r.Effect.gen(function*(){if(e.length===0||e.length<u&&!a)return;let o=yield*r.Ref.getAndUpdate(_,e=>e+1);yield*r.Effect.logDebug(`Uploading part from stream`).pipe(r.Effect.annotateLogs({upload_id:t,part_number:o,part_size:e.length,is_final_part:a}));let s=yield*i.uploadPart({bucket:i.bucket,key:l,uploadId:p,partNumber:o,data:e}).pipe(r.Effect.retry({schedule:r.Schedule.exponential(`1 second`,2).pipe(r.Schedule.intersect(r.Schedule.recurs(3))),while:e=>!m(e)}));yield*r.Ref.update(h,e=>[...e,{PartNumber:o,ETag:s}]),yield*(0,n.s3UploadPartsTotal)(r.Effect.succeed(1)),yield*(0,n.s3PartSizeHistogram)(r.Effect.succeed(e.length))});yield*a.stream.pipe(r.Stream.runForEach(e=>r.Effect.gen(function*(){yield*r.Ref.update(g,t=>t+e.length);let t=yield*r.Ref.get(v),n=new Uint8Array(t.length+e.length);n.set(t),n.set(e,t.length);let i=0;for(;n.length-i>=d;)yield*y(n.slice(i,i+d),!1),i+=d;yield*r.Ref.set(v,n.slice(i))})));let x=yield*r.Ref.get(v);x.length>0&&(yield*y(x,!0));let S=yield*r.Ref.get(h),C=yield*r.Ref.get(g);if(S.length===0)return yield*i.abortMultipartUpload({bucket:i.bucket,key:l,uploadId:p}),yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(-1)),yield*r.Effect.fail(new e.UploadistaError({code:`FILE_WRITE_ERROR`,status:400,body:`Cannot complete upload with no data`,details:`The stream provided no data to upload`}));S.sort((e,t)=>(e.PartNumber??0)-(t.PartNumber??0)),yield*i.completeMultipartUpload({bucket:i.bucket,key:l,uploadId:p},S);let w=Date.now()-c,T=w>0?C*1e3/w:0,E=S.length>0?C/S.length:void 0;return yield*(0,n.logS3UploadCompletion)(t,{fileSize:C,totalDurationMs:w,partsCount:S.length,averagePartSize:E,throughputBps:T}),yield*(0,n.s3UploadSuccessTotal)(r.Effect.succeed(1)),yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(-1)),yield*(0,n.s3FileSizeHistogram)(r.Effect.succeed(C)),yield*r.Effect.logInfo(`Streaming write to S3 completed`).pipe(r.Effect.annotateLogs({upload_id:t,total_bytes:C,parts_count:S.length,duration_ms:w})),{id:l,size:C,path:l,bucket:i.bucket,url:`${s}/${l}`}}).pipe(r.Effect.catchAll(e=>r.Effect.gen(function*(){return yield*(0,n.s3UploadErrorsTotal)(r.Effect.succeed(1)),yield*(0,n.s3ActiveUploadsGauge)(r.Effect.succeed(-1)),yield*r.Effect.fail(e)})))),deleteExpired:B,getCapabilities:W,getChunkerConstraints:G,validateUploadStrategy:K}})}const x=e=>{let{s3ClientConfig:{bucket:t,...n}}=e;return b(e).pipe(r.Effect.provide(v(n,t)))};exports.createS3Store=b,exports.s3Store=x;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@uploadista/data-store-s3",
   "type": "module",
-  "version": "0.1.4-beta.1",
+  "version": "1.0.0-beta.2",
   "description": "AWS S3 data store for Uploadista",
   "license": "MIT",
   "author": "Uploadista",
@@ -14,20 +14,20 @@
     }
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "3.980.0",
-    "@uploadista/core": "0.1.4-beta.1",
-    "@uploadista/observability": "0.1.4-beta.1"
+    "@aws-sdk/client-s3": "3.1000.0",
+    "@uploadista/core": "1.0.0-beta.2",
+    "@uploadista/observability": "1.0.0-beta.2"
   },
   "peerDependencies": {
     "effect": "^3.0.0"
   },
   "devDependencies": {
     "@effect/vitest": "0.27.0",
-    "effect": "3.19.15",
-    "tsdown": "0.20.1",
+    "effect": "3.19.19",
+    "tsdown": "0.20.3",
     "vitest": "4.0.18",
-    "@uploadista/typescript-config": "0.1.4-beta.1",
-    "@uploadista/kv-store-memory": "0.1.4-beta.1"
+    "@uploadista/kv-store-memory": "1.0.0-beta.2",
+    "@uploadista/typescript-config": "1.0.0-beta.2"
   },
   "scripts": {
     "build": "tsc --noEmit && tsdown",