@uploadista/data-store-azure 0.1.4-beta.1 → 0.2.0

package/dist/index.cjs

@@ -1 +1 @@
-let e=require(`@azure/storage-blob`),t=require(`@uploadista/core/errors`),n=require(`@uploadista/core/types`),r=require(`@uploadista/observability`),i=require(`effect`);const a=e=>typeof globalThis<`u`&&`Buffer`in globalThis?globalThis.Buffer.from(e):new Uint8Array(Array.from(e,e=>e.charCodeAt(0)));function o(e){return e&&e.length>0?e.reduce((e,t)=>e+(t?.size??0),0):0}function s({deliveryUrl:s,blockSize:c,minBlockSize:l=1024,maxBlocks:u=5e4,maxConcurrentBlockUploads:d=60,expirationPeriodInMilliseconds:f=1e3*60*60*24*7,connectionString:p,sasUrl:m,credential:h,accountName:g,accountKey:_,containerName:v}){return i.Effect.gen(function*(){let y=yield*n.UploadFileKVStore,b=c||8*1024*1024,x;if(p)x=e.BlobServiceClient.fromConnectionString(p);else if(m)x=new e.BlobServiceClient(m);else if(h){let t=g?`https://${g}.blob.core.windows.net`:m?.split(`?`)[0]||``;if(!t)throw Error(`When using credential authentication, either accountName or a valid sasUrl must be provided to determine the account URL`);x=new e.BlobServiceClient(t,h)}else if(g&&_)try{let t=new e.StorageSharedKeyCredential(g,_);x=new e.BlobServiceClient(`https://${g}.blob.core.windows.net`,t)}catch(e){throw Error(`StorageSharedKeyCredential is only available in Node.js environments. Use sasUrl or credential options for cross-platform compatibility. Original error: ${e}`)}else throw Error(`Azure authentication required. Provide one of: connectionString, sasUrl, credential, or accountName + accountKey (Node.js only)`);let S=x.getContainerClient(v),C=e=>`${e}.incomplete`,w=(e,n,a)=>(0,r.withAzureTimingMetrics)(r.azurePartUploadDurationHistogram,i.Effect.gen(function*(){yield*i.Effect.logInfo(`Uploading block`).pipe(i.Effect.annotateLogs({upload_id:e.id,block_id:a,block_size:n.length})),yield*(0,r.azureUploadPartsTotal)(i.Effect.succeed(1)),yield*(0,r.azurePartSizeHistogram)(i.Effect.succeed(n.length));try{let o=S.getBlockBlobClient(e.id);yield*i.Effect.tryPromise({try:async()=>{await o.stageBlock(a,n,n.length)},catch:o=>(i.Effect.runSync((0,r.trackAzureError)(`uploadBlock`,o,{upload_id:e.id,block_id:a,block_size:n.length})),t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:o}))}),yield*i.Effect.logInfo(`Finished uploading block`).pipe(i.Effect.annotateLogs({upload_id:e.id,block_id:a,block_size:n.length}))}catch(t){throw i.Effect.runSync((0,r.trackAzureError)(`uploadBlock`,t,{upload_id:e.id,block_id:a,block_size:n.length})),t}})),T=(e,n)=>i.Effect.tryPromise({try:async()=>{await S.getBlockBlobClient(C(e)).upload(n,n.length)},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}).pipe(i.Effect.tap(()=>i.Effect.logInfo(`Finished uploading incomplete block`).pipe(i.Effect.annotateLogs({upload_id:e})))),E=e=>i.Effect.tryPromise({try:async()=>{try{return(await S.getBlockBlobClient(C(e)).download()).readableStreamBody}catch(e){if(e&&typeof e==`object`&&`statusCode`in e&&e.statusCode===404)return;throw e}},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),D=e=>i.Effect.tryPromise({try:async()=>{try{return(await S.getBlockBlobClient(C(e)).getProperties()).contentLength}catch(e){if(e&&typeof e==`object`&&`statusCode`in e&&e.statusCode===404)return;throw e}},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),O=e=>i.Effect.tryPromise({try:async()=>{await S.getBlockBlobClient(C(e)).deleteIfExists()},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),k=e=>i.Effect.gen(function*(){let t=yield*E(e);if(!t)return;let n=t.getReader(),r=[],a=0;try{for(;;){let e=yield*i.Effect.promise(()=>n.read());if(e.done)break;r.push(e.value),a+=e.value.length}}finally{n.releaseLock()}let o=i.Stream.fromIterable(r);return{size:a,stream:o}}),A=e=>{let t=e??5497558138880,n;n=t<=b?t:t<=b*u?b:Math.ceil(t/u);let r=Math.max(n,l);return Math.ceil(r/1024)*1024},j=e=>t=>i.Stream.async(n=>{let r=new Uint8Array,a=1,o=0,s=(t,r=!1)=>{i.Effect.runSync(i.Effect.logInfo(`Creating chunk`).pipe(i.Effect.annotateLogs({block_number:a,chunk_size:t.length,expected_size:e,is_final_chunk:r,total_bytes_processed:o+t.length}))),n.single({blockNumber:a++,data:t,size:t.length})},c=t=>{let n=new Uint8Array(r.length+t.length);for(n.set(r),n.set(t,r.length),r=n,o+=t.length;r.length>=e;){let t=r.slice(0,e);r=r.slice(e),s(t,!1)}};i.Effect.runFork(t.pipe(i.Stream.runForEach(e=>i.Effect.sync(()=>c(e))),i.Effect.andThen(()=>i.Effect.sync(()=>{r.length>0&&s(r,!0),n.end()})),i.Effect.catchAll(e=>i.Effect.sync(()=>n.fail(e)))))}),M=(e,t=0)=>n=>e?i.Effect.gen(function*(){let r=yield*i.Ref.make(t);return n.pipe(i.Stream.tap(t=>i.Effect.gen(function*(){e(yield*i.Ref.updateAndGet(r,e=>e+t.length))})))}).pipe(i.Stream.unwrap):n,N=(e,n,o,s,c)=>i.Effect.gen(function*(){yield*i.Effect.logInfo(`Uploading blocks`).pipe(i.Effect.annotateLogs({upload_id:e.id,init_offset:s,file_size:e.size}));let u=e.size,f=A(u);yield*i.Effect.logInfo(`Block size`).pipe(i.Effect.annotateLogs({upload_id:e.id,block_size:f}));let p=n.pipe(M(c,s),j(f)),m=yield*i.Ref.make(s),h=yield*i.Ref.make(0),g=yield*i.Ref.make([]),_=n=>i.Effect.gen(function*(){let s=yield*i.Ref.updateAndGet(m,e=>e+n.size),c=s>=(e.size||0);yield*i.Effect.logDebug(`Processing chunk`).pipe(i.Effect.annotateLogs({upload_id:e.id,cumulative_offset:s,file_size:e.size,chunk_size:n.size,is_final_block:c}));let u=o+n.blockNumber-1;if(n.size>f&&(yield*i.Effect.fail(t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:Error(`Block size ${n.size} exceeds upload block size ${f}`)}))),n.size>=l||c){yield*i.Effect.logDebug(`Uploading multipart chunk`).pipe(i.Effect.annotateLogs({upload_id:e.id,block_number:u,chunk_size:n.size,min_block_size:l,is_final_block:c}));let t=a(`block-${u.toString().padStart(6,`0`)}`).toString(`base64`);yield*w(e,n.data,t),yield*i.Ref.update(g,e=>[...e,t]),yield*(0,r.azurePartSizeHistogram)(i.Effect.succeed(n.size))}else yield*T(e.id,n.data);yield*i.Ref.update(h,e=>e+n.size)});return yield*p.pipe(i.Stream.runForEach(e=>_(e)),i.Effect.withConcurrency(d)),{bytesUploaded:yield*i.Ref.get(h),blockIds:yield*i.Ref.get(g)}}),P=(e,n)=>i.Effect.tryPromise({try:async()=>{await S.getBlockBlobClient(e.id).commitBlockList(n,{blobHTTPHeaders:{blobContentType:e.metadata?.contentType?.toString(),blobCacheControl:e.metadata?.cacheControl?.toString()}})},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),F=e=>i.Effect.tryPromise({try:async()=>{try{return(await S.getBlockBlobClient(e).getBlockList(`committed`)).committedBlocks?.map(e=>({size:e.size}))??[]}catch(e){if(e&&typeof e==`object`&&`statusCode`in e&&e.statusCode===404)return[];throw e}},catch:e=>t.UploadistaError.fromCode(`UPLOAD_ID_NOT_FOUND`,{cause:e})}),I=e=>i.Effect.gen(function*(){yield*i.Effect.logInfo(`Removing cached data`).pipe(i.Effect.annotateLogs({upload_id:e})),yield*y.delete(e)}),L=e=>i.Effect.gen(function*(){return yield*(0,r.azureUploadRequestsTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(1)),yield*(0,r.azureFileSizeHistogram)(i.Effect.succeed(e.size||0)),yield*i.Effect.logInfo(`Initializing Azure blob upload`).pipe(i.Effect.annotateLogs({upload_id:e.id})),e.creationDate=new Date().toISOString(),e.storage={id:e.storage.id,type:e.storage.type,path:e.id,bucket:v},e.url=`${s}/${e.id}`,yield*y.set(e.id,e),yield*i.Effect.logInfo(`Azure blob upload initialized`).pipe(i.Effect.annotateLogs({upload_id:e.id})),e}),R=e=>i.Effect.tryPromise({try:async()=>{let t=await S.getBlockBlobClient(e).download();if(t.blobBody)return t.blobBody;if(t.readableStreamBody)return t.readableStreamBody;throw Error(`No blob body or readable stream body`)},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),z=(e,r)=>i.Effect.gen(function*(){let a={...n.DEFAULT_STREAMING_CONFIG,...r},o=yield*R(e);if(o instanceof Blob){let e=yield*i.Effect.promise(()=>o.arrayBuffer()),t=new Uint8Array(e),n=a.chunkSize,r=[];for(let e=0;e<t.length;e+=n)r.push(t.slice(e,Math.min(e+n,t.length)));return i.Stream.fromIterable(r)}return i.Stream.async(e=>{let n=o.getReader(),r=a.chunkSize,s=new Uint8Array;return(async()=>{try{for(;;){let{done:t,value:i}=await n.read();if(t){s.length>0&&e.single(s),e.end();return}if(i){let t=new Uint8Array(s.length+i.length);for(t.set(s),t.set(i,s.length),s=t;s.length>=r;){let t=s.slice(0,r);s=s.slice(r),e.single(t)}}}}catch(n){e.fail(new t.UploadistaError({code:`FILE_READ_ERROR`,status:500,body:`Failed to read Azure blob stream`,details:`Azure stream read failed: ${String(n)}`}))}})(),i.Effect.sync(()=>{n.releaseLock()})})}),B=e=>i.Effect.gen(function*(){let t=yield*z(e),n=[];yield*i.Stream.runForEach(t,e=>i.Effect.sync(()=>{n.push(e)}));let r=n.reduce((e,t)=>e+t.length,0),a=new Uint8Array(r),o=0;for(let e of n)a.set(e,o),o+=e.length;return a}),V=(e,t,n)=>i.Effect.gen(function*(){let r=yield*y.get(e),a=(yield*F(e)).length+1,o=yield*k(e);if(o){yield*O(e);let s=t-o.size,c=o.stream.pipe(i.Stream.concat(n));return{uploadFile:r,nextBlockNumber:a-1,offset:s,incompleteBlockSize:o.size,data:c}}else return{uploadFile:r,nextBlockNumber:a,offset:t,incompleteBlockSize:0,data:n}}),H=(e,t)=>(0,r.withAzureUploadMetrics)(e.file_id,(0,r.withAzureTimingMetrics)(r.azureUploadDurationHistogram,i.Effect.gen(function*(){let n=Date.now(),{stream:a,file_id:o,offset:s}=e,{onProgress:c}=t,{uploadFile:l,nextBlockNumber:u,offset:d,data:f}=yield*V(o,s,a),{bytesUploaded:p,blockIds:m}=yield*N(l,f,u,d,c),h=d+p;if(l.size===h)try{yield*P(l,m),yield*y.set(o,{...l,offset:h}),yield*(0,r.logAzureUploadCompletion)(o,{fileSize:l.size||0,totalDurationMs:Date.now()-n,partsCount:m.length,averagePartSize:l.size,throughputBps:l.size/(Date.now()-n),retryCount:0}),yield*(0,r.azureUploadSuccessTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1))}catch(e){throw yield*i.Effect.logError(`Failed to finish upload`).pipe(i.Effect.annotateLogs({upload_id:o,error:JSON.stringify(e)})),yield*(0,r.azureUploadErrorsTotal)(i.Effect.succeed(1)),i.Effect.runSync((0,r.trackAzureError)(`write`,e,{upload_id:o,operation:`commit`,blocks:m.length})),e}return h}))),U=e=>i.Effect.gen(function*(){let t=yield*y.get(e),n=0;try{n=o(yield*F(e))}catch(n){if(typeof n==`object`&&n&&`statusCode`in n&&n.statusCode===404)return{...t,offset:t.size,size:t.size,metadata:t.metadata,storage:t.storage};throw yield*i.Effect.logError(`Error on get upload`).pipe(i.Effect.annotateLogs({upload_id:e,error:JSON.stringify(n)})),n}let r=yield*D(e);return{...t,offset:n+(r??0),size:t.size,storage:t.storage}}),W=e=>i.Effect.gen(function*(){try{let t=S.getBlockBlobClient(e);yield*i.Effect.promise(()=>t.deleteIfExists()),yield*O(e)}catch(n){if(typeof n==`object`&&n&&`statusCode`in n&&n.statusCode===404)return yield*i.Effect.logError(`No file found`).pipe(i.Effect.annotateLogs({upload_id:e})),yield*i.Effect.fail(t.UploadistaError.fromCode(`FILE_NOT_FOUND`));throw i.Effect.runSync((0,r.trackAzureError)(`remove`,n,{upload_id:e})),n}yield*I(e),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1))}),G=()=>f,K=e=>{let t=new Date(e);return new Date(t.getTime()+G())},q=()=>i.Effect.tryPromise({try:async()=>{if(G()===0)return 0;let e=0,t=S.listBlobsFlat({includeMetadata:!0}),n=[];for await(let e of t)if(e.metadata?.creationDate){let t=new Date(e.metadata.creationDate);Date.now()>K(t.toISOString()).getTime()&&n.push(e.name)}for(let t of n)await S.deleteBlob(t),e++;return e},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),J=()=>({supportsParallelUploads:!0,supportsConcatenation:!1,supportsDeferredLength:!0,supportsResumableUploads:!0,supportsTransactionalUploads:!0,supportsStreamingRead:!0,supportsStreamingWrite:!0,maxConcurrentUploads:d,minChunkSize:l,maxChunkSize:4e3*1024*1024,maxParts:u,optimalChunkSize:b,requiresOrderedChunks:!1,requiresMimeTypeValidation:!0,maxValidationSize:void 0});return{bucket:v,create:L,remove:W,write:H,getUpload:U,read:B,readStream:z,writeStream:(e,n)=>(0,r.withAzureTimingMetrics)(r.azureUploadDurationHistogram,i.Effect.gen(function*(){let o=Date.now();yield*i.Effect.logInfo(`Starting streaming write to Azure`).pipe(i.Effect.annotateLogs({upload_id:e,container:v,size_hint:n.sizeHint})),yield*(0,r.azureUploadRequestsTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(1));let s=A(n.sizeHint),c=yield*i.Ref.make([]),u=yield*i.Ref.make(0),d=yield*i.Ref.make(1),f=yield*i.Ref.make(new Uint8Array),p=(n,o)=>i.Effect.gen(function*(){if(n.length===0||n.length<l&&!o)return;let s=yield*i.Ref.getAndUpdate(d,e=>e+1),u=a(`stream-block-${s.toString().padStart(6,`0`)}`).toString(`base64`);yield*i.Effect.logDebug(`Staging block from stream`).pipe(i.Effect.annotateLogs({upload_id:e,block_number:s,block_size:n.length,is_final_block:o}));let f=S.getBlockBlobClient(e);yield*i.Effect.tryPromise({try:()=>f.stageBlock(u,n,n.length),catch:a=>(i.Effect.runSync((0,r.trackAzureError)(`writeStream`,a,{upload_id:e,block_number:s,block_size:n.length})),t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:a}))}),yield*i.Ref.update(c,e=>[...e,u]),yield*(0,r.azureUploadPartsTotal)(i.Effect.succeed(1)),yield*(0,r.azurePartSizeHistogram)(i.Effect.succeed(n.length))});yield*n.stream.pipe(i.Stream.runForEach(e=>i.Effect.gen(function*(){yield*i.Ref.update(u,t=>t+e.length);let t=yield*i.Ref.get(f),n=new Uint8Array(t.length+e.length);n.set(t),n.set(e,t.length);let r=0;for(;n.length-r>=s;)yield*p(n.slice(r,r+s),!1),r+=s;yield*i.Ref.set(f,n.slice(r))})));let m=yield*i.Ref.get(f);m.length>0&&(yield*p(m,!0));let h=yield*i.Ref.get(c),g=yield*i.Ref.get(u);if(h.length===0)return yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1)),yield*i.Effect.fail(new t.UploadistaError({code:`FILE_WRITE_ERROR`,status:400,body:`Cannot complete upload with no data`,details:`The stream provided no data to upload`}));let _=S.getBlockBlobClient(e);yield*i.Effect.tryPromise({try:()=>_.commitBlockList(h,{blobHTTPHeaders:{blobContentType:n.contentType}}),catch:n=>(i.Effect.runSync((0,r.trackAzureError)(`writeStream`,n,{upload_id:e,operation:`commit`,blocks:h.length})),t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:n}))});let y=Date.now()-o,b=y>0?g*1e3/y:0,x=h.length>0?g/h.length:void 0;return yield*(0,r.logAzureUploadCompletion)(e,{fileSize:g,totalDurationMs:y,partsCount:h.length,averagePartSize:x,throughputBps:b,retryCount:0}),yield*(0,r.azureUploadSuccessTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1)),yield*(0,r.azureFileSizeHistogram)(i.Effect.succeed(g)),yield*i.Effect.logInfo(`Streaming write to Azure completed`).pipe(i.Effect.annotateLogs({upload_id:e,total_bytes:g,blocks_count:h.length,duration_ms:y})),{id:e,size:g,path:e,bucket:v}}).pipe(i.Effect.catchAll(e=>i.Effect.gen(function*(){return yield*(0,r.azureUploadErrorsTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1)),yield*i.Effect.fail(e)})))),deleteExpired:q,getCapabilities:J,getChunkerConstraints:()=>({minChunkSize:l,maxChunkSize:4e3*1024*1024,optimalChunkSize:b,requiresOrderedChunks:!1}),validateUploadStrategy:e=>{let t=J(),n=(()=>{switch(e){case`parallel`:return t.supportsParallelUploads;case`single`:return!0;default:return!1}})();return i.Effect.succeed(n)}}})}exports.azureStore=s;
+Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});let e=require(`@azure/storage-blob`),t=require(`@uploadista/core/errors`),n=require(`@uploadista/core/types`),r=require(`@uploadista/observability`),i=require(`effect`);const a=e=>typeof globalThis<`u`&&`Buffer`in globalThis?globalThis.Buffer.from(e):new Uint8Array(Array.from(e,e=>e.charCodeAt(0)));function o(e){return e&&e.length>0?e.reduce((e,t)=>e+(t?.size??0),0):0}function s({deliveryUrl:s,blockSize:c,minBlockSize:l=1024,maxBlocks:u=5e4,maxConcurrentBlockUploads:d=60,expirationPeriodInMilliseconds:f=1e3*60*60*24*7,connectionString:p,sasUrl:m,credential:h,accountName:g,accountKey:_,containerName:v}){return i.Effect.gen(function*(){let y=yield*n.UploadFileKVStore,b=c||8*1024*1024,x;if(p)x=e.BlobServiceClient.fromConnectionString(p);else if(m)x=new e.BlobServiceClient(m);else if(h){let t=g?`https://${g}.blob.core.windows.net`:m?.split(`?`)[0]||``;if(!t)throw Error(`When using credential authentication, either accountName or a valid sasUrl must be provided to determine the account URL`);x=new e.BlobServiceClient(t,h)}else if(g&&_)try{let t=new e.StorageSharedKeyCredential(g,_);x=new e.BlobServiceClient(`https://${g}.blob.core.windows.net`,t)}catch(e){throw Error(`StorageSharedKeyCredential is only available in Node.js environments. Use sasUrl or credential options for cross-platform compatibility. Original error: ${e}`)}else throw Error(`Azure authentication required. Provide one of: connectionString, sasUrl, credential, or accountName + accountKey (Node.js only)`);let S=x.getContainerClient(v),C=e=>`${e}.incomplete`,w=(e,n,a)=>(0,r.withAzureTimingMetrics)(r.azurePartUploadDurationHistogram,i.Effect.gen(function*(){yield*i.Effect.logInfo(`Uploading block`).pipe(i.Effect.annotateLogs({upload_id:e.id,block_id:a,block_size:n.length})),yield*(0,r.azureUploadPartsTotal)(i.Effect.succeed(1)),yield*(0,r.azurePartSizeHistogram)(i.Effect.succeed(n.length));try{let o=S.getBlockBlobClient(e.id);yield*i.Effect.tryPromise({try:async()=>{await o.stageBlock(a,n,n.length)},catch:o=>(i.Effect.runSync((0,r.trackAzureError)(`uploadBlock`,o,{upload_id:e.id,block_id:a,block_size:n.length})),t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:o}))}),yield*i.Effect.logInfo(`Finished uploading block`).pipe(i.Effect.annotateLogs({upload_id:e.id,block_id:a,block_size:n.length}))}catch(t){throw i.Effect.runSync((0,r.trackAzureError)(`uploadBlock`,t,{upload_id:e.id,block_id:a,block_size:n.length})),t}})),T=(e,n)=>i.Effect.tryPromise({try:async()=>{await S.getBlockBlobClient(C(e)).upload(n,n.length)},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}).pipe(i.Effect.tap(()=>i.Effect.logInfo(`Finished uploading incomplete block`).pipe(i.Effect.annotateLogs({upload_id:e})))),E=e=>i.Effect.tryPromise({try:async()=>{try{return(await S.getBlockBlobClient(C(e)).download()).readableStreamBody}catch(e){if(e&&typeof e==`object`&&`statusCode`in e&&e.statusCode===404)return;throw e}},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),D=e=>i.Effect.tryPromise({try:async()=>{try{return(await S.getBlockBlobClient(C(e)).getProperties()).contentLength}catch(e){if(e&&typeof e==`object`&&`statusCode`in e&&e.statusCode===404)return;throw e}},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),O=e=>i.Effect.tryPromise({try:async()=>{await S.getBlockBlobClient(C(e)).deleteIfExists()},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),k=e=>i.Effect.gen(function*(){let t=yield*E(e);if(!t)return;let n=t.getReader(),r=[],a=0;try{for(;;){let e=yield*i.Effect.promise(()=>n.read());if(e.done)break;r.push(e.value),a+=e.value.length}}finally{n.releaseLock()}let o=i.Stream.fromIterable(r);return{size:a,stream:o}}),A=e=>{let t=e??5497558138880,n;n=t<=b?t:t<=b*u?b:Math.ceil(t/u);let r=Math.max(n,l);return Math.ceil(r/1024)*1024},j=e=>t=>i.Stream.async(n=>{let r=new Uint8Array,a=1,o=0,s=(t,r=!1)=>{i.Effect.runSync(i.Effect.logInfo(`Creating chunk`).pipe(i.Effect.annotateLogs({block_number:a,chunk_size:t.length,expected_size:e,is_final_chunk:r,total_bytes_processed:o+t.length}))),n.single({blockNumber:a++,data:t,size:t.length})},c=t=>{let n=new Uint8Array(r.length+t.length);for(n.set(r),n.set(t,r.length),r=n,o+=t.length;r.length>=e;){let t=r.slice(0,e);r=r.slice(e),s(t,!1)}};i.Effect.runFork(t.pipe(i.Stream.runForEach(e=>i.Effect.sync(()=>c(e))),i.Effect.andThen(()=>i.Effect.sync(()=>{r.length>0&&s(r,!0),n.end()})),i.Effect.catchAll(e=>i.Effect.sync(()=>n.fail(e)))))}),M=(e,t=0)=>n=>e?i.Effect.gen(function*(){let r=yield*i.Ref.make(t);return n.pipe(i.Stream.tap(t=>i.Effect.gen(function*(){e(yield*i.Ref.updateAndGet(r,e=>e+t.length))})))}).pipe(i.Stream.unwrap):n,N=(e,n,o,s,c)=>i.Effect.gen(function*(){yield*i.Effect.logInfo(`Uploading blocks`).pipe(i.Effect.annotateLogs({upload_id:e.id,init_offset:s,file_size:e.size}));let u=e.size,f=A(u);yield*i.Effect.logInfo(`Block size`).pipe(i.Effect.annotateLogs({upload_id:e.id,block_size:f}));let p=n.pipe(M(c,s),j(f)),m=yield*i.Ref.make(s),h=yield*i.Ref.make(0),g=yield*i.Ref.make([]),_=n=>i.Effect.gen(function*(){let s=yield*i.Ref.updateAndGet(m,e=>e+n.size),c=s>=(e.size||0);yield*i.Effect.logDebug(`Processing chunk`).pipe(i.Effect.annotateLogs({upload_id:e.id,cumulative_offset:s,file_size:e.size,chunk_size:n.size,is_final_block:c}));let u=o+n.blockNumber-1;if(n.size>f&&(yield*i.Effect.fail(t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:Error(`Block size ${n.size} exceeds upload block size ${f}`)}))),n.size>=l||c){yield*i.Effect.logDebug(`Uploading multipart chunk`).pipe(i.Effect.annotateLogs({upload_id:e.id,block_number:u,chunk_size:n.size,min_block_size:l,is_final_block:c}));let t=a(`block-${u.toString().padStart(6,`0`)}`).toString(`base64`);yield*w(e,n.data,t),yield*i.Ref.update(g,e=>[...e,t]),yield*(0,r.azurePartSizeHistogram)(i.Effect.succeed(n.size))}else yield*T(e.id,n.data);yield*i.Ref.update(h,e=>e+n.size)});return yield*p.pipe(i.Stream.runForEach(e=>_(e)),i.Effect.withConcurrency(d)),{bytesUploaded:yield*i.Ref.get(h),blockIds:yield*i.Ref.get(g)}}),P=(e,n)=>i.Effect.tryPromise({try:async()=>{await S.getBlockBlobClient(e.id).commitBlockList(n,{blobHTTPHeaders:{blobContentType:e.metadata?.contentType?.toString(),blobCacheControl:e.metadata?.cacheControl?.toString()}})},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),F=e=>i.Effect.tryPromise({try:async()=>{try{return(await S.getBlockBlobClient(e).getBlockList(`committed`)).committedBlocks?.map(e=>({size:e.size}))??[]}catch(e){if(e&&typeof e==`object`&&`statusCode`in e&&e.statusCode===404)return[];throw e}},catch:e=>t.UploadistaError.fromCode(`UPLOAD_ID_NOT_FOUND`,{cause:e})}),I=e=>i.Effect.gen(function*(){yield*i.Effect.logInfo(`Removing cached data`).pipe(i.Effect.annotateLogs({upload_id:e})),yield*y.delete(e)}),L=e=>i.Effect.gen(function*(){return yield*(0,r.azureUploadRequestsTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(1)),yield*(0,r.azureFileSizeHistogram)(i.Effect.succeed(e.size||0)),yield*i.Effect.logInfo(`Initializing Azure blob upload`).pipe(i.Effect.annotateLogs({upload_id:e.id})),e.creationDate=new Date().toISOString(),e.storage={id:e.storage.id,type:e.storage.type,path:e.id,bucket:v},e.url=`${s}/${e.id}`,yield*y.set(e.id,e),yield*i.Effect.logInfo(`Azure blob upload initialized`).pipe(i.Effect.annotateLogs({upload_id:e.id})),e}),R=e=>i.Effect.tryPromise({try:async()=>{let t=await S.getBlockBlobClient(e).download();if(t.blobBody)return t.blobBody;if(t.readableStreamBody)return t.readableStreamBody;throw Error(`No blob body or readable stream body`)},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),z=(e,r)=>i.Effect.gen(function*(){let a={...n.DEFAULT_STREAMING_CONFIG,...r},o=yield*R(e);if(o instanceof Blob){let e=yield*i.Effect.promise(()=>o.arrayBuffer()),t=new Uint8Array(e),n=a.chunkSize,r=[];for(let e=0;e<t.length;e+=n)r.push(t.slice(e,Math.min(e+n,t.length)));return i.Stream.fromIterable(r)}return i.Stream.async(e=>{let n=o.getReader(),r=a.chunkSize,s=new Uint8Array;return(async()=>{try{for(;;){let{done:t,value:i}=await n.read();if(t){s.length>0&&e.single(s),e.end();return}if(i){let t=new Uint8Array(s.length+i.length);for(t.set(s),t.set(i,s.length),s=t;s.length>=r;){let t=s.slice(0,r);s=s.slice(r),e.single(t)}}}}catch(n){e.fail(new t.UploadistaError({code:`FILE_READ_ERROR`,status:500,body:`Failed to read Azure blob stream`,details:`Azure stream read failed: ${String(n)}`}))}})(),i.Effect.sync(()=>{n.releaseLock()})})}),B=e=>i.Effect.gen(function*(){let t=yield*z(e),n=[];yield*i.Stream.runForEach(t,e=>i.Effect.sync(()=>{n.push(e)}));let r=n.reduce((e,t)=>e+t.length,0),a=new Uint8Array(r),o=0;for(let e of n)a.set(e,o),o+=e.length;return a}),V=(e,t,n)=>i.Effect.gen(function*(){let r=yield*y.get(e),a=(yield*F(e)).length+1,o=yield*k(e);if(o){yield*O(e);let s=t-o.size,c=o.stream.pipe(i.Stream.concat(n));return{uploadFile:r,nextBlockNumber:a-1,offset:s,incompleteBlockSize:o.size,data:c}}else return{uploadFile:r,nextBlockNumber:a,offset:t,incompleteBlockSize:0,data:n}}),H=(e,t)=>(0,r.withAzureUploadMetrics)(e.file_id,(0,r.withAzureTimingMetrics)(r.azureUploadDurationHistogram,i.Effect.gen(function*(){let n=Date.now(),{stream:a,file_id:o,offset:s}=e,{onProgress:c}=t,{uploadFile:l,nextBlockNumber:u,offset:d,data:f}=yield*V(o,s,a),{bytesUploaded:p,blockIds:m}=yield*N(l,f,u,d,c),h=d+p;if(l.size===h)try{yield*P(l,m),yield*y.set(o,{...l,offset:h}),yield*(0,r.logAzureUploadCompletion)(o,{fileSize:l.size||0,totalDurationMs:Date.now()-n,partsCount:m.length,averagePartSize:l.size,throughputBps:l.size/(Date.now()-n),retryCount:0}),yield*(0,r.azureUploadSuccessTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1))}catch(e){throw yield*i.Effect.logError(`Failed to finish upload`).pipe(i.Effect.annotateLogs({upload_id:o,error:JSON.stringify(e)})),yield*(0,r.azureUploadErrorsTotal)(i.Effect.succeed(1)),i.Effect.runSync((0,r.trackAzureError)(`write`,e,{upload_id:o,operation:`commit`,blocks:m.length})),e}return h}))),U=e=>i.Effect.gen(function*(){let t=yield*y.get(e),n=0;try{n=o(yield*F(e))}catch(n){if(typeof n==`object`&&n&&`statusCode`in n&&n.statusCode===404)return{...t,offset:t.size,size:t.size,metadata:t.metadata,storage:t.storage};throw yield*i.Effect.logError(`Error on get upload`).pipe(i.Effect.annotateLogs({upload_id:e,error:JSON.stringify(n)})),n}let r=yield*D(e);return{...t,offset:n+(r??0),size:t.size,storage:t.storage}}),W=e=>i.Effect.gen(function*(){try{let t=S.getBlockBlobClient(e);yield*i.Effect.promise(()=>t.deleteIfExists()),yield*O(e)}catch(n){if(typeof n==`object`&&n&&`statusCode`in n&&n.statusCode===404)return yield*i.Effect.logError(`No file found`).pipe(i.Effect.annotateLogs({upload_id:e})),yield*i.Effect.fail(t.UploadistaError.fromCode(`FILE_NOT_FOUND`));throw i.Effect.runSync((0,r.trackAzureError)(`remove`,n,{upload_id:e})),n}yield*I(e),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1))}),G=()=>f,K=e=>{let t=new Date(e);return new Date(t.getTime()+G())},q=()=>i.Effect.tryPromise({try:async()=>{if(G()===0)return 0;let e=0,t=S.listBlobsFlat({includeMetadata:!0}),n=[];for await(let e of t)if(e.metadata?.creationDate){let t=new Date(e.metadata.creationDate);Date.now()>K(t.toISOString()).getTime()&&n.push(e.name)}for(let t of n)await S.deleteBlob(t),e++;return e},catch:e=>t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})}),J=()=>({supportsParallelUploads:!0,supportsConcatenation:!1,supportsDeferredLength:!0,supportsResumableUploads:!0,supportsTransactionalUploads:!0,supportsStreamingRead:!0,supportsStreamingWrite:!0,maxConcurrentUploads:d,minChunkSize:l,maxChunkSize:4e3*1024*1024,maxParts:u,optimalChunkSize:b,requiresOrderedChunks:!1,requiresMimeTypeValidation:!0,maxValidationSize:void 0});return{bucket:v,create:L,remove:W,write:H,getUpload:U,read:B,readStream:z,writeStream:(e,n)=>(0,r.withAzureTimingMetrics)(r.azureUploadDurationHistogram,i.Effect.gen(function*(){let o=Date.now();yield*i.Effect.logInfo(`Starting streaming write to Azure`).pipe(i.Effect.annotateLogs({upload_id:e,container:v,size_hint:n.sizeHint})),yield*(0,r.azureUploadRequestsTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(1));let s=A(n.sizeHint),c=yield*i.Ref.make([]),u=yield*i.Ref.make(0),d=yield*i.Ref.make(1),f=yield*i.Ref.make(new Uint8Array),p=(n,o)=>i.Effect.gen(function*(){if(n.length===0||n.length<l&&!o)return;let s=yield*i.Ref.getAndUpdate(d,e=>e+1),u=a(`stream-block-${s.toString().padStart(6,`0`)}`).toString(`base64`);yield*i.Effect.logDebug(`Staging block from stream`).pipe(i.Effect.annotateLogs({upload_id:e,block_number:s,block_size:n.length,is_final_block:o}));let f=S.getBlockBlobClient(e);yield*i.Effect.tryPromise({try:()=>f.stageBlock(u,n,n.length),catch:a=>(i.Effect.runSync((0,r.trackAzureError)(`writeStream`,a,{upload_id:e,block_number:s,block_size:n.length})),t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:a}))}),yield*i.Ref.update(c,e=>[...e,u]),yield*(0,r.azureUploadPartsTotal)(i.Effect.succeed(1)),yield*(0,r.azurePartSizeHistogram)(i.Effect.succeed(n.length))});yield*n.stream.pipe(i.Stream.runForEach(e=>i.Effect.gen(function*(){yield*i.Ref.update(u,t=>t+e.length);let t=yield*i.Ref.get(f),n=new Uint8Array(t.length+e.length);n.set(t),n.set(e,t.length);let r=0;for(;n.length-r>=s;)yield*p(n.slice(r,r+s),!1),r+=s;yield*i.Ref.set(f,n.slice(r))})));let m=yield*i.Ref.get(f);m.length>0&&(yield*p(m,!0));let h=yield*i.Ref.get(c),g=yield*i.Ref.get(u);if(h.length===0)return yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1)),yield*i.Effect.fail(new t.UploadistaError({code:`FILE_WRITE_ERROR`,status:400,body:`Cannot complete upload with no data`,details:`The stream provided no data to upload`}));let _=S.getBlockBlobClient(e);yield*i.Effect.tryPromise({try:()=>_.commitBlockList(h,{blobHTTPHeaders:{blobContentType:n.contentType}}),catch:n=>(i.Effect.runSync((0,r.trackAzureError)(`writeStream`,n,{upload_id:e,operation:`commit`,blocks:h.length})),t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:n}))});let y=Date.now()-o,b=y>0?g*1e3/y:0,x=h.length>0?g/h.length:void 0;return yield*(0,r.logAzureUploadCompletion)(e,{fileSize:g,totalDurationMs:y,partsCount:h.length,averagePartSize:x,throughputBps:b,retryCount:0}),yield*(0,r.azureUploadSuccessTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1)),yield*(0,r.azureFileSizeHistogram)(i.Effect.succeed(g)),yield*i.Effect.logInfo(`Streaming write to Azure completed`).pipe(i.Effect.annotateLogs({upload_id:e,total_bytes:g,blocks_count:h.length,duration_ms:y})),{id:e,size:g,path:e,bucket:v}}).pipe(i.Effect.catchAll(e=>i.Effect.gen(function*(){return yield*(0,r.azureUploadErrorsTotal)(i.Effect.succeed(1)),yield*(0,r.azureActiveUploadsGauge)(i.Effect.succeed(-1)),yield*i.Effect.fail(e)})))),deleteExpired:q,getCapabilities:J,getChunkerConstraints:()=>({minChunkSize:l,maxChunkSize:4e3*1024*1024,optimalChunkSize:b,requiresOrderedChunks:!1}),validateUploadStrategy:e=>{let t=J(),n=(()=>{switch(e){case`parallel`:return t.supportsParallelUploads;case`single`:return!0;default:return!1}})();return i.Effect.succeed(n)}}})}exports.azureStore=s;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@uploadista/data-store-azure",
   "type": "module",
-  "version": "0.1.4-beta.1",
+  "version": "0.2.0",
   "description": "Azure Blob Storage data store for Uploadista",
   "license": "MIT",
   "author": "Uploadista",
@@ -15,20 +15,20 @@
   },
   "dependencies": {
     "@azure/core-auth": "^1.8.0",
-    "@azure/storage-blob": "12.30.0",
-    "@uploadista/kv-store-memory": "0.1.4-beta.1",
-    "@uploadista/observability": "0.1.4-beta.1",
-    "@uploadista/core": "0.1.4-beta.1"
+    "@azure/storage-blob": "12.31.0",
+    "@uploadista/core": "0.2.0",
+    "@uploadista/kv-store-memory": "0.2.0",
+    "@uploadista/observability": "0.2.0"
   },
   "peerDependencies": {
     "effect": "^3.0.0"
   },
   "devDependencies": {
     "@effect/vitest": "0.27.0",
-    "effect": "3.19.15",
-    "tsdown": "0.20.1",
+    "effect": "3.19.17",
+    "tsdown": "0.20.3",
     "vitest": "4.0.18",
-    "@uploadista/typescript-config": "0.1.4-beta.1"
+    "@uploadista/typescript-config": "0.2.0"
   },
   "scripts": {
     "build": "tsc --noEmit && tsdown",