aspect-sync 0.1.18 → 0.1.20

package/dist/index.js

@@ -3,10 +3,10 @@ import{Command as Nt}from"commander";import $e from"node:path";import{readFileSy
 `)){if(l.trim().length===0)continue;let u=yt(l);if(!u)continue;let c=Date.now();!(u.percentComplete>=100)&&c-i<500||(i=c,n.onProgress?.(u))}};s.stdout.on("data",a=>{o(a.toString())}),s.stderr.on("data",a=>{let l=a.toString();r+=l,o(l)}),s.on("close",async a=>{a===0?e():t(new A(`rclone exited with code ${a}`,r))}),s.on("error",a=>{t(new A(`Failed to start rclone: ${a.message}`,""))})})}async function ye(){return new Promise((n,e)=>{let t=X("rclone",["version"]);t.on("error",()=>{e(new Error("rclone is not installed or not in PATH. Please install rclone first: https://rclone.org/install/"))}),t.on("close",s=>{s===0?n():e(new Error("rclone is not installed or not in PATH. Please install rclone first: https://rclone.org/install/"))})})}async function St(n,e,t,s,r,i){let o=`${n}:${e}`;return fe({args:ht({remoteSource:o,localPath:t,rcloneOptions:r,batchFilePath:s}),remoteSource:o,onProgress:i})}async function V(n,e,t=[]){let s=`${n}:${e}`;return new Promise((r,i)=>{let o=X("rclone",pt({remoteSource:s,extraArgs:t})),a="",l="";o.stdout.on("data",u=>{a+=u.toString()}),o.stderr.on("data",u=>{l+=u.toString()}),o.on("close",async u=>{if(u===0)try{let c=JSON.parse(a||"[]");if(!Array.isArray(c))throw new Error("rclone lsjson did not return an array");let d=Ct(c),h=await bt({remote:n,files:d.files,directories:d.directories,extraArgs:t});r(h)}catch(c){i(new A(`Failed to parse file listing from ${s}: ${c instanceof Error?c.message:String(c)}`,l))}else i(new A(`Failed to list files from ${s}`,l))}),o.on("error",u=>{i(new A(`Failed to start rclone: ${u.message}`,""))})})}async function ke(n){let e=[],t=[];for(let s of n.files)Pt(s)?t.push(s):e.push(s);e.length>0&&(await G.promises.writeFile(n.batchFilePath,`${e.map(s=>s.path).join(`
 `)}
 `,"utf-8"),await St(n.remote,n.remotePath,n.localPath,n.batchFilePath,n.rcloneOptions,n.onProgress));for(let s of t)await It({remote:n.remote,remotePath:n.remotePath,localPath:n.localPath,file:s,rcloneOptions:n.rcloneOptions,onProgress:n.onProgress})}function Ct(n){let e=[],t=[];for(let s of n){let r=s;if(typeof r.Path!="string")continue;let i=typeof r.ID=="string"&&r.ID.length>0?r.ID:void 0;if(r.IsDir===!0){t.push({path:b(r.Path),id:i});continue}let o=typeof r.Size=="number"&&Number.isFinite(r.Size)?r.Size:0;e.push({path:b(r.Path),size:o,id:i})}return{files:e,directories:t}}async function bt(n){let e=At(n.directories);if(e.length===0)return n.files;let t=e.map(o=>o[0].path),s=n.files.filter(o=>!t.some(a=>ve(o.path,a))),r=Et(n.directories),i=[];for(let o of e)for(let a of o){if(!a.id)throw new Error(`Cannot preserve duplicate remote folder without a stable rclone folder ID: ${a.path}`);let l=N(a.path),u=ge(l,r);r.add(u),i.push(...await _e({remote:n.remote,parentId:a.id,sourcePrefix:a.path,targetPrefix:u,extraArgs:n.extraArgs}))}return[...s,...i]}function At(n){let e=new Map;for(let i of n){let o=e.get(i.path)??[];o.push(i),e.set(i.path,o)}let t=Array.from(e.values()).filter(i=>i.length>1).sort((i,o)=>i[0].path.split("/").length-o[0].path.split("/").length),s=[],r=[];for(let i of t){let o=i[0].path;r.some(a=>ve(o,a))||(r.push(o),s.push(i))}return s}function Et(n){let e=new Map;for(let s of n)e.set(s.path,(e.get(s.path)??0)+1);let t=new Set;for(let s of n)(e.get(s.path)??0)>1||t.add(N(s.path));return t}async function _e(n){let e=await Tt({remote:n.remote,parentId:n.parentId,extraArgs:n.extraArgs}),t=new Set,s=[];for(let r of e){if(typeof r.name!="string"||typeof r.id!="string")continue;let i=`${n.sourcePrefix}/${r.name}`,o=`${n.targetPrefix}/${N(r.name)}`,a=r.mimeType==="application/vnd.google-apps.folder"?ge(o,t):me(o,t);if(t.add(a),r.mimeType==="application/vnd.google-apps.folder"){s.push(...await _e({remote:n.remote,parentId:r.id,sourcePrefix:i,targetPrefix:a,extraArgs:n.extraArgs}));continue}s.push({path:i,targetPath:a,size:Rt(r.size),id:r.id,requiresIdentityDownload:!0})}return s}function Tt(n){let e=`'${Ut(n.parentId)}' in parents and trashed = false`;return new Promise((t,s)=>{let r=X("rclone",mt({remote:n.remote,query:e,extraArgs:n.extraArgs}),{stdio:["ignore","pipe","pipe"]}),i="",o="";r.stdout.on("data",a=>{i+=a.toString()}),r.stderr.on("data",a=>{o+=a.toString()}),r.on("close",a=>{if(a!==0){s(new A(`Failed to query children for Google Drive folder ${n.parentId}`,o));return}try{let l=JSON.parse(i||"[]");if(!Array.isArray(l))throw new Error("rclone backend query did not return an array");t(l)}catch(l){s(new A(`Failed to parse children for Google Drive folder ${n.parentId}: ${l instanceof Error?l.message:String(l)}`,o))}}),r.on("error",a=>{s(new A(`Failed to start rclone: ${a.message}`,""))})})}function Rt(n){if(typeof n=="number"&&Number.isFinite(n))return n;if(typeof n=="string"){let e=Number(n);return Number.isFinite(e)?e:0}return 0}function ve(n,e){return n===e||n.startsWith(`${e}/`)}function Ut(n){return n.replace(/\\/g,"\\\\").replace(/'/g,"\\'")}function Pt(n){let e=k(n),t=b(n.path);return n.requiresIdentityDownload===!0||e!==t}async function It(n){let e=E.join(n.localPath,k(n.file));if(await G.promises.mkdir(E.dirname(e),{recursive:!0}),n.file.id){await fe({args:ft({remote:n.remote,fileId:n.file.id,localTargetPath:e,rcloneOptions:n.rcloneOptions}),remoteSource:`${n.remote}:${n.file.path}`,onProgress:n.onProgress});return}if(n.file.requiresIdentityDownload===!0)throw new Error(`Cannot preserve duplicate remote file without a stable rclone file ID: ${n.file.path}`);await fe({args:gt({remoteFileSource:`${n.remote}:${Ft(n.remotePath,n.file.path)}`,localTargetPath:e,rcloneOptions:n.rcloneOptions}),remoteSource:`${n.remote}:${n.file.path}`,onProgress:n.onProgress})}function Ft(n,e){let t=n.replace(/\/+$/,""),s=e.replace(/^\/+/,"");return t.length===0?s:`${t}/${s}`}async function we(n){let e=[],t=new Set;async function s(r,i=""){let o=await G.promises.readdir(r,{withFileTypes:!0});for(let a of o){let l=E.join(r,a.name),u=i?E.join(i,a.name):a.name;if(a.isDirectory())await s(l,u);else if(a.isFile()){let c=await G.promises.stat(l);e.push({relativePath:u,absolutePath:l,fileName:a.name,size:c.size});let d=E.dirname(u);if(d!=="."){let h=d.split(E.sep);for(let p=0;p<h.length;p++){let g=h.slice(0,p+1).join(E.sep);t.add(g)}}}}}return await s(n),{files:e,directoryCount:t.size}}import kt from"fs/promises";import _t from"path";function xe(n,e){let t=new Map;for(let u of n){let c=_t.dirname(u.targetPath??u.path);t.has(c)||t.set(c,[]),t.get(c).push(u)}let s=new Map;for(let[u,c]of t.entries()){let d=c.reduce((h,p)=>h+p.size,0);s.set(u,d)}let r=Array.from(t.keys()).sort(),i=[],o=[],a=0,l=1;for(let u of r){let c=t.get(u),d=s.get(u);if(d>e){if(o.length>0&&(i.push({batchNumber:l++,files:o,totalSize:a}),o=[],a=0),c.length===1)i.push({batchNumber:l++,files:c,totalSize:d});else for(let h of c)a+h.size>e&&o.length>0&&(i.push({batchNumber:l++,files:o,totalSize:a}),o=[],a=0),o.push(h),a+=h.size;continue}a+d>e&&o.length>0&&(i.push({batchNumber:l++,files:o,totalSize:a}),o=[],a=0),o.push(...c),a+=d}return o.length>0&&i.push({batchNumber:l++,files:o,totalSize:a}),i}async function De(n){try{await kt.unlink(n)}catch(e){if(e.code!=="ENOENT")throw e}}import m from"chalk";import Oe from"log-update";var S=(n,e=2)=>{if(n===0)return"0 B";let t=1024,s=e<0?0:e,r=["B","KB","MB","GB","TB","PB","EB"],i=Math.floor(Math.log(n)/Math.log(t));return`${(n/Math.pow(t,i)).toFixed(s)} ${r[i]}`},Be=n=>{if(n===null)return null;let t=n*8/(1024*1024);return t<1?`${(t*1024).toFixed(0)} Kbps`:`${t.toFixed(1)} Mbps`},H=n=>{if(n===null)return null;let e=Math.floor(n/3600),t=Math.floor(n%3600/60),s=Math.floor(n%60);return e>0?`${e}h ${t}m`:t>0?`${t}m ${s}s`:`${s}s`};var wt=1e3,Se=class{#t=null;#s=!1;#e=null;#i=null;#o=null;#n=new Map;#a=null;#l=null;#r=null;#u=null;start(){this.#s||(this.#s=!0,this.#n.clear(),this.#a=null,this.#l=null,this.#r=null,this.#u=null,this.#t=setInterval(()=>this.render(),wt))}stop(){this.#s&&(this.#s=!1,this.#t&&(clearInterval(this.#t),this.#t=null),this.render(),Oe.done())}updateSnapshot(e){this.#e=e}updateBatchState(e){this.#i=e}updateDownloadProgress(e){if(!e){this.#o=null;return}let t=e.percentComplete>=100||e.bytesTransferred>=e.totalBytes;!t&&e.speed>0&&(this.#r=e),this.#o=t&&this.#r?{...e,speed:this.#r.speed,eta:this.#r.eta}:e}persistCurrentBatchLine(){!this.#i||this.#i.currentPhase!=="complete"||this.#n.set(this.#i.currentBatch,this.#h(this.#i))}resetUploadSnapshot(){this.#e=null,this.#o=null,this.#r=null,this.#u=null}render(){if(!this.#s)return;let e=this.#i?this.#k(this.#i):this.#g();Oe(e.join(`
-`))}#A(e,t){let s=[`${m.green(S(t.bytesTransferred))} / ${m.gray(S(t.totalBytes))}`,m.cyan(`${t.percentComplete.toFixed(1)}%`),m.magenta(Be(t.speed)??"..."),`${m.yellow("ETA")}: ${m.yellow(H(t.eta)??"...")}`];return`${m.blue(e)}: ${s.join(" | ")}`}#g(){let e=this.#c();return[this.#m(e),this.#E(e)]}#k(e){let t=[];for(let s=1;s<e.currentBatch;s++){let r=this.#n.get(s);r&&t.push(r)}return t.push(this.#h(e)),t.push(this.#U(e)),t}#h(e){let t=`Batch ${e.currentBatch}/${e.totalBatches}`;switch(e.currentPhase){case"downloading":return this.#o?`${t}: ${this.#A("Downloading",this.#o)}`:`${t}: ${m.blue("Downloading...")}`;case"scanning":return`${t}: ${m.yellow(e.extraDetails??"Scanning...")}`;case"syncing_folders":return`${t}: ${m.yellow(e.extraDetails??"Syncing folders...")}`;case"uploading":return this.#D(t,e);case"cleaning":return`${t}: ${m.yellow("Cleaning...")}`;case"complete":return`${t}: ${m.green("Complete")} (${e.batchFilesTotal} files, ${S(e.batchBytesTotal)}${e.extraDetails?`, ${e.extraDetails}`:""})`}}#D(e,t){let s=this.#c(),r=s.successFileCount+s.failedFileCount+s.cancelledFileCount,i=Math.max(t.batchFilesTotal-t.batchFilesSkipped,0),o=Math.max(t.batchBytesTotal-t.batchBytesSkipped,0),a=o>0?(s.uploadedBytes/o*100).toFixed(1):"0.0",l=this.#f(t,s,o),u=l?.formattedSpeed??null,c=l?.formattedTime??null,d=u&&c?`, ${u}, ETA: ${c}`:"",h=t.batchFilesSkipped>0?`, ${m.yellow(`Skipped: ${t.batchFilesSkipped} files / ${S(t.batchBytesSkipped)}`)}`:"";return`${e}: ${m.blue("Uploading")} (${r}/${i} files, ${S(s.uploadedBytes)} / ${S(o)}, ${a}%${d}${h})`}#U(e){let t=this.#c(),s=e.batchFilesSkipped+t.successFileCount+t.failedFileCount+t.cancelledFileCount,r=e.batchBytesSkipped+t.uploadedBytes,i=e.overallFilesCompleted+s,o=e.overallBytesCompleted+r,a=Math.max(e.overallBytesTotal-o,0),l=this.#b(e),u=this.#P(e,t),c=l!==null&&u!==null?l+u:null;return[`Total: ${i}/${e.overallFilesTotal} files`,`${S(o)} / ${S(e.overallBytesTotal)}`,m.yellow(`${S(a)} remaining`),`Download ETA: ${H(l)??"..."}`,`Upload ETA: ${H(u)??"..."}`,`Total ETA: ${H(c)??"..."}`].join(", ")}#b(e){let t=e.overallBytesCompleted+e.batchBytesSkipped+(this.#o?.bytesTransferred??0),s=Math.max(e.overallBytesTotal-t,0);if(s===0)return e.currentPhase==="downloading"&&this.#o?(this.#a=this.#o.eta,this.#o.eta):(this.#a=0,0);if(this.#o&&this.#o.speed>0){let r=Math.ceil(s/this.#o.speed);return this.#a=r,r}return this.#a}#P(e,t){let s=e.overallBytesCompleted+e.batchBytesSkipped+t.uploadedBytes,r=Math.max(e.overallBytesTotal-s,0),i=Math.max(e.batchBytesTotal-e.batchBytesSkipped,0),o=this.#f(e,t,i);if(r===0)return e.currentPhase==="uploading"&&o?(this.#l=o.timeRemainingSeconds,o.timeRemainingSeconds):(this.#l=0,0);if(o&&o.uploadSpeedMbps>0){let a=o.uploadSpeedMbps*1024*1024/8,l=Math.ceil(r/a);return this.#l=l,l}return this.#l}#f(e,t,s){let r=this.#e?.uploadStats,i=s>0&&t.uploadedBytes>=s;return r&&r.uploadSpeedMbps>0&&r.formattedSpeed!=="Calculating..."&&!i?(this.#u={uploadSpeedMbps:r.uploadSpeedMbps,formattedSpeed:r.formattedSpeed,formattedTime:r.formattedTime,timeRemainingSeconds:r.timeRemainingSeconds},this.#u):e.currentPhase==="uploading"&&i&&this.#u?this.#u:r?{uploadSpeedMbps:r.uploadSpeedMbps,formattedSpeed:r.formattedSpeed,formattedTime:r.formattedTime,timeRemainingSeconds:r.timeRemainingSeconds}:null}#m(e){return[`Total: ${e.totalFileCount}`,m.green(`Success: ${e.successFileCount}`),m.red(`Failed: ${e.failedFileCount}`),m.red(`Cancelled: ${e.cancelledFileCount}`),m.blue(`Active: ${e.activeFileCount}`),m.gray(`Queued: ${e.queuedFileCount}`)].join(" | ")}#E(e){let t=Math.max(e.totalBytes-e.uploadedBytes,0),s=this.#e?` | ${m.cyan(`Speed: ${this.#e.uploadStats.formattedSpeed}`)} | ${m.magenta(`Time remaining: ${this.#e.uploadStats.formattedTime}`)}`:"";return[`Total: ${S(e.totalBytes)}`,m.green(`Uploaded: ${S(e.uploadedBytes)}`),m.yellow(`Remaining: ${S(t)}`)].join(" | ")+s}#c(){return this.#e?Object.values(this.#e.uploadAssets).reduce((t,s)=>{switch(t.totalFileCount+=1,t.totalBytes+=s.totalBytesToUpload,t.uploadedBytes+=s.bytesUploaded,s.status){case"success":t.successFileCount+=1;break;case"failed":t.failedFileCount+=1;break;case"cancelled":t.cancelledFileCount+=1;break;case"in-progress":t.activeFileCount+=1;break;default:t.queuedFileCount+=1;break}return t},{totalFileCount:0,successFileCount:0,failedFileCount:0,cancelledFileCount:0,activeFileCount:0,queuedFileCount:0,uploadedBytes:0,totalBytes:0}):{totalFileCount:0,successFileCount:0,failedFileCount:0,cancelledFileCount:0,activeFileCount:0,queuedFileCount:0,uploadedBytes:0,totalBytes:0}}},f=new Se;import{Readable as xt}from"node:stream";import Le from"axios";var U=class{#t;#s;#e;#i;#o=new M;#n=new Set;constructor(e){this.#t=Le.create({baseURL:e.apiUrl}),this.#s=Le.create({baseURL:e.edgeWorkerUrl}),this.#e=e.apiKey,this.#i=e.sessionId}get skippedAssetIds(){return this.#n}async post(e,t,s){let r=await this.#t.post(e,t,{headers:{Authorization:`Bearer ${this.#e}`,"X-Aspect-Session-Id":this.#i,"Content-Type":"application/json",...s}});return this.#l(e,t,r.data),r.data}async get(e,t){return(await this.#t.get(e,{headers:{Authorization:`Bearer ${this.#e}`,"X-Aspect-Session-Id":this.#i,...t}})).data}async delete(e,t){await this.#t.delete(e,{headers:{Authorization:`Bearer ${this.#e}`,"X-Aspect-Session-Id":this.#i,...t}})}async putWithToken(e,t,s,r){let i=await this.#a(t),o=this.#o.createIterator(i,r.getMaxUploadRate??(()=>null),r.signal),a=xt.from(o,{highWaterMark:1}),l=await this.#s.put(e,a,{timeout:0,signal:r.signal,headers:{"Content-Type":r.contentType??"","Content-Length":i.byteLength.toString(),Authorization:`Bearer ${s}`},transformRequest:[u=>u],maxRedirects:0,onUploadProgress:u=>r.onProgress?.(u.loaded)});return{status:l.status,statusText:l.statusText,headers:Object.fromEntries(Object.entries(l.headers)),data:l.data,raw:l}}async#a(e){if(e instanceof Uint8Array)return e;let t=await e.arrayBuffer();return new Uint8Array(t)}#l(e,t,s){if(e!=="/assets/chunked/initiate"||!Array.isArray(t)||!Array.isArray(s))return;let r=t;s.forEach((o,a)=>{let l=r[a];o.skipped===!0&&typeof l?.id=="string"&&this.#n.add(l.id)})}};import z from"node:path";var J=class{#t;#s;#e=new Map;constructor(e){this.#t=e.rootDirectoryId,this.#s=e.httpClient}async ensureDirectories(e){await this.loadExistingDirectories();let t=this.#i(e);for(let s of t){if(this.#e.has(s))continue;let r=z.posix.dirname(s),i=z.posix.basename(s),o=r==="."?this.#t:this.#e.get(r);if(!o)throw new Error(`Parent directory ID not found for path: ${s}`);let a=await this.#a(o,i);if(a.name!==i)throw new Error(`Server created renamed directory "${a.name}" for planned path "${s}". Refusing to map uploads to an unexpected target path.`);this.#e.set(s,a.id)}return e.map(s=>({file:s,directoryId:this.getDirectoryIdForFile(s.relativePath)}))}async loadExistingDirectories(){let e=await this.#o(this.#t);this.#e.clear(),this.#n(e,"")}getDirectoryIdForFile(e){let t=b(e),s=z.posix.dirname(t);if(s===".")return this.#t;let r=this.#e.get(s);if(!r)throw new Error(`Directory ID not found for path: ${s}`);return r}getExistingDirectoryIdForFile(e){let t=b(e),s=z.posix.dirname(t);return s==="."?this.#t:this.#e.get(s)??null}async checkAssetsExistence(e,t=!0){return e.length===0?[]:(await this.#s.post("/assets/check-exists-and-uploaded",{items:e,delete_if_not_exist:t})).items}#i(e){let t=new Set;for(let s of e){let r=b(s.relativePath),i=z.posix.dirname(r);if(i===".")continue;let o=i.split("/");for(let a=0;a<o.length;a++)t.add(o.slice(0,a+1).join("/"))}return Array.from(t).sort((s,r)=>s.split("/").length-r.split("/").length)}async#o(e){return this.#s.get(`/directories/${e}/tree`)}#n(e,t){let s=t===""?"":t==="."?e.name:`${t}/${e.name}`;s!==""&&this.#e.set(s,e.id);for(let r of e.children){let i=t===""?".":s;this.#n(r,i)}}async#a(e,t){return this.#s.post(`/directories/${e}/directories`,{name:t,auto_rename:"numeric"})}};import{PostHog as Dt}from"posthog-node";var P=null,Ce="",Me={};function Ne(n){P||n.config.analytics.disabled||!n.config.analytics.posthogKey||(Ce=n.user.id,Me={service:"data-sync",package_version:n.packageVersion,platform:process.platform,arch:process.arch,node_version:process.version,max_concurrent:n.config.maxConcurrent,batch_mode:n.config.batchSizeBytes!==void 0,batch_size_bytes:n.config.batchSizeBytes??null,rclone_transfers:n.config.rcloneOptions.transfers,rclone_multi_thread_streams:n.config.rcloneOptions.multiThreadStreams},P=new Dt(n.config.analytics.posthogKey,{host:n.config.analytics.posthogHost,flushAt:20,flushInterval:1e4}),P.identify({distinctId:Ce,properties:{user_id:n.user.id,email:n.user.email,first_name:n.user.first_name,last_name:n.user.last_name,utm_source_latest:null,utm_medium_latest:null,utm_campaign_latest:null,utm_content_latest:null}}))}function Ge(n,e){P&&P.capture({distinctId:Ce,event:n,properties:{...Me,...e}})}async function He(){if(!P)return;let n=P;P=null;try{await n.flush()}catch{}}import*as ee from"fs/promises";import*as te from"path";var Bt={".mp4":"video/mp4",".mov":"video/quicktime",".avi":"video/x-msvideo",".mkv":"video/x-matroska",".webm":"video/webm",".wmv":"video/x-ms-wmv",".flv":"video/x-flv",".m4v":"video/x-m4v",".mpg":"video/mpeg",".mpeg":"video/mpeg",".3gp":"video/3gpp",".ts":"video/mp2t",".mts":"video/mp2t",".mxf":"application/mxf",".jpg":"image/jpeg",".jpeg":"image/jpeg",".png":"image/png",".gif":"image/gif",".webp":"image/webp",".bmp":"image/bmp",".tiff":"image/tiff",".tif":"image/tiff",".svg":"image/svg+xml",".heic":"image/heic",".heif":"image/heif",".avif":"image/avif",".mp3":"audio/mpeg",".wav":"audio/wav",".aac":"audio/aac",".ogg":"audio/ogg",".flac":"audio/flac",".m4a":"audio/mp4",".wma":"audio/x-ms-wma",".pdf":"application/pdf",".doc":"application/msword",".docx":"application/vnd.openxmlformats-officedocument.wordprocessingml.document",".ppt":"application/vnd.ms-powerpoint",".pptx":"application/vnd.openxmlformats-officedocument.presentationml.presentation",".xls":"application/vnd.ms-excel",".xlsx":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",".rtf":"application/rtf",".json":"application/json",".xml":"application/xml"};function Ot(n){let e=te.extname(n).toLowerCase();return Bt[e]??""}var Z=class n{#t;#s=null;#e=null;#i=!1;#o;#n;#a;constructor(e,t,s,r){this.#t=e,this.#o=t,this.#n=s,this.#a=r}static async create(e){let t=await ee.stat(e),s=te.basename(e),r=Ot(s);return new n(e,t.size,s,r)}get name(){return this.#n}get size(){return this.#o}get type(){return this.#a}async readChunk(e,t){if(!this.#s){this.#e||(this.#e=ee.open(this.#t,"r"));let i=await this.#e;if(this.#e=null,this.#i)return Buffer.alloc(0);this.#s=i}let s=t-e,r=Buffer.alloc(s);return await this.#s.read(r,0,s,e),r}async close(){this.#i=!0;let e=this.#e;await this.#s?.close(),this.#s=null,e&&await(await e.catch(()=>null))?.close(),this.#e=null}};var Lt=new Set(["success","failed","cancelled"]),ze=1e3,Mt=4*1024*1024,se=class{#t;#s;#e;#i;#o;constructor(e){this.#t=e.config,this.#s=e.httpClient??new U({apiUrl:e.config.apiUrl,apiKey:e.config.apiKey,edgeWorkerUrl:e.config.edgeWorkerUrl,sessionId:e.config.sessionName}),this.#e=e.createFileReader??(t=>Z.create(t)),this.#i=e.onSnapshot,this.#o=e.analytics}async uploadFiles(e){let t=e.filter(o=>o.preSkipped===!0),s=e.filter(o=>o.preSkipped!==!0);if(s.length===0)return{totalFileCount:e.length,successFileCount:0,skippedFileCount:t.length,failedFileCount:0,cancelledFileCount:0,failedFiles:[]};let r=new L({httpClient:this.#s,maxConcurrency:this.#t.maxConcurrent,deltaIntervalMs:ze,adaptiveConcurrency:!1,analytics:this.#o}),i=[];try{let o=r.startGroupUpload({name:`${s.length} files`,type:"files",fileCount:s.length,folderCount:0,topLevelFileCount:s.length,topLevelFolderCount:0,rootDirectoryId:this.#t.directoryId,projectId:this.#t.projectId,totalBytes:s.reduce((l,u)=>l+u.file.size,0),conflictResolutionOverride:"skip"});for(let l of s){let u=await this.#e(l.file.absolutePath),c=r.startUpload({fileReader:u,directoryId:l.directoryId,projectId:this.#t.projectId,groupId:o,chunkSize:Mt});r.addAssetToGroup(o,c),i.push({assetId:c,file:l.file,fileReader:u})}await this.#n(r);let a=r.getSnapshot();return this.#i?.(a),this.#a({snapshot:a,startedUploads:i,preSkippedCount:t.length})}finally{r.destroy()}}async#n(e){let s=Date.now();for(;Date.now()-s<864e5;){let r=e.getSnapshot();this.#i?.(r);let i=Object.values(r.uploadAssets);if(i.length>0&&i.every(o=>Lt.has(o.status)))return;await new Promise(o=>setTimeout(o,ze))}throw new Error("Timed out waiting for upload session to finish")}#a(e){let t=[],s=0,r=0,i=0,o=0;for(let a of e.startedUploads){let l=e.snapshot.uploadAssets[a.assetId];if(!l){i+=1,t.push({fileName:a.file.fileName,relativePath:a.file.relativePath,errorMessage:"Upload state missing from upload engine"});continue}if(l.status==="success"){this.#s.skippedAssetIds.has(a.assetId)?r+=1:s+=1;continue}l.status==="cancelled"?o+=1:i+=1,t.push({fileName:a.file.fileName,relativePath:a.file.relativePath,errorMessage:l.errorMessage})}return{totalFileCount:e.startedUploads.length+e.preSkippedCount,successFileCount:s,skippedFileCount:e.preSkippedCount+r,failedFileCount:i,cancelledFileCount:o,failedFiles:t}}};var ne=class{#t;#s;#e;#i;constructor(e){this.#t=e,this.#s=new U({apiUrl:e.apiUrl,apiKey:e.apiKey,edgeWorkerUrl:e.edgeWorkerUrl,sessionId:e.sessionName}),this.#e=new J({rootDirectoryId:e.directoryId,httpClient:this.#s}),this.#i=new se({config:e,httpClient:this.#s,onSnapshot:t=>f.updateSnapshot(t),analytics:Ge})}async run(){this.#k("Starting data sync..."),await this.#a(),console.log("Listing all remote files...");let e=await V(this.#t.remote,this.#t.remotePath,this.#t.rcloneOptions.extraRcloneArgs??[]);console.log(`Found ${e.length} files`),console.log(`Total size: ${S(this.#f(e))}`),console.log("");let t=this.#o(e);if(t.length===0){console.log("No files to sync. Exiting."),await this.#l();return}let s={totalFileCount:0,successFileCount:0,skippedFileCount:0,failedFileCount:0,cancelledFileCount:0,completedByteCount:0},r=[],i={batchNumber:1,files:t,totalSize:this.#f(t)};f.start();try{let a=await this.#n({batch:i,totalBatches:1,overallFilesTotal:t.length,overallBytesTotal:i.totalSize,completedTotals:s,batchDir:this.#t.localTempDir});this.#U(s,r,a,i.totalSize)}finally{f.stop()}let o={...s,failedFiles:r};this.#h({summary:o}),await this.#r(o),this.#b(o)}async runBatched(){if(!this.#t.batchSizeBytes)throw new Error("Batch size not configured. Use batchSizeBytes in config.");this.#k("Starting batched data sync...",[`Batch size: ${S(this.#t.batchSizeBytes)}`,`Max concurrent chunks: ${this.#t.maxConcurrent}`]),await this.#a(),console.log("Listing all remote files...");let e=await V(this.#t.remote,this.#t.remotePath,this.#t.rcloneOptions.extraRcloneArgs??[]);console.log(`Found ${e.length} files`),console.log(`Total size: ${S(this.#f(e))}`),console.log("");let t=this.#o(e);if(t.length===0){console.log("No files to sync. Exiting."),await this.#l();return}let s=xe(t,this.#t.batchSizeBytes);await this.#g(s);let r={totalFileCount:0,successFileCount:0,skippedFileCount:0,failedFileCount:0,cancelledFileCount:0,completedByteCount:0},i=[];f.start();try{for(let a of s){let l=await this.#n({batch:a,totalBatches:s.length,overallFilesTotal:t.length,overallBytesTotal:this.#f(t),completedTotals:r});this.#U(r,i,l,a.totalSize)}}finally{f.stop()}this.#h({summary:{...r,failedFiles:i},extraLines:[`Batches: ${s.length}`]});let o={...r,failedFiles:i};await this.#r(o),this.#b(o)}async runCheckOnly(){console.log("Running in check-only mode (no downloads/uploads)..."),await ye(),console.log(`Listing remote files from ${this.#t.remote}:${this.#t.remotePath}...`);let e=await V(this.#t.remote,this.#t.remotePath,this.#t.rcloneOptions.extraRcloneArgs??[]);console.log(`Found ${e.length} remote files to check`);let t=this.#o(e);if(t.length===0){console.log("No remote files to check. Exiting.");return}console.log("Fetching existing directory structure..."),await this.#e.loadExistingDirectories();let s=[],r=[];for(let a of t){let l=k(a),u=b(l),c=this.#e.getExistingDirectoryIdForFile(u);if(!c){r.push(l);continue}s.push({displayPath:l,request:{directory_id:c,name:re.posix.basename(u),size_bytes:a.size}})}let i=await this.#e.checkAssetsExistence(s.map(a=>a.request),!1),o=0;for(let a=0;a<s.length;a++)i[a]?.exists?o+=1:r.push(s[a].displayPath);this.#D({totalRemoteCount:e.length,existingCount:o,missingFiles:r})}#o(e){let t=Pe(e);if(t.changes.length===0)return t.files;let s=t.changes.slice(0,10).map(o=>`${o.sourcePath} -> ${o.targetPath}`).join(`
+`))}#A(e,t){let s=[`${m.green(S(t.bytesTransferred))} / ${m.gray(S(t.totalBytes))}`,m.cyan(`${t.percentComplete.toFixed(1)}%`),m.magenta(Be(t.speed)??"..."),`${m.yellow("ETA")}: ${m.yellow(H(t.eta)??"...")}`];return`${m.blue(e)}: ${s.join(" | ")}`}#g(){let e=this.#c();return[this.#m(e),this.#E(e)]}#k(e){let t=[];for(let s=1;s<e.currentBatch;s++){let r=this.#n.get(s);r&&t.push(r)}return t.push(this.#h(e)),t.push(this.#U(e)),t}#h(e){let t=`Batch ${e.currentBatch}/${e.totalBatches}`;switch(e.currentPhase){case"downloading":return this.#o?`${t}: ${this.#A("Downloading",this.#o)}`:`${t}: ${m.blue("Downloading...")}`;case"scanning":return`${t}: ${m.yellow(e.extraDetails??"Scanning...")}`;case"syncing_folders":return`${t}: ${m.yellow(e.extraDetails??"Syncing folders...")}`;case"uploading":return this.#D(t,e);case"cleaning":return`${t}: ${m.yellow("Cleaning...")}`;case"complete":return`${t}: ${m.green("Complete")} (${e.batchFilesTotal} files, ${S(e.batchBytesTotal)}${e.extraDetails?`, ${e.extraDetails}`:""})`}}#D(e,t){let s=this.#c(),r=s.successFileCount+s.failedFileCount+s.cancelledFileCount,i=Math.max(t.batchFilesTotal-t.batchFilesSkipped,0),o=Math.max(t.batchBytesTotal-t.batchBytesSkipped,0),a=o>0?(s.uploadedBytes/o*100).toFixed(1):"0.0",l=this.#f(t,s,o),u=l?.formattedSpeed??null,c=l?.formattedTime??null,d=u&&c?`, ${u}, ETA: ${c}`:"",h=t.batchFilesSkipped>0?`, ${m.yellow(`Skipped: ${t.batchFilesSkipped} files / ${S(t.batchBytesSkipped)}`)}`:"",p=o>0&&s.uploadedBytes>=o&&r<i?"Finalizing":"Uploading";return`${e}: ${m.blue(p)} (${r}/${i} files, ${S(s.uploadedBytes)} / ${S(o)}, ${a}%${d}${h})`}#U(e){let t=this.#c(),s=e.batchFilesSkipped+t.successFileCount+t.failedFileCount+t.cancelledFileCount,r=e.batchBytesSkipped+t.uploadedBytes,i=e.overallFilesCompleted+s,o=e.overallBytesCompleted+r,a=Math.max(e.overallBytesTotal-o,0),l=this.#b(e),u=this.#P(e,t),c=l!==null&&u!==null?l+u:null;return[`Total: ${i}/${e.overallFilesTotal} files`,`${S(o)} / ${S(e.overallBytesTotal)}`,m.yellow(`${S(a)} remaining`),`Download ETA: ${H(l)??"..."}`,`Upload ETA: ${H(u)??"..."}`,`Total ETA: ${H(c)??"..."}`].join(", ")}#b(e){let t=e.overallBytesCompleted+e.batchBytesSkipped+(this.#o?.bytesTransferred??0),s=Math.max(e.overallBytesTotal-t,0);if(s===0)return e.currentPhase==="downloading"&&this.#o?(this.#a=this.#o.eta,this.#o.eta):(this.#a=0,0);if(this.#o&&this.#o.speed>0){let r=Math.ceil(s/this.#o.speed);return this.#a=r,r}return this.#a}#P(e,t){let s=e.overallBytesCompleted+e.batchBytesSkipped+t.uploadedBytes,r=Math.max(e.overallBytesTotal-s,0),i=Math.max(e.batchBytesTotal-e.batchBytesSkipped,0),o=this.#f(e,t,i);if(r===0)return e.currentPhase==="uploading"&&o?(this.#l=o.timeRemainingSeconds,o.timeRemainingSeconds):(this.#l=0,0);if(o&&o.uploadSpeedMbps>0){let a=o.uploadSpeedMbps*1024*1024/8,l=Math.ceil(r/a);return this.#l=l,l}return this.#l}#f(e,t,s){let r=this.#e?.uploadStats,i=s>0&&t.uploadedBytes>=s;return r&&r.uploadSpeedMbps>0&&r.formattedSpeed!=="Calculating..."&&!i?(this.#u={uploadSpeedMbps:r.uploadSpeedMbps,formattedSpeed:r.formattedSpeed,formattedTime:r.formattedTime,timeRemainingSeconds:r.timeRemainingSeconds},this.#u):e.currentPhase==="uploading"&&i&&this.#u?this.#u:r?{uploadSpeedMbps:r.uploadSpeedMbps,formattedSpeed:r.formattedSpeed,formattedTime:r.formattedTime,timeRemainingSeconds:r.timeRemainingSeconds}:null}#m(e){return[`Total: ${e.totalFileCount}`,m.green(`Success: ${e.successFileCount}`),m.red(`Failed: ${e.failedFileCount}`),m.red(`Cancelled: ${e.cancelledFileCount}`),m.blue(`Active: ${e.activeFileCount}`),m.gray(`Queued: ${e.queuedFileCount}`)].join(" | ")}#E(e){let t=Math.max(e.totalBytes-e.uploadedBytes,0),s=this.#e?` | ${m.cyan(`Speed: ${this.#e.uploadStats.formattedSpeed}`)} | ${m.magenta(`Time remaining: ${this.#e.uploadStats.formattedTime}`)}`:"";return[`Total: ${S(e.totalBytes)}`,m.green(`Uploaded: ${S(e.uploadedBytes)}`),m.yellow(`Remaining: ${S(t)}`)].join(" | ")+s}#c(){return this.#e?Object.values(this.#e.uploadAssets).reduce((t,s)=>{switch(t.totalFileCount+=1,t.totalBytes+=s.totalBytesToUpload,t.uploadedBytes+=s.status==="success"?s.totalBytesToUpload:s.bytesUploaded,s.status){case"success":t.successFileCount+=1;break;case"failed":t.failedFileCount+=1;break;case"cancelled":t.cancelledFileCount+=1;break;case"in-progress":t.activeFileCount+=1;break;default:t.queuedFileCount+=1;break}return t},{totalFileCount:0,successFileCount:0,failedFileCount:0,cancelledFileCount:0,activeFileCount:0,queuedFileCount:0,uploadedBytes:0,totalBytes:0}):{totalFileCount:0,successFileCount:0,failedFileCount:0,cancelledFileCount:0,activeFileCount:0,queuedFileCount:0,uploadedBytes:0,totalBytes:0}}},f=new Se;import{Readable as xt}from"node:stream";import Le from"axios";var U=class{#t;#s;#e;#i;#o=new M;#n=new Set;constructor(e){this.#t=Le.create({baseURL:e.apiUrl}),this.#s=Le.create({baseURL:e.edgeWorkerUrl}),this.#e=e.apiKey,this.#i=e.sessionId}get skippedAssetIds(){return this.#n}async post(e,t,s){let r=await this.#t.post(e,t,{headers:{Authorization:`Bearer ${this.#e}`,"X-Aspect-Session-Id":this.#i,"Content-Type":"application/json",...s}});return this.#l(e,t,r.data),r.data}async get(e,t){return(await this.#t.get(e,{headers:{Authorization:`Bearer ${this.#e}`,"X-Aspect-Session-Id":this.#i,...t}})).data}async delete(e,t){await this.#t.delete(e,{headers:{Authorization:`Bearer ${this.#e}`,"X-Aspect-Session-Id":this.#i,...t}})}async putWithToken(e,t,s,r){let i=await this.#a(t),o=this.#o.createIterator(i,r.getMaxUploadRate??(()=>null),r.signal),a=xt.from(o,{highWaterMark:1}),l=await this.#s.put(e,a,{timeout:0,signal:r.signal,headers:{"Content-Type":r.contentType??"","Content-Length":i.byteLength.toString(),Authorization:`Bearer ${s}`},transformRequest:[u=>u],maxRedirects:0,onUploadProgress:u=>r.onProgress?.(u.loaded)});return{status:l.status,statusText:l.statusText,headers:Object.fromEntries(Object.entries(l.headers)),data:l.data,raw:l}}async#a(e){if(e instanceof Uint8Array)return e;let t=await e.arrayBuffer();return new Uint8Array(t)}#l(e,t,s){if(e!=="/assets/chunked/initiate"||!Array.isArray(t)||!Array.isArray(s))return;let r=t;s.forEach((o,a)=>{let l=r[a];o.skipped===!0&&typeof l?.id=="string"&&this.#n.add(l.id)})}};import z from"node:path";var J=class{#t;#s;#e=new Map;constructor(e){this.#t=e.rootDirectoryId,this.#s=e.httpClient}async ensureDirectories(e){await this.loadExistingDirectories();let t=this.#i(e);for(let s of t){if(this.#e.has(s))continue;let r=z.posix.dirname(s),i=z.posix.basename(s),o=r==="."?this.#t:this.#e.get(r);if(!o)throw new Error(`Parent directory ID not found for path: ${s}`);let a=await this.#a(o,i);if(a.name!==i)throw new Error(`Server created renamed directory "${a.name}" for planned path "${s}". Refusing to map uploads to an unexpected target path.`);this.#e.set(s,a.id)}return e.map(s=>({file:s,directoryId:this.getDirectoryIdForFile(s.relativePath)}))}async loadExistingDirectories(){let e=await this.#o(this.#t);this.#e.clear(),this.#n(e,"")}getDirectoryIdForFile(e){let t=b(e),s=z.posix.dirname(t);if(s===".")return this.#t;let r=this.#e.get(s);if(!r)throw new Error(`Directory ID not found for path: ${s}`);return r}getExistingDirectoryIdForFile(e){let t=b(e),s=z.posix.dirname(t);return s==="."?this.#t:this.#e.get(s)??null}async checkAssetsExistence(e,t=!0){return e.length===0?[]:(await this.#s.post("/assets/check-exists-and-uploaded",{items:e,delete_if_not_exist:t})).items}#i(e){let t=new Set;for(let s of e){let r=b(s.relativePath),i=z.posix.dirname(r);if(i===".")continue;let o=i.split("/");for(let a=0;a<o.length;a++)t.add(o.slice(0,a+1).join("/"))}return Array.from(t).sort((s,r)=>s.split("/").length-r.split("/").length)}async#o(e){return this.#s.get(`/directories/${e}/tree`)}#n(e,t){let s=t===""?"":t==="."?e.name:`${t}/${e.name}`;s!==""&&this.#e.set(s,e.id);for(let r of e.children){let i=t===""?".":s;this.#n(r,i)}}async#a(e,t){return this.#s.post(`/directories/${e}/directories`,{name:t,auto_rename:"numeric"})}};import{PostHog as Dt}from"posthog-node";var P=null,Ce="",Me={};function Ne(n){P||n.config.analytics.disabled||!n.config.analytics.posthogKey||(Ce=n.user.id,Me={service:"data-sync",package_version:n.packageVersion,platform:process.platform,arch:process.arch,node_version:process.version,max_concurrent:n.config.maxConcurrent,batch_mode:n.config.batchSizeBytes!==void 0,batch_size_bytes:n.config.batchSizeBytes??null,rclone_transfers:n.config.rcloneOptions.transfers,rclone_multi_thread_streams:n.config.rcloneOptions.multiThreadStreams},P=new Dt(n.config.analytics.posthogKey,{host:n.config.analytics.posthogHost}),P.identify({distinctId:Ce,properties:{user_id:n.user.id,email:n.user.email,first_name:n.user.first_name,last_name:n.user.last_name,utm_source_latest:null,utm_medium_latest:null,utm_campaign_latest:null,utm_content_latest:null}}))}function Ge(n,e){P&&P.capture({distinctId:Ce,event:n,properties:{...Me,...e}})}async function He(){if(!P)return;let n=P;P=null;try{await n.flush()}catch{}}import*as ee from"fs/promises";import*as te from"path";var Bt={".mp4":"video/mp4",".mov":"video/quicktime",".avi":"video/x-msvideo",".mkv":"video/x-matroska",".webm":"video/webm",".wmv":"video/x-ms-wmv",".flv":"video/x-flv",".m4v":"video/x-m4v",".mpg":"video/mpeg",".mpeg":"video/mpeg",".3gp":"video/3gpp",".ts":"video/mp2t",".mts":"video/mp2t",".mxf":"application/mxf",".jpg":"image/jpeg",".jpeg":"image/jpeg",".png":"image/png",".gif":"image/gif",".webp":"image/webp",".bmp":"image/bmp",".tiff":"image/tiff",".tif":"image/tiff",".svg":"image/svg+xml",".heic":"image/heic",".heif":"image/heif",".avif":"image/avif",".mp3":"audio/mpeg",".wav":"audio/wav",".aac":"audio/aac",".ogg":"audio/ogg",".flac":"audio/flac",".m4a":"audio/mp4",".wma":"audio/x-ms-wma",".pdf":"application/pdf",".doc":"application/msword",".docx":"application/vnd.openxmlformats-officedocument.wordprocessingml.document",".ppt":"application/vnd.ms-powerpoint",".pptx":"application/vnd.openxmlformats-officedocument.presentationml.presentation",".xls":"application/vnd.ms-excel",".xlsx":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",".rtf":"application/rtf",".json":"application/json",".xml":"application/xml"};function Ot(n){let e=te.extname(n).toLowerCase();return Bt[e]??""}var Z=class n{#t;#s=null;#e=null;#i=!1;#o;#n;#a;constructor(e,t,s,r){this.#t=e,this.#o=t,this.#n=s,this.#a=r}static async create(e){let t=await ee.stat(e),s=te.basename(e),r=Ot(s);return new n(e,t.size,s,r)}get name(){return this.#n}get size(){return this.#o}get type(){return this.#a}async readChunk(e,t){if(!this.#s){this.#e||(this.#e=ee.open(this.#t,"r"));let i=await this.#e;if(this.#e=null,this.#i)return Buffer.alloc(0);this.#s=i}let s=t-e,r=Buffer.alloc(s);return await this.#s.read(r,0,s,e),r}async close(){this.#i=!0;let e=this.#e;await this.#s?.close(),this.#s=null,e&&await(await e.catch(()=>null))?.close(),this.#e=null}};var Lt=new Set(["success","failed","cancelled"]),ze=1e3,Mt=4*1024*1024,se=class{#t;#s;#e;#i;#o;constructor(e){this.#t=e.config,this.#s=e.httpClient??new U({apiUrl:e.config.apiUrl,apiKey:e.config.apiKey,edgeWorkerUrl:e.config.edgeWorkerUrl,sessionId:e.config.sessionName}),this.#e=e.createFileReader??(t=>Z.create(t)),this.#i=e.onSnapshot,this.#o=e.analytics}async uploadFiles(e){let t=e.filter(o=>o.preSkipped===!0),s=e.filter(o=>o.preSkipped!==!0);if(s.length===0)return{totalFileCount:e.length,successFileCount:0,skippedFileCount:t.length,failedFileCount:0,cancelledFileCount:0,failedFiles:[]};let r=new L({httpClient:this.#s,maxConcurrency:this.#t.maxConcurrent,deltaIntervalMs:ze,adaptiveConcurrency:!1,analytics:this.#o}),i=[];try{let o=r.startGroupUpload({name:`${s.length} files`,type:"files",fileCount:s.length,folderCount:0,topLevelFileCount:s.length,topLevelFolderCount:0,rootDirectoryId:this.#t.directoryId,projectId:this.#t.projectId,totalBytes:s.reduce((l,u)=>l+u.file.size,0),conflictResolutionOverride:"skip"});for(let l of s){let u=await this.#e(l.file.absolutePath),c=r.startUpload({fileReader:u,directoryId:l.directoryId,projectId:this.#t.projectId,groupId:o,chunkSize:Mt});r.addAssetToGroup(o,c),i.push({assetId:c,file:l.file,fileReader:u})}await this.#n(r);let a=r.getSnapshot();return this.#i?.(a),this.#a({snapshot:a,startedUploads:i,preSkippedCount:t.length})}finally{r.destroy()}}async#n(e){let s=Date.now();for(;Date.now()-s<864e5;){let r=e.getSnapshot();this.#i?.(r);let i=Object.values(r.uploadAssets);if(i.length>0&&i.every(o=>Lt.has(o.status)))return;await new Promise(o=>setTimeout(o,ze))}throw new Error("Timed out waiting for upload session to finish")}#a(e){let t=[],s=0,r=0,i=0,o=0;for(let a of e.startedUploads){let l=e.snapshot.uploadAssets[a.assetId];if(!l){i+=1,t.push({fileName:a.file.fileName,relativePath:a.file.relativePath,errorMessage:"Upload state missing from upload engine"});continue}if(l.status==="success"){this.#s.skippedAssetIds.has(a.assetId)?r+=1:s+=1;continue}l.status==="cancelled"?o+=1:i+=1,t.push({fileName:a.file.fileName,relativePath:a.file.relativePath,errorMessage:l.errorMessage})}return{totalFileCount:e.startedUploads.length+e.preSkippedCount,successFileCount:s,skippedFileCount:e.preSkippedCount+r,failedFileCount:i,cancelledFileCount:o,failedFiles:t}}};var ne=class{#t;#s;#e;#i;constructor(e){this.#t=e,this.#s=new U({apiUrl:e.apiUrl,apiKey:e.apiKey,edgeWorkerUrl:e.edgeWorkerUrl,sessionId:e.sessionName}),this.#e=new J({rootDirectoryId:e.directoryId,httpClient:this.#s}),this.#i=new se({config:e,httpClient:this.#s,onSnapshot:t=>f.updateSnapshot(t),analytics:Ge})}async run(){this.#k("Starting data sync..."),await this.#a(),console.log("Listing all remote files...");let e=await V(this.#t.remote,this.#t.remotePath,this.#t.rcloneOptions.extraRcloneArgs??[]);console.log(`Found ${e.length} files`),console.log(`Total size: ${S(this.#f(e))}`),console.log("");let t=this.#o(e);if(t.length===0){console.log("No files to sync. Exiting."),await this.#l();return}let s={totalFileCount:0,successFileCount:0,skippedFileCount:0,failedFileCount:0,cancelledFileCount:0,completedByteCount:0},r=[],i={batchNumber:1,files:t,totalSize:this.#f(t)};f.start();try{let a=await this.#n({batch:i,totalBatches:1,overallFilesTotal:t.length,overallBytesTotal:i.totalSize,completedTotals:s,batchDir:this.#t.localTempDir});this.#U(s,r,a,i.totalSize)}finally{f.stop()}let o={...s,failedFiles:r};this.#h({summary:o}),await this.#r(o),this.#b(o)}async runBatched(){if(!this.#t.batchSizeBytes)throw new Error("Batch size not configured. Use batchSizeBytes in config.");this.#k("Starting batched data sync...",[`Batch size: ${S(this.#t.batchSizeBytes)}`,`Max concurrent chunks: ${this.#t.maxConcurrent}`]),await this.#a(),console.log("Listing all remote files...");let e=await V(this.#t.remote,this.#t.remotePath,this.#t.rcloneOptions.extraRcloneArgs??[]);console.log(`Found ${e.length} files`),console.log(`Total size: ${S(this.#f(e))}`),console.log("");let t=this.#o(e);if(t.length===0){console.log("No files to sync. Exiting."),await this.#l();return}let s=xe(t,this.#t.batchSizeBytes);await this.#g(s);let r={totalFileCount:0,successFileCount:0,skippedFileCount:0,failedFileCount:0,cancelledFileCount:0,completedByteCount:0},i=[];f.start();try{for(let a of s){let l=await this.#n({batch:a,totalBatches:s.length,overallFilesTotal:t.length,overallBytesTotal:this.#f(t),completedTotals:r});this.#U(r,i,l,a.totalSize)}}finally{f.stop()}this.#h({summary:{...r,failedFiles:i},extraLines:[`Batches: ${s.length}`]});let o={...r,failedFiles:i};await this.#r(o),this.#b(o)}async runCheckOnly(){console.log("Running in check-only mode (no downloads/uploads)..."),await ye(),console.log(`Listing remote files from ${this.#t.remote}:${this.#t.remotePath}...`);let e=await V(this.#t.remote,this.#t.remotePath,this.#t.rcloneOptions.extraRcloneArgs??[]);console.log(`Found ${e.length} remote files to check`);let t=this.#o(e);if(t.length===0){console.log("No remote files to check. Exiting.");return}console.log("Fetching existing directory structure..."),await this.#e.loadExistingDirectories();let s=[],r=[];for(let a of t){let l=k(a),u=b(l),c=this.#e.getExistingDirectoryIdForFile(u);if(!c){r.push(l);continue}s.push({displayPath:l,request:{directory_id:c,name:re.posix.basename(u),size_bytes:a.size}})}let i=await this.#e.checkAssetsExistence(s.map(a=>a.request),!1),o=0;for(let a=0;a<s.length;a++)i[a]?.exists?o+=1:r.push(s[a].displayPath);this.#D({totalRemoteCount:e.length,existingCount:o,missingFiles:r})}#o(e){let t=Pe(e);if(t.changes.length===0)return t.files;let s=t.changes.slice(0,10).map(o=>`${o.sourcePath} -> ${o.targetPath}`).join(`
 `),r=t.changes.length-10,i=r>0?`
 ...and ${r} more renamed target path(s)`:"";return console.warn(`Warning: ${t.changes.length} remote file path(s) were renamed for Aspect compatibility.
 ${s}${i}
 `),console.log(""),t.files}async#n(e){let t=e.batchDir??re.join(this.#t.localTempDir,`batch${e.batch.batchNumber}`);await $.mkdir(t,{recursive:!0});let s=e.batch.files.map(d=>Ie(d,t));f.updateBatchState({currentBatch:e.batch.batchNumber,totalBatches:e.totalBatches,batchFilesTotal:s.length,batchBytesTotal:e.batch.totalSize,batchFilesSkipped:0,batchBytesSkipped:0,overallFilesCompleted:e.completedTotals.successFileCount+e.completedTotals.skippedFileCount,overallFilesTotal:e.overallFilesTotal,overallBytesCompleted:this.#m(e.completedTotals,e.overallBytesTotal),overallBytesTotal:e.overallBytesTotal,currentPhase:"scanning",extraDetails:`Preparing ${s.length} files...`}),f.updateBatchState({currentBatch:e.batch.batchNumber,totalBatches:e.totalBatches,batchFilesTotal:s.length,batchBytesTotal:e.batch.totalSize,batchFilesSkipped:0,batchBytesSkipped:0,overallFilesCompleted:e.completedTotals.successFileCount+e.completedTotals.skippedFileCount,overallFilesTotal:e.overallFilesTotal,overallBytesCompleted:this.#m(e.completedTotals,e.overallBytesTotal),overallBytesTotal:e.overallBytesTotal,currentPhase:"syncing_folders",extraDetails:"Ensuring target directories exist..."});let r=await this.#e.ensureDirectories(s);f.updateBatchState({currentBatch:e.batch.batchNumber,totalBatches:e.totalBatches,batchFilesTotal:s.length,batchBytesTotal:e.batch.totalSize,batchFilesSkipped:0,batchBytesSkipped:0,overallFilesCompleted:e.completedTotals.successFileCount+e.completedTotals.skippedFileCount,overallFilesTotal:e.overallFilesTotal,overallBytesCompleted:this.#m(e.completedTotals,e.overallBytesTotal),overallBytesTotal:e.overallBytesTotal,currentPhase:"scanning",extraDetails:"Checking existing assets..."});let i=await this.#u(r),o=this.#A(i),a=e.batch.files.filter((d,h)=>i[h]?.preSkipped!==!0),l=re.join(this.#t.localTempDir,`batch${e.batch.batchNumber}.txt`),u=async()=>{await De(l),this.#t.keepLocal||await $.rm(t,{recursive:!0,force:!0}),f.resetUploadSnapshot(),f.updateDownloadProgress(null)};if(a.length===0){let d={totalFileCount:i.length,successFileCount:0,skippedFileCount:i.length,failedFileCount:0,cancelledFileCount:0,failedFiles:[]};return f.updateBatchState({currentBatch:e.batch.batchNumber,totalBatches:e.totalBatches,batchFilesTotal:s.length,batchBytesTotal:e.batch.totalSize,batchFilesSkipped:o.skippedFileCount,batchBytesSkipped:o.skippedByteCount,overallFilesCompleted:e.completedTotals.successFileCount+e.completedTotals.skippedFileCount+d.skippedFileCount,overallFilesTotal:e.overallFilesTotal,overallBytesCompleted:this.#m(e.completedTotals,e.overallBytesTotal)+e.batch.totalSize,overallBytesTotal:e.overallBytesTotal,currentPhase:"complete",extraDetails:"everything already synced"}),f.persistCurrentBatchLine(),await u(),d}f.updateBatchState({currentBatch:e.batch.batchNumber,totalBatches:e.totalBatches,batchFilesTotal:s.length,batchBytesTotal:e.batch.totalSize,batchFilesSkipped:o.skippedFileCount,batchBytesSkipped:o.skippedByteCount,overallFilesCompleted:e.completedTotals.successFileCount+e.completedTotals.skippedFileCount,overallFilesTotal:e.overallFilesTotal,overallBytesCompleted:this.#m(e.completedTotals,e.overallBytesTotal),overallBytesTotal:e.overallBytesTotal,currentPhase:"downloading"}),await ke({remote:this.#t.remote,remotePath:this.#t.remotePath,localPath:t,batchFilePath:l,files:a,rcloneOptions:this.#t.rcloneOptions,onProgress:d=>f.updateDownloadProgress(d)}),await we(t),f.updateBatchState({currentBatch:e.batch.batchNumber,totalBatches:e.totalBatches,batchFilesTotal:s.length,batchBytesTotal:e.batch.totalSize,batchFilesSkipped:o.skippedFileCount,batchBytesSkipped:o.skippedByteCount,overallFilesCompleted:e.completedTotals.successFileCount+e.completedTotals.skippedFileCount,overallFilesTotal:e.overallFilesTotal,overallBytesCompleted:this.#m(e.completedTotals,e.overallBytesTotal),overallBytesTotal:e.overallBytesTotal,currentPhase:"uploading"});let c=await this.#i.uploadFiles(i);return this.#P(c)?(f.updateBatchState({currentBatch:e.batch.batchNumber,totalBatches:e.totalBatches,batchFilesTotal:s.length,batchBytesTotal:e.batch.totalSize,batchFilesSkipped:o.skippedFileCount,batchBytesSkipped:o.skippedByteCount,overallFilesCompleted:e.completedTotals.successFileCount+e.completedTotals.skippedFileCount+c.successFileCount+c.skippedFileCount,overallFilesTotal:e.overallFilesTotal,overallBytesCompleted:this.#m(e.completedTotals,e.overallBytesTotal)+e.batch.totalSize,overallBytesTotal:e.overallBytesTotal,currentPhase:"complete",extraDetails:"failed; local files preserved"}),f.persistCurrentBatchLine(),c):(f.updateBatchState({currentBatch:e.batch.batchNumber,totalBatches:e.totalBatches,batchFilesTotal:s.length,batchBytesTotal:e.batch.totalSize,batchFilesSkipped:o.skippedFileCount,batchBytesSkipped:o.skippedByteCount,overallFilesCompleted:e.completedTotals.successFileCount+e.completedTotals.skippedFileCount+c.successFileCount+c.skippedFileCount,overallFilesTotal:e.overallFilesTotal,overallBytesCompleted:this.#m(e.completedTotals,e.overallBytesTotal)+e.batch.totalSize,overallBytesTotal:e.overallBytesTotal,currentPhase:"cleaning"}),await u(),f.updateBatchState({currentBatch:e.batch.batchNumber,totalBatches:e.totalBatches,batchFilesTotal:s.length,batchBytesTotal:e.batch.totalSize,batchFilesSkipped:o.skippedFileCount,batchBytesSkipped:o.skippedByteCount,overallFilesCompleted:e.completedTotals.successFileCount+e.completedTotals.skippedFileCount+c.successFileCount+c.skippedFileCount,overallFilesTotal:e.overallFilesTotal,overallBytesCompleted:this.#m(e.completedTotals,e.overallBytesTotal)+e.batch.totalSize,overallBytesTotal:e.overallBytesTotal,currentPhase:"complete"}),f.persistCurrentBatchLine(),c)}async#a(){console.log("Checking rclone installation..."),await ye(),console.log("rclone is installed"),console.log(""),console.log("Creating session directory..."),await $.mkdir(this.#t.localTempDir,{recursive:!0}),console.log(`Session directory: ${this.#t.localTempDir}`),console.log("")}async#l(){this.#t.keepLocal||(console.log(""),console.log("Cleaning up session directory..."),await $.rm(this.#t.localTempDir,{recursive:!0,force:!0}),console.log(`Deleted session directory: ${this.#t.localTempDir}`))}async#r(e){if(this.#P(e)){console.log(""),console.log(`Preserving session directory for retry/debugging: ${this.#t.localTempDir}`);return}await this.#l()}async#u(e){let t=await this.#e.checkAssetsExistence(e.map(s=>({directory_id:s.directoryId,name:s.file.fileName,size_bytes:s.file.size})));return e.map((s,r)=>({file:s.file,directoryId:s.directoryId,preSkipped:t[r]?.exists===!0}))}#A(e){return e.reduce((t,s)=>(s.preSkipped===!0&&(t.skippedFileCount+=1,t.skippedByteCount+=s.file.size),t),{skippedFileCount:0,skippedByteCount:0})}async#g(e){console.log("Creating batches..."),console.log(`Created ${e.length} batches`);let t=[];for(let s of e){let r=`Batch ${s.batchNumber}: ${s.files.length} files, ${S(s.totalSize)}`;t.push(r),console.log(r)}await $.writeFile(re.join(this.#t.localTempDir,"batches.txt"),`${t.join(`
 `)}
-`,"utf-8"),console.log("")}#k(e,t=[]){console.log(e),console.log(`Session: ${this.#t.sessionName}`),console.log(`Remote: ${this.#t.remote}:${this.#t.remotePath}`),console.log(`Root directory ID: ${this.#t.directoryId}`),console.log(`Project ID: ${this.#t.projectId}`),console.log(`Temp directory: ${this.#t.localTempDir}`),console.log(`Edge worker URL: ${this.#t.edgeWorkerUrl}`);for(let s of t)console.log(s);console.log("")}#h(e){console.log(""),console.log("=".repeat(60)),console.log("Sync Summary"),console.log("=".repeat(60)),console.log(`Total files: ${e.summary.totalFileCount}`),console.log(`Successful: ${e.summary.successFileCount}`),console.log(`Skipped: ${e.summary.skippedFileCount}`),console.log(`Failed: ${e.summary.failedFileCount}`),console.log(`Cancelled: ${e.summary.cancelledFileCount}`);for(let t of e.extraLines??[])console.log(t);if(console.log("=".repeat(60)),e.summary.failedFiles.length>0){console.log(""),console.log("Failed files:");for(let t of e.summary.failedFiles)console.log(`  - ${t.relativePath}: ${t.errorMessage??"Unknown error"}`)}}#D(e){if(console.log(""),console.log("=".repeat(60)),console.log("Check Summary"),console.log("=".repeat(60)),console.log(`Total (remote): ${e.totalRemoteCount}`),console.log(`Already exist: ${e.existingCount}`),console.log(`Missing: ${e.missingFiles.length}`),console.log("=".repeat(60)),e.missingFiles.length>0){console.log(""),console.log("Files missing on server:");for(let t of e.missingFiles)console.log(`  - ${t}`)}}#U(e,t,s,r){e.totalFileCount+=s.totalFileCount,e.successFileCount+=s.successFileCount,e.skippedFileCount+=s.skippedFileCount,e.failedFileCount+=s.failedFileCount,e.cancelledFileCount+=s.cancelledFileCount,e.completedByteCount+=r,t.push(...s.failedFiles)}#b(e){let t=e.failedFileCount+e.cancelledFileCount;if(t>0)throw new Error(`Sync completed with ${t} unsuccessful upload(s)`)}#P(e){return e.failedFileCount+e.cancelledFileCount>0}#f(e){return e.reduce((t,s)=>t+s.size,0)}#m(e,t){return Math.min(e.completedByteCount,t)}};var zt=Ht(import.meta.url),$t=$e.dirname(zt),je=JSON.parse(Gt($e.join($t,"..","package.json"),"utf-8")),be=new Nt;be.name("aspect-sync").description("Sync files from external services to Aspect via rclone").version(je.version);be.requiredOption("--remote <remote>","rclone remote name (e.g., dropbox)").requiredOption("--path <path>","remote path to sync from").requiredOption("--directory-id <id>","Aspect directory ID to upload to").requiredOption("--project-id <id>","Aspect project ID").option("--api-url <url>","Aspect API URL").option("--edge-worker-url <url>","Aspect edge worker URL").option("--api-key <key>","Aspect API key").option("--upload-concurrent <number>","Maximum concurrent chunk uploads to Aspect (default: 16)","16").option("--keep-local","Keep local files after upload (for debugging)",!1).option("--temp-dir <path>","Local temporary directory for synced files").option("--session <name>","Session name for isolation (default: auto-generated timestamp)").option("--check","Check remote files against Aspect without downloading/uploading",!1).option("--batch-size <size>","Enable batched mode with max batch size (e.g., 500GB, 1TB). Helps with large migrations.","").option("--rclone-transfers <number>","Number of parallel file transfers (default: 4)","4").option("--rclone-multi-thread-streams <number>","Streams per large file (default: 4, higher = faster large files)","4").option("--rclone-multi-thread-chunk-size <size>","Chunk size for multi-threading (default: 64MiB)","64MiB").option("--rclone-multi-thread-cutoff <size>","Minimum file size for multi-threading (default: 64MiB)","64MiB").option("--rclone-buffer-size <size>","Buffer size per transfer (default: 64M, memory = transfers \xD7 buffer-size)","64M").option("--rclone-no-mmap","Disable memory-mapped I/O (reduces memory efficiency)",!1).option("--rclone-extra <arg>","Additional rclone argument to append (repeatable)",(n,e)=>(e??[]).concat(n),[]).action(async n=>{let e=0;try{let t={ASPECT_API_URL:process.env.ASPECT_API_URL,ASPECT_EDGE_WORKER_URL:process.env.ASPECT_EDGE_WORKER_URL,ASPECT_API_KEY:process.env.ASPECT_API_KEY,ASPECT_POSTHOG_KEY:process.env.ASPECT_POSTHOG_KEY,ASPECT_POSTHOG_HOST:process.env.ASPECT_POSTHOG_HOST,ASPECT_DISABLE_POSTHOG:process.env.ASPECT_DISABLE_POSTHOG,POSTHOG_KEY:process.env.POSTHOG_KEY,POSTHOG_API_KEY:process.env.POSTHOG_API_KEY,POSTHOG_HOST:process.env.POSTHOG_HOST,POSTHOG_DISABLED:process.env.POSTHOG_DISABLED},s=Te({options:n,environment:t}),r=await jt(s);r&&Ne({config:s,packageVersion:je.version,user:r});let i=new ne(s);n.check?await i.runCheckOnly():s.batchSizeBytes!==void 0?await i.runBatched():await i.run()}catch(t){console.error("Fatal error:",t),e=1}finally{await He()}process.exit(e)});be.parse();async function jt(n){if(n.analytics.disabled||!n.analytics.posthogKey)return null;try{return await new U({apiUrl:n.apiUrl,apiKey:n.apiKey,edgeWorkerUrl:n.edgeWorkerUrl,sessionId:n.sessionName}).get("/users/me")}catch{return null}}
+`,"utf-8"),console.log("")}#k(e,t=[]){console.log(e),console.log(`Session: ${this.#t.sessionName}`),console.log(`Remote: ${this.#t.remote}:${this.#t.remotePath}`),console.log(`Root directory ID: ${this.#t.directoryId}`),console.log(`Project ID: ${this.#t.projectId}`),console.log(`Temp directory: ${this.#t.localTempDir}`),console.log(`Edge worker URL: ${this.#t.edgeWorkerUrl}`);for(let s of t)console.log(s);console.log("")}#h(e){console.log(""),console.log("=".repeat(60)),console.log("Sync Summary"),console.log("=".repeat(60)),console.log(`Total files: ${e.summary.totalFileCount}`),console.log(`Successful: ${e.summary.successFileCount}`),console.log(`Skipped: ${e.summary.skippedFileCount}`),console.log(`Failed: ${e.summary.failedFileCount}`),console.log(`Cancelled: ${e.summary.cancelledFileCount}`);for(let t of e.extraLines??[])console.log(t);if(console.log("=".repeat(60)),e.summary.failedFiles.length>0){console.log(""),console.log("Failed files:");for(let t of e.summary.failedFiles)console.log(`  - ${t.relativePath}: ${t.errorMessage??"Unknown error"}`)}}#D(e){if(console.log(""),console.log("=".repeat(60)),console.log("Check Summary"),console.log("=".repeat(60)),console.log(`Total (remote): ${e.totalRemoteCount}`),console.log(`Already exist: ${e.existingCount}`),console.log(`Missing: ${e.missingFiles.length}`),console.log("=".repeat(60)),e.missingFiles.length>0){console.log(""),console.log("Files missing on server:");for(let t of e.missingFiles)console.log(`  - ${t}`)}}#U(e,t,s,r){e.totalFileCount+=s.totalFileCount,e.successFileCount+=s.successFileCount,e.skippedFileCount+=s.skippedFileCount,e.failedFileCount+=s.failedFileCount,e.cancelledFileCount+=s.cancelledFileCount,e.completedByteCount+=r,t.push(...s.failedFiles)}#b(e){let t=e.failedFileCount+e.cancelledFileCount;if(t>0)throw new Error(`Sync completed with ${t} unsuccessful upload(s)`)}#P(e){return e.failedFileCount+e.cancelledFileCount>0}#f(e){return e.reduce((t,s)=>t+s.size,0)}#m(e,t){return Math.min(e.completedByteCount,t)}};var zt=Ht(import.meta.url),$t=$e.dirname(zt),je=JSON.parse(Gt($e.join($t,"..","package.json"),"utf-8")),be=new Nt;be.name("aspect-sync").description("Sync files from external services to Aspect via rclone").version(je.version);be.requiredOption("--remote <remote>","rclone remote name (e.g., dropbox)").requiredOption("--path <path>","remote path to sync from").requiredOption("--directory-id <id>","Aspect directory ID to upload to").requiredOption("--project-id <id>","Aspect project ID").option("--api-url <url>","Aspect API URL").option("--edge-worker-url <url>","Aspect edge worker URL").option("--api-key <key>","Aspect API key").option("--upload-concurrent <number>","Maximum concurrent chunk uploads to Aspect (default: 16)","16").option("--keep-local","Keep local files after upload (for debugging)",!1).option("--temp-dir <path>","Local temporary directory for synced files").option("--session <name>","Session name for isolation (default: auto-generated timestamp)").option("--check","Check remote files against Aspect without downloading/uploading",!1).option("--batch-size <size>","Enable batched mode with max batch size (e.g., 500GB, 1TB). Helps with large migrations.","").option("--rclone-transfers <number>","Number of parallel file transfers (default: 4)","4").option("--rclone-multi-thread-streams <number>","Streams per large file (default: 4, higher = faster large files)","4").option("--rclone-multi-thread-chunk-size <size>","Chunk size for multi-threading (default: 64MiB)","64MiB").option("--rclone-multi-thread-cutoff <size>","Minimum file size for multi-threading (default: 64MiB)","64MiB").option("--rclone-buffer-size <size>","Buffer size per transfer (default: 64M, memory = transfers \xD7 buffer-size)","64M").option("--rclone-no-mmap","Disable memory-mapped I/O (reduces memory efficiency)",!1).option("--rclone-extra <arg>","Additional rclone argument to append (repeatable)",(n,e)=>(e??[]).concat(n),[]).action(async n=>{let e=0;try{let t={ASPECT_API_URL:process.env.ASPECT_API_URL,ASPECT_EDGE_WORKER_URL:process.env.ASPECT_EDGE_WORKER_URL,ASPECT_API_KEY:process.env.ASPECT_API_KEY,ASPECT_POSTHOG_KEY:process.env.ASPECT_POSTHOG_KEY,ASPECT_POSTHOG_HOST:process.env.ASPECT_POSTHOG_HOST,ASPECT_DISABLE_POSTHOG:process.env.ASPECT_DISABLE_POSTHOG,POSTHOG_KEY:process.env.POSTHOG_KEY,POSTHOG_API_KEY:process.env.POSTHOG_API_KEY,POSTHOG_HOST:process.env.POSTHOG_HOST,POSTHOG_DISABLED:process.env.POSTHOG_DISABLED},s=Te({options:n,environment:t}),r=await jt(s);r&&Ne({config:s,packageVersion:je.version,user:r});let i=new ne(s);n.check?await i.runCheckOnly():s.batchSizeBytes!==void 0?await i.runBatched():await i.run()}catch(t){console.error("Fatal error:",t),e=1}finally{await He()}process.exit(e)});be.parse();async function jt(n){return n.analytics.disabled||!n.analytics.posthogKey?null:await new U({apiUrl:n.apiUrl,apiKey:n.apiKey,edgeWorkerUrl:n.edgeWorkerUrl,sessionId:n.sessionName}).get("/users/me")}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aspect-sync",
-  "version": "0.1.18",
+  "version": "0.1.20",
   "description": "CLI tool to sync files from external services to Aspect via rclone",
   "main": "dist/index.js",
   "bin": {