@eluvio/elv-client-js 4.2.27 → 4.2.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eluvio/elv-client-js",
-  "version": "4.2.27",
+  "version": "4.2.29",
   "description": "Javascript client for the Eluvio Content Fabric",
   "main": "src/index.js",
   "author": "Kevin Talmadge",

package/src/EthClient.js

@@ -309,6 +309,7 @@ class EthClient {
     methodName,
     methodArgs,
     value,
+    overrides={},
     timeout,
     formatArguments=true,
     cacheContract=true,
@@ -319,7 +320,6 @@ class EthClient {
     if(!abi) { abi = await this.client.ContractAbi({contractAddress}); }
 
     const contract = this.Contract({contractAddress, abi, cacheContract, overrideCachedContract});
-
     // Make method call
     const createMethodCall = await this.CallContractMethod({
       contract,
@@ -327,6 +327,7 @@ class EthClient {
       methodName,
       methodArgs,
       value,
+      overrides,
       formatArguments,
       cacheContract
     });

package/src/client/ContentAccess.js

@@ -9,6 +9,7 @@ const objectPath = require("object-path");
 
 const HttpClient = require("../HttpClient");
 const ContentObjectAudit = require("../ContentObjectAudit");
+const bs58 = require("bs58");
 
 const {
   ValidateLibrary,
@@ -17,6 +18,7 @@ const {
   ValidatePartHash,
   ValidateWriteToken,
   ValidateParameters,
+  ValidatePresence,
 } = require("../Validation");
 
 const MergeWith = require("lodash/mergeWith");
@@ -2713,7 +2715,6 @@ exports.LinkData = async function({libraryId, objectId, versionHash, writeToken,
   );
 };
 
-
 /* Encryption */
 
 exports.CreateEncryptionConk = async function({libraryId, objectId, versionHash, writeToken, createKMSConk=true}) {
@@ -2793,6 +2794,74 @@ exports.CreateEncryptionConk = async function({libraryId, objectId, versionHash,
   return this.encryptionConks[objectId];
 };
 
+/**
+ * Encrypt the owner encryption key using the new user details provided
+ *
+ * @methodGroup Encryption
+ * @namedParams
+ * @param {string=} libraryId - ID of a library
+ * @param {string=} objectId - ID of an object
+ * @param {string=} versionHash - Hash of an object version
+ * @param {string=} writeToken - The write token for the object
+ * @param {Object=} newEncodedUserPublicKey - raw public key or base-58 encoded publicKey of the new user prefixed 'kupk'
+ */
+exports.MigrateEncryptionConkForUserProvided = async function({libraryId, objectId, versionHash, writeToken, newEncodedUserPublicKey}) {
+  if(this.signer.remoteSigner) {
+    return;
+  }
+
+  ValidateParameters({libraryId, objectId, versionHash});
+  ValidateWriteToken(writeToken);
+
+  if(!objectId) {
+    objectId = this.utils.DecodeVersionHash(versionHash).objectId;
+  }
+
+  ValidatePresence("newEncodedUserPublicKey", newEncodedUserPublicKey);
+
+  let publicKey;
+  if(!newEncodedUserPublicKey.startsWith("kupk")){
+    publicKey = newEncodedUserPublicKey;
+  } else {
+    const publicKeyBytes = bs58.decode(newEncodedUserPublicKey.slice(4));
+    if(publicKeyBytes.length !== 65) {
+      publicKey = "0x" + Buffer.concat([Buffer.from([0x04]), publicKeyBytes]).toString("hex");
+    } else {
+      publicKey = "0x" + publicKeyBytes.toString("hex");
+    }
+  }
+
+  // Decrypt using existing user key
+  const existingCapKey = `eluv.caps.iusr${this.utils.AddressToHash(this.signer.address)}`;
+  const existingUserCap =
+    await this.ContentObjectMetadata({
+      libraryId,
+      objectId,
+      writeToken,
+      metadataSubtree: existingCapKey
+    });
+
+  if(!existingUserCap) {
+    throw (`No encryption conk present for ${objectId} for user ${this.signer.address}`);
+  }
+
+  this.encryptionConks[objectId] = await this.Crypto.DecryptCap(existingUserCap, this.signer._signingKey().privateKey);
+
+  // Encrypt with the new key
+  const encryptedConk = await this.Crypto.EncryptConk(this.encryptionConks[objectId], publicKey);
+  const newUserCap = `eluv.caps.iusr${this.utils.AddressToHash(this.utils.PublicKeyToAddress(publicKey))}`;
+
+  await this.ReplaceMetadata({
+    libraryId,
+    objectId,
+    writeToken,
+    metadataSubtree: newUserCap,
+    metadata: encryptedConk
+  });
+
+  return this.encryptionConks[objectId];
+};
+
 /**
  * Retrieve the encryption conk for the specified object. If one has not yet been created
  * and a writeToken has been specified, this method will create a new conk and

package/src/client/Files.js

@@ -23,6 +23,13 @@ const {
   ValidateParameters
 } = require("../Validation");
 
+const JobStatus = Object.freeze({
+  STOPPED: "STOPPED",
+  FAILED: "FAILED",
+  IN_PROGRESS:"IN_PROGRESS",
+  COMPLETED: "COMPLETED"
+});
+
 
 /* Files */
 
@@ -52,8 +59,9 @@ exports.ListFiles = async function({libraryId, objectId, path = "", versionHash,
   });
 };
 
+
 /**
- * Copy/reference files from S3 to a content object.
+ * Upload or resume copy/reference files from S3 to a content object
  *
  * S3 authentication is done by either providing an access key and secret or a presigned URL. The credentials will not be stored (neither in the client nor in the Fabric)
  *
@@ -61,12 +69,12 @@ exports.ListFiles = async function({libraryId, objectId, path = "", versionHash,
  *
  * Expected format of fileInfo:
  *
-     [
-       {
-         path: string,
-         source: string // either a full path e.g. "s3://BUCKET_NAME/path..." or just the path part without "s3://BUCKET_NAME/"
-       }
-     ]
+    [
+      {
+        path: string,
+        source: string // either a full path e.g. "s3://BUCKET_NAME/path..." or just the path part without "s3://BUCKET_NAME/"
+      }
+    ]
  *
  * @memberof module:ElvClient/Files+Parts
  * @methodGroup Files
@@ -82,6 +90,7 @@ exports.ListFiles = async function({libraryId, objectId, path = "", versionHash,
  * @param {string=} signedUrl
  * @param {string} encryption="none" - Encryption for uploaded files (copy only) - cgck | none
  * @param {boolean} copy=false - If true, will copy the data from S3 into the fabric. Otherwise, a reference to the content will be made.
+ * @param {boolean} resume=false - If true, will resume the file jobs stopped
  * @param {function=} callback - If specified, will be periodically called with current upload status
  * - Arguments (copy): { done: boolean, uploaded: number, total: number, uploadedFiles: number, totalFiles: number, fileStatus: Object }
  * - Arguments (reference): { done: boolean, uploadedFiles: number, totalFiles: number }
@@ -96,61 +105,41 @@ exports.UploadFilesFromS3 = async function({
   accessKey,
   secret,
   signedUrl,
-  encryption="none",
-  copy=false,
+  encryption = "none",
+  copy = false,
+  resume = false,
   callback
 }) {
-  ValidateParameters({libraryId, objectId});
+  ValidateParameters({ libraryId, objectId });
   ValidateWriteToken(writeToken);
 
   const s3prefixRegex = /^s3:\/\/([^/]+)\//i; // for matching and extracting bucket name when full s3:// path is specified
   // if fileInfo source paths start with s3://bucketName/, check against bucket arg passed in, and strip
   for(let i = 0; i < fileInfo.length; i++) {
     const fileSourcePath = fileInfo[i].source;
-    const s3prefixMatch = (s3prefixRegex.exec(fileSourcePath));
+    const s3prefixMatch = s3prefixRegex.exec(fileSourcePath);
     if(s3prefixMatch) {
       const bucketName = s3prefixMatch[1];
       if(bucketName !== bucket) {
         throw Error("Full S3 file path \"" + fileSourcePath + "\" specified, but does not match provided bucket name '" + bucket + "'");
-      } else {
-        // strip prefix
-        fileInfo[i].source = fileSourcePath.replace(s3prefixRegex,"");
       }
+      // strip prefix
+      fileInfo[i].source = fileSourcePath.replace(s3prefixRegex, "");
     }
   }
 
-  if(copy) {
-    this.Log(`Copying files from S3: ${libraryId} ${objectId} ${writeToken}`);
-  } else {
-    this.Log(`Adding links to files in S3: ${libraryId} ${objectId} ${writeToken}`);
-  }
+  this.Log(`${resume ? "Resuming" : (copy ? "Copying" : "Referencing")} files from S3: ${libraryId} ${objectId} ${writeToken}`);
 
   let encryption_key;
   if(encryption === "cgck") {
-    let conk = await this.EncryptionConk({
-      libraryId,
-      objectId,
-      writeToken
-    });
-
-    conk = {
-      ...conk,
-      secret_key: ""
-    };
-
+    let conk = await this.EncryptionConk({ libraryId, objectId, writeToken });
+    conk = { ...conk, secret_key: "" };
     encryption_key = `kp__${this.utils.B58(Buffer.from(JSON.stringify(conk)))}`;
   }
 
-  let cloudCredentials = {
-    access_key_id: accessKey,
-    secret_access_key: secret
-  };
-
-  if(signedUrl) {
-    cloudCredentials = {
-      signed_url: signedUrl
-    };
-  }
+  const cloudCredentials = signedUrl
+    ? { signed_url: signedUrl }
+    : { access_key_id: accessKey, secret_access_key: secret };
 
   const remoteAccess = {
     protocol: "s3",
@@ -191,55 +180,94 @@ exports.UploadFilesFromS3 = async function({
     }
   });
 
-  // eslint-disable-next-line no-unused-vars
-  const {id} = await this.CreateFileUploadJob({libraryId, objectId, writeToken, ops, defaults});
-
-  // eslint-disable-next-line no-constant-condition
-  while(true) {
-    await new Promise(resolve => setTimeout(resolve, 1000));
+  const trackUploadStatus = async ({ uploadId }) => {
+    // eslint-disable-next-line no-constant-condition
+    while(true) {
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      const status = await this.UploadStatus({ libraryId, objectId, writeToken, uploadId });
+
+      if(status.errors && status.errors.length >= 1) {
+        throw new Error(status.errors.join("\n"));
+      } else if(status.error) {
+        this.Log(`S3 file upload failed:\n${JSON.stringify(status, null, 2)}`);
+        throw new Error(status.error);
+      } else if(status.status.toLowerCase() === "failed") {
+        throw new Error("File upload failed");
+      }
 
-    const status = await this.UploadStatus({libraryId, objectId, writeToken, uploadId: id});
+      let done = false;
+      if(copy) {
+        done = status.ingest_copy.done;
+        if(callback) {
+          const progress = status.ingest_copy.progress;
+
+          callback({
+            done,
+            uploaded: progress.bytes.completed,
+            total: progress.bytes.total,
+            uploadedFiles: progress.files.completed,
+            totalFiles: progress.files.total,
+            fileStatus: progress.files.details
+          });
+        }
+      } else {
+        done = status.add_reference.done;
+        if(callback) {
+          const progress = status.add_reference.progress;
+
+          callback({
+            done,
+            uploadedFiles: progress.completed,
+            totalFiles: progress.total,
+          });
+        }
+      }
 
-    if(status.errors && status.errors.length > 1) {
-      throw status.errors.join("\n");
-    } else if(status.error) {
-      this.Log(`S3 file upload failed:\n${JSON.stringify(status, null, 2)}`);
-      throw status.error;
-    } else if(status.status.toLowerCase() === "failed") {
-      throw "File upload failed";
+      if(done) break;
     }
+  };
 
-    let done = false;
-    if(copy) {
-      done = status.ingest_copy.done;
-
-      if(callback) {
-        const progress = status.ingest_copy.progress;
-
-        callback({
-          done,
-          uploaded: progress.bytes.completed,
-          total: progress.bytes.total,
-          uploadedFiles: progress.files.completed,
-          totalFiles: progress.files.total,
-          fileStatus: progress.files.details
-        });
-      }
-    } else {
-      done = status.add_reference.done;
+  if(resume) {
+    const jobIds = await this.ListFilesJob({ libraryId, objectId, writeToken, encryption });
+    const jobsByStatus = jobIds.reduce((acc, job) => {
+      acc[job.status] = acc[job.status] || new Map();
+      acc[job.status].set(job.id, true);
+      return acc;
+    }, {});
 
-      if(callback) {
-        const progress = status.add_reference.progress;
+    const resumeAndTrack = async (jobIds) => {
+      const responses = await this.ResumeFileUploadJob({
+        libraryId,
+        objectId,
+        writeToken,
+        ops,
+        defaults,
+        encryption,
+        stoppedOrFailedJobIds: jobIds
+      });
+      await this.utils.LimitedMap(5, responses, async res => {
+        this.Log(`Tracking resumed job id: ${res.id}`);
+        await trackUploadStatus({ uploadId: res.id });
+      });
+    };
 
-        callback({
-          done,
-          uploadedFiles: progress.completed,
-          totalFiles: progress.total,
-        });
+    for(const status of [JobStatus.STOPPED, JobStatus.FAILED]) {
+      if(jobsByStatus[status]){
+        await resumeAndTrack([...jobsByStatus[status].keys()]);
       }
     }
 
-    if(done) { break; }
+    const trackOnlyEntries = Object.entries(jobsByStatus)
+      .filter(([status]) => [JobStatus.IN_PROGRESS, JobStatus.COMPLETED].includes(status))
+      .flatMap(([status, jobs]) => [...jobs.keys()].map(jobId => ({ status, jobId })));
+
+    await this.utils.LimitedMap(5, trackOnlyEntries, async ({ status, jobId }) => {
+      this.Log(`Tracking ${status.toLowerCase()} job id: ${jobId}`);
+      await trackUploadStatus({ uploadId: jobId });
+    });
+  } else {
+    const { id: uploadId } = await this.CreateFileUploadJob({ libraryId, objectId, writeToken, ops, defaults });
+    await trackUploadStatus({ uploadId });
   }
 };
 
@@ -248,14 +276,14 @@ exports.UploadFilesFromS3 = async function({
  *
  * Expected format of fileInfo:
  *
-     [
-         {
-            path: string,
-            mime_type: string,
-            size: number,
-            data: File | ArrayBuffer | Buffer | File Descriptor (Node)
-          }
-     ]
+   [
+     {
+       path: string,
+       mime_type: string,
+       size: number,
+       data: File | ArrayBuffer | Buffer | File Descriptor (Node)
+     }
+   ]
  *
  *
  * @memberof module:ElvClient/Files+Parts
@@ -265,11 +293,12 @@ exports.UploadFilesFromS3 = async function({
  * @param {string} objectId - ID of the object
  * @param {string} writeToken - Write token of the draft
  * @param {Array<object>} fileInfo - List of files to upload, including their size, type, and contents
+ * @param {string} resume - Resume the upload jobs
  * @param {string} encryption="none" - Encryption for uploaded files - cgck | none
  * @param {function=} callback - If specified, will be called after each job segment is finished with the current upload progress
  * - Format: {"filename1": {uploaded: number, total: number}, ...}
  */
-exports.UploadFiles = async function({libraryId, objectId, writeToken, fileInfo, encryption="none", callback}) {
+exports.UploadFiles = async function({libraryId, objectId, writeToken, fileInfo, resume=false,encryption="none", callback}) {
   ValidateParameters({libraryId, objectId});
   ValidateWriteToken(writeToken);
   ValidatePresence("fileInfo", fileInfo);
@@ -312,160 +341,253 @@ exports.UploadFiles = async function({libraryId, objectId, writeToken, fileInfo,
 
   this.Log(fileInfo);
 
-  if(callback) {
+  // On a fresh upload nothing has happened yet, so 0% is accurate. On resume, the real starting
+  // point isn't known until job status is fetched below, so skip this to avoid a misleading
+  // initial 0% line ahead of the actual (possibly much higher) baseline.
+  if(callback && !resume) {
     callback(progress);
   }
 
-  const {id, jobs} = await this.CreateFileUploadJob({
-    libraryId,
-    objectId,
-    writeToken,
-    ops: fileInfo,
-    encryption
-  });
+  let idJobMap = new Map();
+  if(resume) {
+    const ids = await this.ListFilesJob({ libraryId, objectId, writeToken, encryption });
 
-  this.Log(`Upload ID: ${id}`);
-  this.Log(jobs);
+    const jobsByStatus = ids.reduce((acc, job) => {
+      acc[job.status] = acc[job.status] || [];
+      acc[job.status].push(job.id);
+      return acc;
+    }, {});
 
-  // How far encryption can get ahead of upload
-  const bufferSize = 500 * 1024 * 1024;
+    const stoppedJobIds = jobsByStatus.STOPPED || [];
+    const failedJobIds = jobsByStatus.FAILED || [];
+    const inProgressJobIds = jobsByStatus.IN_PROGRESS || [];
 
-  let jobSpecs = [];
-  let prepared = 0;
-  let uploaded = 0;
+    const resumeJobIds = [...stoppedJobIds, ...failedJobIds, ...inProgressJobIds];
+    if(resumeJobIds.length === 0) {
+      this.Log("No job Ids to resume");
+      return;
+    }
 
-  // Insert the data to upload into the job spec, encrypting if necessary
-  const PrepareJobs = async () => {
-    for(let j = 0; j < jobs.length; j++) {
-      while(prepared - uploaded > bufferSize) {
-        // Wait for more data to be uploaded
-        await new Promise(resolve => setTimeout(resolve, 500));
+    const resumeJobs = async(jobs) => {
+      await this.ResumeFileUploadJob({
+        libraryId,
+        objectId,
+        writeToken,
+        ops: fileInfo,
+        encryption,
+        stoppedOrFailedJobIds: jobs
+      });
+    };
+
+    // resume stopped and failed jobs
+    for(const jobsToResume of [stoppedJobIds, failedJobIds]){
+      if(jobsToResume.length > 0){
+        await resumeJobs(jobsToResume);
       }
+    }
 
-      // Retrieve job info
-      const jobId = jobs[j];
-      let job = await this.UploadJobStatus({
+    // for each uploadId, make a map containing jobIds
+    const uploadJobsResults = await this.utils.LimitedMap(5, resumeJobIds, async uploadId => {
+      const result = await this.ListFilesUploadJobs({
         libraryId,
         objectId,
         writeToken,
-        uploadId: id,
-        jobId
+        jobId: uploadId,
+        encryption
       });
+      return result.jobs;
+    });
+    resumeJobIds.forEach((uploadId, i) => idJobMap.set(uploadId, uploadJobsResults[i]));
+    
+  } else {
+    const jobResponse = await this.CreateFileUploadJob({
+      libraryId,
+      objectId,
+      writeToken,
+      ops: fileInfo,
+      encryption
+    });
+    idJobMap.set(jobResponse.id, jobResponse.jobs);
+  }
 
-      for(let f = 0; f < job.files.length; f++) {
-        const fileInfo = job.files[f];
+  // How far encryption can get ahead of upload (500 MB)
+  const bufferSize = 500 * 1024 * 1024;
 
-        let data;
-        if(typeof fileDataMap[fileInfo.path] === "number") {
-          // File descriptor - Read data from file
-          data = Buffer.alloc(fileInfo.len);
-          fs.readSync(fileDataMap[fileInfo.path], data, 0, fileInfo.len, fileInfo.off);
-        } else {
-          // Full data - Slice requested chunk
-          data = fileDataMap[fileInfo.path].slice(fileInfo.off, fileInfo.off + fileInfo.len);
+  // Process each uploadId and its jobs
+  for(const [uploadId, jobs] of idJobMap.entries()) {
+    this.Log(`Upload ID: ${uploadId}`);
+    this.Log(jobs.length);
+
+    // Each upload ID gets its own counters
+    let prepared = 0;
+    let uploaded = 0;
+    let jobSpecs = [];
+
+    // `${jobId}:${path}` pairs already reflected in progress by the resume baseline scan below,
+    // so the normal per-chunk completion credit further down doesn't double-count them
+    // It stores the jobId whose remaining bytes = 0
+    let alreadyCreditedKeys = new Set();
+
+    if(resume && callback) {
+      // Establish the true resume baseline across ALL chunks up front, before showing any
+      // progress. Crediting chunks one at a time as PrepareJobs reaches them (sequentially, each
+      // with its own round trip) would make an already-mostly-done file look like it's slowly
+      // re-climbing from scratch instead of immediately reflecting what's already been received.
+      await this.utils.LimitedMap(5, jobs, async jobId => {
+        const job = await this.UploadJobStatus({ libraryId, objectId, writeToken, uploadId, jobId });
+
+        for(const fileInfo of job.files) {
+          // When encrypted, the top-level rem/skip are sentinel (-1) - the real, already-received
+          // byte counts for this chunk are under `encrypted` (same lookup UploadFileData uses)
+          const fileStatus = (encryption && encryption !== "none") ? fileInfo.encrypted : fileInfo;
+          if(fileStatus.rem === 0) {
+            alreadyCreditedKeys.add(`${jobId}:${fileInfo.path}`);
+            progress[fileInfo.path] = {
+              ...progress[fileInfo.path],
+              uploaded: progress[fileInfo.path].uploaded + fileInfo.len
+            };
+          }
         }
+      });
 
-        if(encryption === "cgck") {
-          data = await this.Crypto.Encrypt(conk, data);
-        }
+      callback(progress);
+    }
 
-        job.files[f].data = data;
+    // Insert the data to upload into the job spec, encrypting if necessary
+    const PrepareJobs = async () => {
+      for(let j = 0; j < jobs.length; j++) {
+        while(prepared - uploaded > bufferSize) {
+          // Wait for more data to be uploaded
+          await new Promise(resolve => setTimeout(resolve, 500));
+        }
 
-        prepared += fileInfo.len;
-      }
+        // Retrieve job info
+        const jobId = jobs[j];
+        let job = await this.UploadJobStatus({
+          libraryId,
+          objectId,
+          writeToken,
+          uploadId,
+          jobId
+        });
 
-      jobSpecs[j] = job;
+        for(let f = 0; f < job.files.length; f++) {
+          const fileInfo = job.files[f];
+
+          let data;
+          if(typeof fileDataMap[fileInfo.path] === "number") {
+            // File descriptor - Read data from file
+            data = Buffer.alloc(fileInfo.len);
+            fs.readSync(fileDataMap[fileInfo.path], data, 0, fileInfo.len, fileInfo.off);
+          } else {
+            // Full data - Slice requested chunk
+            data = fileDataMap[fileInfo.path].slice(fileInfo.off, fileInfo.off + fileInfo.len);
+          }
 
-      // Wait for a bit to let upload start
-      await new Promise(resolve => setTimeout(resolve, 50));
-    }
-  };
+          if(encryption === "cgck") {
+            data = await this.Crypto.Encrypt(conk, data);
+          }
 
-  const UploadJob = async (jobId, j)  => {
-    while(!jobSpecs[j]) {
-      // Wait for more jobs to be prepared
-      await new Promise(resolve => setTimeout(resolve, 500));
-    }
+          job.files[f].data = data;
 
-    const jobSpec = jobSpecs[j];
-    const files = jobSpec.files;
+          prepared += fileInfo.len;
+        }
 
-    // Upload each item
-    for(let f = 0; f < files.length; f++) {
-      const fileInfo = files[f];
+        jobSpecs[j] = job;
 
-      let retries = 0;
-      let succeeded = false;
-      do {
-        try {
-          await this.UploadFileData({
-            libraryId,
-            objectId,
-            writeToken,
-            uploadId: id,
-            jobId,
-            filePath: fileInfo.path,
-            fileData: fileInfo.data,
-            encryption
-          });
+        // Wait for a bit to let upload start
+        await new Promise(resolve => setTimeout(resolve, 50));
+      }
 
-          succeeded = true;
-        } catch(error) {
-          this.Log(error, true);
+    };
 
-          retries += 1;
+    const UploadJob = async (jobId, j) => {
+      while(!jobSpecs[j]) {
+        // Wait for more jobs to be prepared
+        await new Promise(resolve => setTimeout(resolve, 500));
+      }
 
-          if(retries >= 10) {
-            throw error;
+      const jobSpec = jobSpecs[j];
+      const files = jobSpec.files;
+
+      // Upload each item
+      for(let f = 0; f < files.length; f++) {
+        const fileInfo = files[f];
+
+        let retries = 0;
+        let succeeded = false;
+        do {
+          try {
+            await this.UploadFileData({
+              libraryId,
+              objectId,
+              writeToken,
+              uploadId,
+              jobId,
+              filePath: fileInfo.path,
+              fileData: fileInfo.data,
+              encryption
+            });
+
+            succeeded = true;
+          } catch(error) {
+            this.Log(error, true);
+
+            retries += 1;
+
+            if(retries >= 10) {
+              throw error;
+            }
+
+            await new Promise(resolve => setTimeout(resolve, 10 * retries * 1000));
           }
+        } while(!succeeded && retries < 10);
 
-          await new Promise(resolve => setTimeout(resolve, 10 * retries * 1000));
-        }
-      } while(!succeeded && retries < 10);
+        delete jobSpecs[j].files[f].data;
+        uploaded += fileInfo.len;
 
-      delete jobSpecs[j].files[f].data;
-      uploaded += fileInfo.len;
+        if(callback && !alreadyCreditedKeys.has(`${jobId}:${fileInfo.path}`)) {
+          progress[fileInfo.path] = {
+            ...progress[fileInfo.path],
+            uploaded: progress[fileInfo.path].uploaded + fileInfo.len
+          };
 
-      if(callback) {
-        progress[fileInfo.path] = {
-          ...progress[fileInfo.path],
-          uploaded: progress[fileInfo.path].uploaded + fileInfo.len
-        };
-
-        callback(progress);
+          callback(progress);
+        }
       }
+    };
+
+    // Preparing jobs is done asynchronously
+    PrepareJobs().catch(e => { throw e; });
+
+    // Upload the first several chunks in sequence, to determine average upload rate
+    const rateTestJobs = Math.min(3, jobs.length);
+    let rates = [];
+    for(let j = 0; j < rateTestJobs; j++) {
+      const start = new Date().getTime();
+      await UploadJob(jobs[j], j);
+      const elapsed = (new Date().getTime() - start) / 1000;
+      const size = jobSpecs[j].files.map(file => file.len).reduce((length, total) => length + total, 0);
+      rates.push(size / elapsed / (1024 * 1024));
     }
-  };
 
-  // Preparing jobs is done asynchronously
-  PrepareJobs().catch(e => {
-    throw e;
-  });
+    const averageRate = rates.reduce((mbps, total) => mbps + total, 0) / rateTestJobs;
 
-  // Upload the first several chunks in sequence, to determine average upload rate
-  const rateTestJobs = Math.min(3, jobs.length);
-  let rates = [];
-  for(let j = 0; j < rateTestJobs; j++) {
-    const start = new Date().getTime();
-    await UploadJob(jobs[j], j);
-    const elapsed = (new Date().getTime() - start) / 1000;
-    const size = jobSpecs[j].files.map(file => file.len).reduce((length, total) => length + total, 0);
-    rates.push(size / elapsed / (1024 * 1024));
+    // Upload remaining jobs in parallel
+    const concurrentUploads = Math.min(5, Math.ceil(averageRate / 2));
+    await this.utils.LimitedMap(
+      concurrentUploads,
+      jobs,
+      async (jobId, j) => {
+        if(j < rateTestJobs) { return; }
+        try {
+          await UploadJob(jobId, j);
+        } catch(err) {
+          throw Error (`Failed upload for job ${jobId}: ${err}`);
+        }
+      }
+    );
   }
-
-  const averageRate = rates.reduce((mbps, total) => mbps + total, 0) / rateTestJobs;
-
-  // Upload remaining jobs in parallel
-  const concurrentUploads = Math.min(5, Math.ceil(averageRate / 2));
-  await this.utils.LimitedMap(
-    concurrentUploads,
-    jobs,
-    async (jobId, j)  => {
-      if(j < rateTestJobs) { return; }
-
-      await UploadJob(jobId, j);
-    }
-  );
 };
 
 exports.CreateFileUploadJob = async function({libraryId, objectId, writeToken, ops, defaults={}, encryption="none"}) {
@@ -497,12 +619,84 @@ exports.CreateFileUploadJob = async function({libraryId, objectId, writeToken, o
   });
 };
 
+exports.ListFilesJob = async function({libraryId, objectId, writeToken, encryption="none"}) {
+  ValidateParameters({libraryId, objectId});
+  ValidateWriteToken(writeToken);
+
+  this.Log(`List file jobs: ${libraryId} ${objectId} ${writeToken}`);
+
+  const path = UrlJoin( "q", writeToken, "file_jobs");
+
+  return this.HttpClient.RequestJsonBody({
+    headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true, encryption}),
+    method: "GET",
+    path: path,
+    allowFailover: false
+  });
+};
+
+exports.ListFilesUploadJobs = async function({libraryId, objectId, writeToken, jobId, encryption="none"}) {
+  ValidateParameters({libraryId, objectId});
+  ValidateWriteToken(writeToken);
+
+  this.Log(`List file upload jobs: ${libraryId} ${objectId} ${writeToken}`);
+
+  const path = UrlJoin("qlibs", libraryId, "q", writeToken, "file_jobs", jobId, "uploads");
+
+  return this.HttpClient.RequestJsonBody({
+    headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true, encryption}),
+    method: "GET",
+    path: path,
+    allowFailover: false
+  });
+};
+
+exports.ResumeFileUploadJob = async function({libraryId, objectId, writeToken, ops, defaults={}, encryption="none", stoppedOrFailedJobIds=[]}) {
+  ValidateParameters({libraryId, objectId});
+  ValidateWriteToken(writeToken);
+
+  if(stoppedOrFailedJobIds.length === 0) {
+    this.Log("No stopped/failed jobs found");
+    throw new Error("No STOPPED/FAILED jobs found");
+  }
+
+  this.Log(`Resuming file upload job: ${libraryId} ${objectId} ${writeToken}`);
+
+  if(encryption === "cgck") {
+    defaults.encryption = { scheme: "cgck" };
+  }
+
+  const body = {
+    seq: 0,
+    seq_complete: true,
+    clear_resolve: true,
+    defaults,
+    ops
+  };
+
+  // resume and change jobId status from 'STOPPED' to 'IN_PROGRESS'
+  const responses = [];
+  this.Log(`Stopped or Failed job IDs: ${stoppedOrFailedJobIds.join(",")}`);
+  for(const jobId of stoppedOrFailedJobIds) {
+    const path = UrlJoin("q", writeToken, "file_jobs" , jobId, "resume");
+
+    let res = await this.HttpClient.RequestJsonBody({
+      headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true, encryption}),
+      method: "PUT",
+      path: path,
+      body,
+      allowFailover: false
+    });
+    responses.push({message: res, id: jobId});
+  }
+  return responses;
+};
+
 exports.UploadStatus = async function({libraryId, objectId, writeToken, uploadId}) {
   ValidateParameters({libraryId, objectId});
   ValidateWriteToken(writeToken);
 
   const path = UrlJoin("q", writeToken, "file_jobs", uploadId);
-
   return this.utils.ResponseToJson(
     this.HttpClient.Request({
       headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}),
@@ -571,7 +765,7 @@ exports.UploadFileData = async function({libraryId, objectId, writeToken, encryp
 
   let path = UrlJoin("q", writeToken, "file_jobs", uploadId, jobId);
 
-  return await this.utils.ResponseToJson(
+  const response = await this.utils.ResponseToJson(
     this.HttpClient.Request({
       method: "POST",
       path: path,
@@ -585,6 +779,17 @@ exports.UploadFileData = async function({libraryId, objectId, writeToken, encryp
       allowRetry: false
     })
   );
+
+  if(response && response.part && !response.part.hash) {
+    // The request succeeded but the server received fewer bytes than the part needed to seal
+    // (e.g. a dropped/truncated connection) - part.hash is only populated once finalized, so an
+    // empty hash here means this chunk isn't actually done despite the 200. Throw so the caller's
+    // retry loop re-reads the job status (updated skip/rem) and resends the missing remainder,
+    // instead of the chunk silently being counted as complete.
+    throw new Error(`Upload did not finalize part for ${filePath} (job ${jobId})`);
+  }
+
+  return response;
 };
 
 exports.FinalizeUploadJob = async function({libraryId, objectId, writeToken}) {

package/src/client/LiveStream.js

@@ -1090,9 +1090,9 @@ exports.StreamStatus = async function({name, showParams=false, writeToken}) {
         status.recordingStatus = lroStatus.custom.status;
       }
     } catch(error) {
-      console.log("LRO Status (failed): ", error.response.statusCode);
+      console.log("LRO Status (failed): ", error.response?.statusCode);
       status.state = "stopped";
-      status.error = error.response;
+      status.error = error.response || error.message;
       return status;
     }
 
@@ -3532,6 +3532,13 @@ exports.OutputsList = async function({libraryId, objectId, includeState=true}) {
       method: "live/outputs",
       constant:  true
     });
+  } catch(error) {
+    if(error.status === 404) {
+      // Stream object may have been deleted
+      return {};
+    }
+
+    throw error;
   } finally {
     restore();
   }
@@ -3544,17 +3551,22 @@ exports.OutputsList = async function({libraryId, objectId, includeState=true}) {
       const streamId = value.input?.stream;
       if(!streamId) { return; }
 
-      const [streamLibraryId, streamStatus] = await Promise.all([
-        this.ContentObjectLibraryId({objectId: streamId}),
-        this.StreamStatus({name: streamId})
-      ]);
-
-      value.input.name = await this.ContentObjectMetadata({
-        libraryId: streamLibraryId,
-        objectId: streamId,
-        metadataSubtree: "/public/name",
-      });
-      value.input.status = streamStatus?.state;
+      try {
+        const [streamLibraryId, streamStatus] = await Promise.all([
+          this.ContentObjectLibraryId({objectId: streamId}),
+          this.StreamStatus({name: streamId})
+        ]);
+
+        value.input.name = await this.ContentObjectMetadata({
+          libraryId: streamLibraryId,
+          objectId: streamId,
+          metadataSubtree: "/public/name",
+        });
+        value.input.status = streamStatus?.state;
+      } catch(error) {
+        // Stream object may have been deleted
+        value.input.status = "unavailable";
+      }
     }
   );
 
@@ -3569,9 +3581,13 @@ exports.OutputsList = async function({libraryId, objectId, includeState=true}) {
 
   const nodeUrls = {};
   await this.utils.LimitedMap(10, nodeIds, async nodeId => {
-    const nodes = await this.SpaceNodes({matchNodeId: nodeId});
-    const fabricUrl = nodes?.[0]?.services?.fabric_api?.urls?.[0];
-    if(fabricUrl) { nodeUrls[nodeId] = fabricUrl; }
+    try {
+      const nodes = await this.SpaceNodes({matchNodeId: nodeId});
+      const fabricUrl = nodes?.[0]?.services?.fabric_api?.urls?.[0];
+      if(fabricUrl) { nodeUrls[nodeId] = fabricUrl; }
+    } catch(error) {
+      this.Log(`Failed to resolve node ${nodeId}: ${error.message}`, true);
+    }
   });
 
   // Rewrite fabric-generated srt_pull URLs to the egress host, and group outputs by node

package/utilities/MigrateEncryptionConkForNewUser.js

@@ -0,0 +1,62 @@
+const {NewOpt} = require("./lib/options");
+const Utility = require("./lib/Utility");
+
+const Client = require("./lib/concerns/Client");
+const Edit = require("./lib/concerns/Edit");
+
+class MigrateEncryptionConkForNewUser extends Utility {
+  blueprint() {
+    return {
+      concerns: [Client, Edit],
+      options: [
+        NewOpt("objectId",
+          {
+            descTemplate: "object Id",
+            type: "string",
+            demand: true
+          }),
+        NewOpt("publicKey",
+          {
+            descTemplate: "new user public key with/without encoded form",
+            type: "string",
+            demand: true
+          })
+      ]
+    };
+  }
+
+  async body() {
+    const logger = this.logger;
+    const {objectId, publicKey} = this.args;
+
+    const client = await this.concerns.Client.get();
+    const libraryId = await client.ContentObjectLibraryId({objectId});
+
+    const res =  await client.EditContentObject({libraryId, objectId});
+
+    await client.MigrateEncryptionConkForUserProvided({
+      libraryId,
+      objectId,
+      writeToken: res.writeToken,
+      newEncodedUserPublicKey: publicKey
+    });
+    logger.log("Migration of encryption conk is complete...");
+
+    const versionHash = await this.concerns.Edit.finalize({
+      libraryId,
+      objectId,
+      writeToken:res.writeToken
+    });
+    logger.log("version hash", versionHash);
+  }
+
+  header() {
+    return `Add encryption conk for user provided '${this.args.objectId}':`;
+  }
+}
+
+if(require.main === module) {
+  Utility.cmdLineInvoke(MigrateEncryptionConkForNewUser);
+} else {
+  module.exports = MigrateEncryptionConkForNewUser;
+}

package/utilities/ObjectAddFiles.js

@@ -1,6 +1,6 @@
 // Create new production master from specified file(s)
 
-const {ModOpt, StdOpt} = require("./lib/options");
+const {ModOpt, StdOpt, NewOpt} = require("./lib/options");
 const Utility = require("./lib/Utility");
 
 const ArgNoWait = require("./lib/concerns/ArgNoWait");
@@ -16,14 +16,19 @@ class ObjectAddFiles extends Utility {
       concerns: [Logger, ExistObj, Edit, ArgNoWait, LocalFile, CloudFile],
       options: [
         ModOpt("files", {X: "to add"}),
-        StdOpt("encrypt", {X: "uploaded files"})
+        StdOpt("encrypt", {X: "uploaded files"}),
+        NewOpt("resume", {
+          descTemplate: "If specified, resume jobs for the given write token (--resume 'write_token')",
+          group: "Main",
+          type: "string"
+        })
       ]
     };
   }
 
   async body() {
     const logger = this.logger;
-    const {encrypt, noWait} = this.args;
+    const {encrypt, noWait, resume: resumeArg} = this.args;
 
     let access;
     if(this.args.s3Reference || this.args.s3Copy) access = this.concerns.CloudFile.credentialSet();
@@ -35,10 +40,22 @@ class ObjectAddFiles extends Utility {
 
     const {libraryId, objectId} = await this.concerns.ExistObj.argsProc();
 
-    const writeToken = await this.concerns.Edit.getWriteToken({
-      libraryId,
-      objectId
-    });
+    let writeToken;
+    let resume=false;
+    if(resumeArg) {
+      writeToken = resumeArg;
+      resume=true;
+    } else {
+      // create new write token
+      writeToken = await this.concerns.Edit.getWriteToken({
+        libraryId,
+        objectId
+      });
+    }
+    logger.data("WRITE_TOKEN", writeToken);
+    // eslint-disable-next-line no-console
+    console.log("Using write token:", writeToken);
+
 
     if(access) {
       await this.concerns.CloudFile.add({
@@ -47,7 +64,8 @@ class ObjectAddFiles extends Utility {
         writeToken,
         access,
         fileInfo,
-        encrypt
+        encrypt,
+        resume,
       });
     } else {
       await this.concerns.LocalFile.add({
@@ -55,12 +73,14 @@ class ObjectAddFiles extends Utility {
         objectId,
         writeToken,
         fileInfo,
-        encrypt
+        encrypt,
+        resume,
       });
       // Close file handles
       this.concerns.LocalFile.closeFileHandles(fileHandles);
     }
 
+    // finalize the write token
     const hash = await this.concerns.Edit.finalize({
       libraryId,
       noWait,

package/utilities/lib/concerns/CloudFile.js

@@ -95,7 +95,7 @@ const New = context => {
     );
   };
 
-  const add = async ({libraryId, objectId, writeToken, files, access, encrypt, copy}) => {
+  const add = async ({libraryId, objectId, writeToken, files, access, encrypt, copy, resume}) => {
     if(kindOf(copy) === "undefined") copy = isCopy();
     files = files || context.args.files;
     const groupedFiles = groupByPathMatch(access, files);
@@ -125,6 +125,7 @@ const New = context => {
         secret,
         signedUrl,
         copy,
+        resume,
         callback,
         encryption: encrypt ? "cgck" : "none"
       });

package/utilities/lib/concerns/LocalFile.js

@@ -19,7 +19,7 @@ const blueprint = {
 
 const New = context => {
 
-  const add = async ({libraryId, objectId, writeToken, fileInfo, encrypt}) => {
+  const add = async ({libraryId, objectId, writeToken, fileInfo, encrypt, resume}) => {
     const client = await context.concerns.Client.get();
     await client.UploadFiles({
       libraryId,
@@ -27,6 +27,7 @@ const New = context => {
       writeToken,
       fileInfo,
       callback,
+      resume,
       encryption: encrypt ? "cgck" : "none"
     });
   };