@eluvio/elv-client-js 4.0.21 → 4.0.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eluvio/elv-client-js",
-  "version": "4.0.21",
+  "version": "4.0.22",
   "description": "Javascript client for the Eluvio Content Fabric",
   "main": "src/index.js",
   "author": "Kevin Talmadge",

package/src/HttpClient.js

@@ -12,6 +12,7 @@ class HttpClient {
     this.uriIndex = 0;
     this.debug = debug;
     this.draftURIs = {};
+    this.retries = Math.max(3, uris.length);
   }
 
   BaseURI() {
@@ -50,8 +51,9 @@ class HttpClient {
     bodyType="JSON",
     headers={},
     attempts=0,
-    failover=true,
-    forceFailover=false
+    allowFailover=true,
+    forceFailover=false,
+    allowRetry=true
   }) {
     let baseURI = this.BaseURI();
 
@@ -60,6 +62,8 @@ class HttpClient {
     const writeToken = writeTokenMatch ? writeTokenMatch[2] : undefined;
 
     if(writeToken) {
+      allowFailover = false;
+
       if(this.draftURIs[writeToken]) {
         // Use saved write token URI
         baseURI = this.draftURIs[writeToken];
@@ -107,10 +111,21 @@ class HttpClient {
 
     if(!response.ok) {
       // Fail over if not a write token request, the response was a server error, and we haven't tried all available nodes
-      if(!writeToken && ((failover && parseInt(response.status) >= 500) || forceFailover) && attempts < this.uris.length) {
-        // Server error - Try next node
-        this.Log(`HttpClient failing over from ${this.BaseURI()}: ${attempts + 1} attempts`, true);
-        this.uriIndex = (this.uriIndex + 1) % this.uris.length;
+      if(
+        (parseInt(response.status) >= 500 || forceFailover) &&
+        allowRetry &&
+        attempts < this.retries
+      ) {
+        // Server error
+        if(allowFailover) {
+          // Fail over to alternate node
+          this.uriIndex = (this.uriIndex + 1) % this.uris.length;
+          this.Log(`HttpClient failing over from ${baseURI.toString()}: ${attempts + 1} attempts`, true);
+        } else {
+          // Wait and retry
+          this.Log(`HttpClient retrying request from ${baseURI.toString()}: ${attempts + 1} attempts`, true);
+          await new Promise(resolve => setTimeout(resolve, 1000));
+        }
 
         return await this.Request({
           method,

package/src/client/ContentAccess.js

@@ -1831,7 +1831,7 @@ exports.CallBitcodeMethod = async function({
       method: constant ? "GET" : "POST",
       path,
       queryParams,
-      failover: false
+      allowFailover: false
     })
   );
 };

package/src/client/ContentManagement.js

@@ -967,7 +967,7 @@ exports.FinalizeContentObject = async function({
       headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}),
       method: "POST",
       path: path,
-      failover: false
+      allowFailover: false
     })
   );
 
@@ -1127,7 +1127,7 @@ exports.MergeMetadata = async function({libraryId, objectId, writeToken, metadat
     method: "POST",
     path: path,
     body: metadata,
-    failover: false
+    allowFailover: false
   });
 };
 
@@ -1159,7 +1159,7 @@ exports.ReplaceMetadata = async function({libraryId, objectId, writeToken, metad
     method: "PUT",
     path: path,
     body: metadata,
-    failover: false
+    allowFailover: false
   });
 };
 
@@ -1190,7 +1190,7 @@ exports.DeleteMetadata = async function({libraryId, objectId, writeToken, metada
     headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}),
     method: "DELETE",
     path: path,
-    failover: false
+    allowFailover: false
   });
 };
 

package/src/client/Files.js

@@ -267,6 +267,7 @@ exports.UploadFilesFromS3 = async function({
 exports.UploadFiles = async function({libraryId, objectId, writeToken, fileInfo, encryption="none", callback}) {
   ValidateParameters({libraryId, objectId});
   ValidateWriteToken(writeToken);
+  ValidatePresence("fileInfo", fileInfo);
 
   this.Log(`Uploading files: ${libraryId} ${objectId} ${writeToken}`);
 
@@ -279,8 +280,10 @@ exports.UploadFiles = async function({libraryId, objectId, writeToken, fileInfo,
   let progress = {};
   let fileDataMap = {};
 
-  for(let i = 0; i < fileInfo.length; i++) {
-    let entry = { ...fileInfo[i], data: undefined };
+  let originalFileInfo = fileInfo;
+  fileInfo = [];
+  for(let i = 0; i < originalFileInfo.length; i++) {
+    let entry = { ...originalFileInfo[i], data: undefined };
 
     entry.path = entry.path.replace(/^\/+/, "");
 
@@ -290,9 +293,8 @@ exports.UploadFiles = async function({libraryId, objectId, writeToken, fileInfo,
       };
     }
 
-    fileDataMap[entry.path] = fileInfo[i].data;
+    fileDataMap[entry.path] = originalFileInfo[i].data;
 
-    delete entry.data;
     entry.type = "file";
 
     progress[entry.path] = {
@@ -300,7 +302,7 @@ exports.UploadFiles = async function({libraryId, objectId, writeToken, fileInfo,
       total: entry.size
     };
 
-    fileInfo[i] = entry;
+    fileInfo.push(entry);
   }
 
   this.Log(fileInfo);
@@ -321,7 +323,7 @@ exports.UploadFiles = async function({libraryId, objectId, writeToken, fileInfo,
   this.Log(jobs);
 
   // How far encryption can get ahead of upload
-  const bufferSize = 100 * 1024 * 1024;
+  const bufferSize = 500 * 1024 * 1024;
 
   let jobSpecs = [];
   let prepared = 0;
@@ -387,16 +389,34 @@ exports.UploadFiles = async function({libraryId, objectId, writeToken, fileInfo,
     for(let f = 0; f < files.length; f++) {
       const fileInfo = files[f];
 
-      await this.UploadFileData({
-        libraryId,
-        objectId,
-        writeToken,
-        uploadId: id,
-        jobId,
-        filePath: fileInfo.path,
-        fileData: fileInfo.data,
-        encryption
-      });
+      let retries = 0;
+      let succeeded = false;
+      do {
+        try {
+          await this.UploadFileData({
+            libraryId,
+            objectId,
+            writeToken,
+            uploadId: id,
+            jobId,
+            filePath: fileInfo.path,
+            fileData: fileInfo.data,
+            encryption
+          });
+
+          succeeded = true;
+        } catch(error) {
+          this.Log(error, true);
+
+          retries += 1;
+
+          if(retries >= 10) {
+            throw error;
+          }
+
+          await new Promise(resolve => setTimeout(resolve, 10 * retries * 1000));
+        }
+      } while(!succeeded && retries < 10);
 
       delete jobSpecs[j].files[f].data;
       uploaded += fileInfo.len;
@@ -469,7 +489,7 @@ exports.CreateFileUploadJob = async function({libraryId, objectId, writeToken, o
       method: "POST",
       path: path,
       body,
-      failover: false
+      allowFailover: false
     })
   );
 };
@@ -485,7 +505,7 @@ exports.UploadStatus = async function({libraryId, objectId, writeToken, uploadId
       headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}),
       method: "GET",
       path: path,
-      failover: false
+      allowFailover: false
     })
   );
 };
@@ -496,82 +516,51 @@ exports.UploadJobStatus = async function({libraryId, objectId, writeToken, uploa
 
   const path = UrlJoin("q", writeToken, "file_jobs", uploadId, "uploads", jobId);
 
-  // This request is sent during file data upload and might fail due to congestion
-  let retries = 0;
-  do {
-    try {
-      let jobStatus = this.utils.ResponseToJson(
-        this.HttpClient.Request({
-          headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}),
-          method: "GET",
-          path: path,
-          failover: false
-        })
-      );
-      return jobStatus;
-    } catch(error) {
-      this.Log(error, true);
-
-      retries += 1;
-      if(retries >= 5) {
-        throw error;
-      }
-      await new Promise(resolve => setTimeout(resolve, 10 * retries * 1000));
-    }
-  } while(retries < 5);
-
+  return await this.utils.ResponseToJson(
+    this.HttpClient.Request({
+      headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}),
+      method: "GET",
+      path: path,
+      allowFailover: false
+    })
+  );
 };
 
 exports.UploadFileData = async function({libraryId, objectId, writeToken, encryption, uploadId, jobId, filePath, fileData}) {
   ValidateParameters({libraryId, objectId});
   ValidateWriteToken(writeToken);
 
-  let retries = 0;
-  do {
-    try {
-
-      const jobStatus = await this.UploadJobStatus({libraryId, objectId, writeToken, uploadId, jobId});
+  const jobStatus = await this.UploadJobStatus({libraryId, objectId, writeToken, uploadId, jobId});
 
-      // Find the status of this file
-      let fileStatus = jobStatus.files.find(item => item.path == filePath);
-      if(encryption && encryption !== "none") {
-        fileStatus = fileStatus.encrypted;
-      }
-
-      if(fileStatus.rem === 0) {
-        // Job is actually done
-        return;
-      } else if(fileStatus.skip) {
-        fileData = fileData.slice(fileStatus.skip);
-      }
+  // Find the status of this file
+  let fileStatus = jobStatus.files.find(item => item.path == filePath);
+  if(encryption && encryption !== "none") {
+    fileStatus = fileStatus.encrypted;
+  }
 
-      let path = UrlJoin("q", writeToken, "file_jobs", uploadId, jobId);
-
-      return await this.utils.ResponseToJson(
-        this.HttpClient.Request({
-          method: "POST",
-          path: path,
-          body: fileData,
-          bodyType: "BINARY",
-          headers: {
-            "Content-type": "application/octet-stream",
-            ...(await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}))
-          },
-          failover: false
-        })
-      );
-    } catch(error){
-      this.Log(error, true);
+  if(fileStatus.rem === 0) {
+    // Job is actually done
+    return;
+  } else if(fileStatus.skip) {
+    fileData = fileData.slice(fileStatus.skip);
+  }
 
-      retries += 1;
+  let path = UrlJoin("q", writeToken, "file_jobs", uploadId, jobId);
 
-      if(retries >= 5) {
-        throw error;
-      }
-
-      await new Promise(resolve => setTimeout(resolve, 10 * retries * 1000));
-    }
-  } while(retries < 5);
+  return await this.utils.ResponseToJson(
+    this.HttpClient.Request({
+      method: "POST",
+      path: path,
+      body: fileData,
+      bodyType: "BINARY",
+      headers: {
+        "Content-type": "application/octet-stream",
+        ...(await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}))
+      },
+      allowFailover: false,
+      allowRetry: false
+    })
+  );
 };
 
 exports.FinalizeUploadJob = async function({libraryId, objectId, writeToken}) {
@@ -587,7 +576,7 @@ exports.FinalizeUploadJob = async function({libraryId, objectId, writeToken}) {
     path: path,
     bodyType: "BINARY",
     headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}),
-    failover: false
+    allowFailover: false
   });
 };
 
@@ -1051,7 +1040,7 @@ exports.CreatePart = async function({libraryId, objectId, writeToken, encryption
       path,
       bodyType: "BINARY",
       body: "",
-      failover: false
+      allowFailover: false
     })
   );
 
@@ -1090,7 +1079,7 @@ exports.UploadPartChunk = async function({libraryId, objectId, writeToken, partW
       path: UrlJoin(path, partWriteToken),
       body: chunk,
       bodyType: "BINARY",
-      failover: false
+      allowFailover: false
     })
   );
 };
@@ -1121,7 +1110,7 @@ exports.FinalizePart = async function({libraryId, objectId, writeToken, partWrit
       path: UrlJoin(path, partWriteToken),
       bodyType: "BINARY",
       body: "",
-      failover: false
+      allowFailover: false
     })
   );
 };
@@ -1200,6 +1189,6 @@ exports.DeletePart = async function({libraryId, objectId, writeToken, partHash})
     headers: await this.authClient.AuthorizationHeader({libraryId, objectId, update: true}),
     method: "DELETE",
     path: path,
-    failover: false
+    allowFailover: false
   });
 };

package/testScripts/Test.js

@@ -1,8 +1,7 @@
 const { ElvClient } = require("../src/ElvClient");
 const { ElvWalletClient } = require("../src/walletClient/index");
 const ClientConfiguration = require("../TestConfiguration.json");
-
-const ethers = require("ethers");
+const fs = require("fs");
 
 const Test = async () => {
   try {

package/testScripts/UploadFilesTest.js

@@ -0,0 +1,84 @@
+const { ElvClient } = require("../src/ElvClient");
+const { ElvWalletClient } = require("../src/walletClient/index");
+const ClientConfiguration = require("../TestConfiguration.json");
+const fs = require("fs");
+const path = require("path");
+
+const createHash = require("node:crypto").createHash;
+const MD5 = str => {
+  const f = createHash("md5");
+  f.update(str);
+  return f.digest("hex");
+};
+
+const Test = async () => {
+  try {
+    const client = await ElvClient.FromNetworkName({
+      networkName: "demo"
+    });
+
+    const wallet = client.GenerateWallet();
+    const signer = wallet.AddAccount({
+      privateKey: process.env.PRIVATE_KEY
+    });
+
+    client.SetSigner({signer});
+
+    const uploadDir = process.argv[2];
+
+    const dirname = `upload-test-${Date.now()}`;
+    const fileList = fs.readdirSync(uploadDir);
+    const fileInfo = fileList.map(filename => {
+      const data = fs.readFileSync(path.join(uploadDir, filename));
+
+      return {
+        path: `${dirname}/${filename}`,
+        size: data.length,
+        data
+      };
+    }).sort(() => Math.random() > 0.5 ? -1 : 1);
+
+
+    const libraryId = "ilib3Drbefo7VPfWvY1NVup4VZFzDJ68";
+    const objectId = "iq__2aqbcJhSKdkuRmAUqs6v99SLRRp";
+
+    const {hash} = await client.EditAndFinalizeContentObject({
+      libraryId,
+      objectId,
+      callback: async ({writeToken}) => {
+        await client.UploadFiles({
+          libraryId,
+          objectId,
+          writeToken,
+          //callback: console.log,
+          //encryption: "cgck",
+          fileInfo
+        });
+      }
+    });
+
+    console.log("\n\n");
+    for(let i = 0; i < fileInfo.length; i++) {
+      const {path, data} = fileInfo[i];
+
+      const downloadedData = await client.DownloadFile({
+        versionHash: hash,
+        format: "buffer",
+        filePath: path
+      });
+
+      console.log(path);
+      console.log(MD5(data.toString()));
+      console.log(MD5(downloadedData.toString()));
+      console.log(MD5(data.toString()) === MD5(downloadedData.toString()));
+      console.log();
+    }
+  } catch(error) {
+    console.error(error);
+    console.error(JSON.stringify(error, null, 2));
+  }
+
+  process.exit(0);
+};
+
+Test();

package/testScripts/abr_profile_4k_both.json

@@ -105,16 +105,6 @@
         "type": "ProtoHls"
       }
     },
-    "hls-fairplay": {
-      "drm": {
-        "enc_scheme_name": "cbcs",
-        "license_servers": [],
-        "type": "DrmFairplay"
-      },
-      "protocol": {
-        "type": "ProtoHls"
-      }
-    },
     "hls-sample-aes": {
       "drm": {
         "enc_scheme_name": "cbcs",

package/testScripts/abr_profile_4k_drm_public_access.json

@@ -0,0 +1,128 @@
+{
+  "drm_optional": false,
+  "store_clear": false,
+  "ladder_specs": {
+    "{\"media_type\":\"audio\",\"channels\":1}": {
+      "rung_specs": [
+        {
+          "bit_rate": 128000,
+          "media_type": "audio",
+          "pregenerate": true
+        }
+      ]
+    },
+    "{\"media_type\":\"audio\",\"channels\":2}": {
+      "rung_specs": [
+        {
+          "bit_rate": 192000,
+          "media_type": "audio",
+          "pregenerate": true
+        }
+      ]
+    },
+    "{\"media_type\":\"audio\",\"channels\":6}": {
+      "rung_specs": [
+        {
+          "bit_rate": 384000,
+          "media_type": "audio",
+          "pregenerate": true
+        }
+      ]
+    },
+    "{\"media_type\":\"video\",\"aspect_ratio_height\":9,\"aspect_ratio_width\":16}": {
+      "rung_specs": [
+        {
+          "bit_rate": 20000000,
+          "height": 2160,
+          "media_type": "video",
+          "pregenerate": true,
+          "width": 3840
+        },
+        {
+          "bit_rate": 9500000,
+          "height": 1080,
+          "media_type": "video",
+          "pregenerate": false,
+          "width": 1920
+        },
+        {
+          "bit_rate": 4500000,
+          "height":720 ,
+          "media_type": "video",
+          "pregenerate": false,
+          "width": 1280
+        },
+        {
+          "bit_rate": 2000000,
+          "height": 540,
+          "media_type": "video",
+          "pregenerate": false,
+          "width": 960
+        },
+        {
+          "bit_rate": 1100000,
+          "height": 432,
+          "media_type": "video",
+          "pregenerate": false,
+          "width": 768
+        },
+        {
+          "bit_rate": 810000,
+          "height": 360,
+          "media_type": "video",
+          "pregenerate": false,
+          "width": 640
+        },
+        {
+          "bit_rate": 520000,
+          "height": 360,
+          "media_type": "video",
+          "pregenerate": false,
+          "width": 640
+        }
+      ]
+    }
+  },
+  "playout_formats": {
+    "dash-widevine": {
+      "drm": {
+        "content_id": "",
+        "enc_scheme_name": "cenc",
+        "license_servers": [],
+        "type": "DrmWidevine"
+      },
+      "protocol": {
+        "min_buffer_length": 2,
+        "type": "ProtoDash"
+      }
+    },
+    "hls-sample-aes": {
+      "drm": {
+        "enc_scheme_name": "cbcs",
+        "type": "DrmSampleAes"
+      },
+      "protocol": {
+        "type": "ProtoHls"
+      }
+    },
+    "hls-aes128": {
+      "drm": {
+        "enc_scheme_name": "aes-128",
+        "type": "DrmAes128"
+      },
+      "protocol": {
+        "type": "ProtoHls"
+      }
+    }
+  },
+  "segment_specs": {
+    "audio": {
+      "segs_per_chunk": 15,
+      "target_dur": 2
+    },
+    "video": {
+      "segs_per_chunk": 15,
+      "target_dur": 2
+    }
+  }
+}

package/testScripts/abr_profile_both.json

@@ -1930,16 +1930,6 @@
         "type": "ProtoDash"
       }
     },
-    "hls-fairplay": {
-      "drm": {
-        "enc_scheme_name": "cbcs",
-        "license_servers": [],
-        "type": "DrmFairplay"
-      },
-      "protocol": {
-        "type": "ProtoHls"
-      }
-    },
     "hls-sample-aes": {
       "drm": {
         "enc_scheme_name": "cbcs",