@eluvio/elv-client-js 4.2.25 → 4.2.29

package/src/client/LiveStream.js

@@ -322,15 +322,16 @@ exports.StreamCreate = async function({
         video_type: "live",
         slug: slugify(name)
       }
-    },
-    "live_recording_config": liveRecordingConfig
+    }
   };
 
-  const oldProfile = await this.ContentObjectMetadata({
+  const currentLiveRecordingConfig = await this.ContentObjectMetadata({
     libraryId,
     objectId,
-    metadataSubtree: "live_recording_config/name"
-  });
+    metadataSubtree: "live_recording_config"
+  }) || {};
+
+  const oldProfile = currentLiveRecordingConfig.name;
 
   if(liveRecordingConfig?.name && liveRecordingConfig.name !== oldProfile) {
     metadata.public.asset_metadata.profile_last_updated = new Date().toISOString();
@@ -343,6 +344,15 @@ exports.StreamCreate = async function({
     metadata
   });
 
+  // Full overwrite avoids the server-side merge mixing streams from the old and new profile
+  await this.ReplaceMetadata({
+    libraryId,
+    objectId,
+    writeToken,
+    metadataSubtree: "live_recording_config",
+    metadata: R.mergeDeepRight(currentLiveRecordingConfig, liveRecordingConfig)
+  });
+
   try {
     await this.CreateLinks({
       libraryId,
@@ -1080,9 +1090,9 @@ exports.StreamStatus = async function({name, showParams=false, writeToken}) {
         status.recordingStatus = lroStatus.custom.status;
       }
     } catch(error) {
-      console.log("LRO Status (failed): ", error.response.statusCode);
+      console.log("LRO Status (failed): ", error.response?.statusCode);
       status.state = "stopped";
-      status.error = error.response;
+      status.error = error.response || error.message;
       return status;
     }
 
@@ -3522,6 +3532,13 @@ exports.OutputsList = async function({libraryId, objectId, includeState=true}) {
       method: "live/outputs",
       constant:  true
     });
+  } catch(error) {
+    if(error.status === 404) {
+      // Stream object may have been deleted
+      return {};
+    }
+
+    throw error;
   } finally {
     restore();
   }
@@ -3534,17 +3551,22 @@ exports.OutputsList = async function({libraryId, objectId, includeState=true}) {
       const streamId = value.input?.stream;
       if(!streamId) { return; }
 
-      const [streamLibraryId, streamStatus] = await Promise.all([
-        this.ContentObjectLibraryId({objectId: streamId}),
-        this.StreamStatus({name: streamId})
-      ]);
-
-      value.input.name = await this.ContentObjectMetadata({
-        libraryId: streamLibraryId,
-        objectId: streamId,
-        metadataSubtree: "/public/name",
-      });
-      value.input.status = streamStatus?.state;
+      try {
+        const [streamLibraryId, streamStatus] = await Promise.all([
+          this.ContentObjectLibraryId({objectId: streamId}),
+          this.StreamStatus({name: streamId})
+        ]);
+
+        value.input.name = await this.ContentObjectMetadata({
+          libraryId: streamLibraryId,
+          objectId: streamId,
+          metadataSubtree: "/public/name",
+        });
+        value.input.status = streamStatus?.state;
+      } catch(error) {
+        // Stream object may have been deleted
+        value.input.status = "unavailable";
+      }
     }
   );
 
@@ -3559,9 +3581,13 @@ exports.OutputsList = async function({libraryId, objectId, includeState=true}) {
 
   const nodeUrls = {};
   await this.utils.LimitedMap(10, nodeIds, async nodeId => {
-    const nodes = await this.SpaceNodes({matchNodeId: nodeId});
-    const fabricUrl = nodes?.[0]?.services?.fabric_api?.urls?.[0];
-    if(fabricUrl) { nodeUrls[nodeId] = fabricUrl; }
+    try {
+      const nodes = await this.SpaceNodes({matchNodeId: nodeId});
+      const fabricUrl = nodes?.[0]?.services?.fabric_api?.urls?.[0];
+      if(fabricUrl) { nodeUrls[nodeId] = fabricUrl; }
+    } catch(error) {
+      this.Log(`Failed to resolve node ${nodeId}: ${error.message}`, true);
+    }
   });
 
   // Rewrite fabric-generated srt_pull URLs to the egress host, and group outputs by node

package/utilities/MediaIngest.js

@@ -0,0 +1,424 @@
+// Create or update a playable media object from specified file(s)
+const R = require("ramda");
+const fs = require("fs");
+
+const {ModOpt, NewOpt} = require("./lib/options");
+const Utility = require("./lib/Utility");
+
+const ABR = require("@eluvio/elv-abr-profile");
+
+const Client = require("./lib/concerns/Client");
+const CloudFile = require("./lib/concerns/CloudFile");
+const Finalize = require("./lib/concerns/Finalize");
+const LocalFile = require("./lib/concerns/LocalFile");
+const LRO = require("./lib/concerns/LRO");
+const ArgLibraryId = require("./lib/concerns/ArgLibraryId");
+const ArgTenant = require("./lib/concerns/ArgTenant");
+const {seconds} = require("./lib/helpers");
+
+const AbrProfile = require("./lib/abr_profiles/abr_profile_4k_all.json");
+const STATUS_FILE = "./ingest-status.json";
+
+// Status object - used for reporting and resume.
+const status = {
+  state: "",
+  object_id: "",
+  write_token: "",
+  master_created: false
+};
+
+const SetStatus = statusUpdate => {
+  Object.assign(status, statusUpdate);
+  fs.writeFileSync(STATUS_FILE, `${JSON.stringify(status, null, 2)}\n`);
+};
+
+class MediaIngest extends Utility {
+  blueprint() {
+    return {
+      concerns: [Client, CloudFile, Finalize, LocalFile, ArgLibraryId, ArgTenant, LRO],
+      options: [
+        ModOpt("libraryId", {demand: false, forX: "new media object"}),
+        NewOpt("objectId", {
+          demand: false,
+          descTemplate: "Existing media object to ingest into",
+          type: "string"
+        }),
+        NewOpt("title", {
+          demand: false,
+          descTemplate: "Title for new media object (required unless --object-id is given)",
+          type: "string"
+        }),
+        NewOpt("drm", {
+          default: false,
+          descTemplate: "Use DRM for playback",
+          type: "boolean"
+        }),
+        NewOpt("hdrInfoFile", {
+          descTemplate: "Path to HDR info JSON file to enable HDR10 ingest. File format: { \"master_display\": \"G(...)B(...)R(...)WP(...)L(...)\", \"max_cll\": \"...,...\" }",
+          type: "string"
+        }),
+        NewOpt("finalize", {
+          default: true,
+          descTemplate: "Finalize the content object write token after ingest. Use --no-finalize to leave the draft open.",
+          type: "boolean"
+        }),
+        NewOpt("offeringKey", {
+          default: "default",
+          descTemplate: "Offering key (default: 'default')",
+          type: "string"
+        }),
+        NewOpt("variantKey", {
+          default: "default",
+          descTemplate: "Master variant key (default: 'default')",
+          type: "string"
+        }),
+        ModOpt("files", {forX: "for new media object"})
+      ]
+    };
+  }
+
+  async body() {
+    const logger = this.logger;
+
+    const s3Reference = Boolean(this.args.s3Reference);
+    const useS3 = Boolean(this.args.s3Copy || s3Reference);
+    const access = useS3
+      ? this.concerns.CloudFile.credentialSet()
+      : undefined;
+
+    let fileHandles = [];
+    const fileInfo = useS3
+      ? this.concerns.CloudFile.fileInfo()
+      : this.concerns.LocalFile.fileInfo(fileHandles);
+    const fileConcern = useS3
+      ? this.concerns.CloudFile
+      : this.concerns.LocalFile;
+
+    // delay getting elvClient until this point so script exits faster
+    // if there is a validation error above
+    const client = await this.concerns.Client.get();
+
+    // get type from Tenant
+    const tenantInfo = await this.concerns.ArgTenant.tenantInfo();
+
+    const type = tenantInfo.typeTitle;
+
+    if(R.isNil(type)) throw Error("Library does not specify content type for media ingests");
+
+    let {drm, finalize, hdrInfoFile, libraryId, objectId, offeringKey, title, variantKey} = this.args;
+    const noFinalize = !finalize;
+    const copy = !s3Reference;
+    const encrypt = !s3Reference;
+
+    let id;
+    let nodeUrl;
+    let writeToken;
+    if(status.write_token) {
+      id = status.object_id;
+      writeToken = status.write_token;
+      if(!libraryId) {
+        libraryId = await client.ContentObjectLibraryId({objectId: id});
+      }
+      // Re-pin client to the node holding the draft
+      nodeUrl = await client.WriteTokenNodeUrlNetwork({writeToken});
+      await client.SetNodes({fabricURIs: [nodeUrl]});
+      logger.log(`Resuming from write token ${writeToken} on ${nodeUrl}`);
+    } else {
+      if(!libraryId && !objectId) {
+        throw Error("One of library-id or object-id must be specified");
+      }
+
+      if(!objectId && !title) {
+        throw Error("--title is required when creating a new media object");
+      }
+
+      if(objectId) {
+        if(!libraryId) {
+          libraryId = await client.ContentObjectLibraryId({objectId});
+        }
+        const editResp = await client.EditContentObject({
+          libraryId,
+          objectId,
+          options: type ? { type } : {}
+        });
+        id = objectId;
+        writeToken = editResp.writeToken;
+        nodeUrl = editResp.nodeUrl;
+      } else {
+        const createResp = await client.CreateContentObject({
+          libraryId,
+          options: type ? { type } : {}
+        });
+        id = createResp.id;
+        writeToken = createResp.writeToken;
+        nodeUrl = createResp.nodeUrl;
+      }
+
+      await client.authClient.GenerateAuthorizationToken({
+        libraryId,
+        objectId: id,
+        update: true
+      });
+
+      status.object_id = id;
+      status.write_token = writeToken;
+    }
+
+    SetStatus({state: "object_created"});
+
+    if(!nodeUrl) {
+      nodeUrl = client.WriteTokenNodeUrlLocal({writeToken});
+    }
+    if(!nodeUrl) {
+      nodeUrl = await client.WriteTokenNodeUrlNetwork({writeToken});
+    }
+
+    logger.data("object_id", id);
+
+    if(!status.master_created) {
+      logger.log("Uploading files...");
+
+      const createMasterResponse = await client.CreateProductionMaster({
+        libraryId,
+        type,
+        name: title,
+        fileInfo,
+        encrypt,
+        access,
+        copy,
+        callback: fileConcern.callback,
+        writeToken
+      });
+
+      // Close file handles (if any)
+      this.concerns.LocalFile.closeFileHandles(fileHandles);
+
+      logger.errorsAndWarnings(createMasterResponse);
+
+      logger.logList(
+        "",
+        "Production master default variant created:",
+        `  Object ID: ${id}`,
+        `  Write token: ${writeToken}`,
+        ""
+      );
+
+      if(!R.isNil(createMasterResponse.errors) && !R.isEmpty(createMasterResponse.errors)) throw Error(`Error(s) encountered while inspecting uploaded files: ${createMasterResponse.errors.join("\n")}`);
+
+      status.master_created = true;
+    } else {
+      this.concerns.LocalFile.closeFileHandles(fileHandles);
+      logger.log(`Skipping upload + master creation (resume), write token ${writeToken}`);
+    }
+
+    SetStatus({state: "master_created", master_created: true});
+
+    // get production master metadata
+    const masterMetadata = (await client.ContentObjectMetadata({
+      libraryId,
+      objectId: id,
+      writeToken,
+      metadataSubtree: "/production_master"
+    }));
+
+    const sources = R.prop("sources", masterMetadata);
+    const variant = R.path(["variants", variantKey], masterMetadata);
+    if(R.isNil(variant)) throw Error(`Variant '${variantKey}' not found in production master metadata`);
+
+    // add info on source files and variant to data if --json selected
+    if(this.args.json) {
+      logger.data("media_files", sources);
+      logger.data("variant_key", variantKey);
+      logger.data("variant", variant);
+    }
+
+    // HDR: inject HDR info into first video stream's sources metadata
+    let hdrInfo;
+    if(hdrInfoFile) {
+      hdrInfo = JSON.parse(fs.readFileSync(hdrInfoFile, "utf8"));
+      const fileName = Object.keys(sources)[0];
+      const streams = sources[fileName].streams;
+      const videoStreamIdx = streams.findIndex(s => s.type === "StreamVideo");
+      if(videoStreamIdx < 0) throw Error(`No video stream found in /production_master/sources/${fileName}`);
+
+      streams[videoStreamIdx].hdr = hdrInfo;
+      logger.log(`Adding HDR info to /production_master/sources/${fileName}/streams/${videoStreamIdx}/hdr`);
+
+      await client.ReplaceMetadata({
+        libraryId,
+        objectId: id,
+        writeToken,
+        metadataSubtree: "/production_master/sources",
+        metadata: sources
+      });
+    }
+
+    // generate ABR profile
+    const genProfileRetVal = ABR.ABRProfileForVariant(sources, variant, AbrProfile);
+    if(!genProfileRetVal.ok) throw Error(`Error(s) encountered while generating ABR profile: ${genProfileRetVal.errors.join("\n")}`);
+
+    // filter DRM/clear as needed
+    const filterProfileRetVal = drm ?
+      ABR.ProfileExcludeClear(genProfileRetVal.result) :
+      ABR.ProfileExcludeDRM(genProfileRetVal.result);
+    if(!filterProfileRetVal.ok) throw Error(`Error(s) encountered while setting playout formats: ${filterProfileRetVal.errors.join("\n")}`);
+
+    // HDR: add HEVC/10-bit segment specs to the profile
+    if(hdrInfo) {
+      filterProfileRetVal.result.segment_specs = filterProfileRetVal.result.segment_specs || {};
+      filterProfileRetVal.result.segment_specs.video = {
+        ...(filterProfileRetVal.result.segment_specs.video || {}),
+        bit_depth: 10,
+        encoding: "h265"
+      };
+    }
+
+    // set up mezzanine offering
+    logger.log("Setting up media file conversion...");
+    const createMezResponse = await client.CreateABRMezzanine({
+      name: title,
+      libraryId,
+      writeToken,
+      type,
+      masterWriteToken: writeToken,
+      variant: variantKey,
+      offeringKey,
+      abrProfile: filterProfileRetVal.result
+    });
+
+    logger.errorsAndWarnings(createMezResponse);
+    const createMezErrors = createMezResponse.errors;
+    if(!R.isNil(createMezErrors) && !R.isEmpty(createMezErrors)) throw Error(`Error(s) encountered while setting up media file conversion: ${createMezErrors.join("\n")}`);
+
+    SetStatus({state: "mez_created", mez_created: true});
+
+    logger.log("Starting conversion to streaming format...");
+
+    const startJobsResponse = await client.StartABRMezzanineJobs({
+      libraryId,
+      objectId: id,
+      offeringKey,
+      writeToken
+    });
+
+    logger.errorsAndWarnings(startJobsResponse);
+    const startJobsErrors = createMezResponse.errors;
+    if(!R.isNil(startJobsErrors) && !R.isEmpty(startJobsErrors)) throw Error(`Error(s) encountered while starting file conversion: ${startJobsErrors.join("\n")}`);
+
+    writeToken = startJobsResponse.writeToken || writeToken;
+    nodeUrl = startJobsResponse.nodeUrl || nodeUrl;
+
+    const lroWriteToken = R.path(["lro_draft", "write_token"], startJobsResponse) || writeToken;
+    const lroNode = R.path(["lro_draft", "node"], startJobsResponse) || nodeUrl;
+
+    logger.data("library_id", libraryId);
+    logger.data("object_id", id);
+    logger.data("offering_key", offeringKey);
+    logger.data("write_token", lroWriteToken);
+    logger.data("write_node", lroNode);
+    logger.data("node_endpoint", lroNode);
+
+    logger.logList(
+      "",
+      `Library ID: ${libraryId}`,
+      `Object ID: ${id}`,
+      `Variant: ${variantKey}`,
+      `Offering: ${offeringKey}`,
+      `Write Token: ${lroWriteToken}`,
+      `Write Node: ${lroNode}`,
+      ""
+    );
+
+    SetStatus({state: "mez_started", mez_started: true, mez_job: startJobsResponse});
+
+    logger.log("Progress:");
+
+    const lro = this.concerns.LRO;
+    let done = false;
+    let lastStatus;
+    while(!done) {
+      const statusMap = await lro.status({libraryId, objectId: id, writeToken}); // TODO: check how offering key is used, if at all
+      const statusSummary = lro.statusSummary(statusMap);
+      lastStatus = statusSummary.run_state;
+      if(lastStatus !== LRO.STATE_RUNNING) done = true;
+      logger.log(`run_state: ${lastStatus}`);
+      const eta = statusSummary.estimated_time_left_h_m_s;
+      if(eta) logger.log(`estimated time left: ${eta}`);
+      await seconds(15);
+    }
+
+    const finalizeAbrResponse = await client.FinalizeABRMezzanine({
+      libraryId,
+      objectId: id,
+      writeToken,
+      offeringKey
+    });
+
+    logger.errorsAndWarnings(finalizeAbrResponse);
+    const finalizeErrors = finalizeAbrResponse.errors;
+    if(!R.isNil(finalizeErrors) && !R.isEmpty(finalizeErrors)) throw Error(`Error(s) encountered while finalizing object: ${finalizeErrors.join("\n")}`);
+
+    if(s3Reference) {
+      logger.log("Retaining S3 source file reference.");
+    } else {
+      logger.log("Deleting source files...");
+      await client.DeleteFiles({
+        libraryId,
+        objectId: id,
+        writeToken,
+        filePaths: fileInfo.map(f => f.path)
+      });
+    }
+    nodeUrl = client.WriteTokenNodeUrlLocal({writeToken}) || nodeUrl;
+
+    let latestHash = "not finalized";
+
+    if(noFinalize) {
+      logger.log("Skipping content object finalization (--no-finalize).");
+    } else {
+      logger.log("Finalizing content object...");
+      const sourceFileName = fileInfo[0].path.split("/").pop();
+      const finalizeObjResponse = await client.FinalizeContentObject({
+        libraryId,
+        objectId: id,
+        writeToken,
+        commitMessage: `Ingest ${sourceFileName}${hdrInfoFile ? " [HDR]" : ""}`
+      });
+
+      latestHash = finalizeObjResponse.hash;
+    }
+
+    logger.data("version_hash", latestHash);
+    logger.data("write_token", writeToken);
+    logger.data("write_node", nodeUrl);
+    logger.data("node_endpoint", nodeUrl);
+
+    logger.logList(
+      "",
+      noFinalize ? "Playable media object draft prepared:" : "Playable media object created:",
+      `  Object ID: ${id}`,
+      `  Version Hash: ${latestHash}`,
+      `  Write Token: ${writeToken}`,
+      `  Node Endpoint: ${nodeUrl}`,
+      ""
+    );
+
+    if(!noFinalize) {
+      await this.concerns.Finalize.waitForPublish({
+        latestHash,
+        libraryId,
+        objectId: id
+      });
+    }
+  }
+
+  header() {
+    return "Create playable media object via media ingest";
+  }
+}
+
+if(require.main === module) {
+  Utility.cmdLineInvoke(MediaIngest);
+} else {
+  module.exports = MediaIngest;
+}

package/utilities/MigrateEncryptionConkForNewUser.js

@@ -0,0 +1,62 @@
+const {NewOpt} = require("./lib/options");
+const Utility = require("./lib/Utility");
+
+const Client = require("./lib/concerns/Client");
+const Edit = require("./lib/concerns/Edit");
+
+class MigrateEncryptionConkForNewUser extends Utility {
+  blueprint() {
+    return {
+      concerns: [Client, Edit],
+      options: [
+        NewOpt("objectId",
+          {
+            descTemplate: "object Id",
+            type: "string",
+            demand: true
+          }),
+        NewOpt("publicKey",
+          {
+            descTemplate: "new user public key with/without encoded form",
+            type: "string",
+            demand: true
+          })
+      ]
+    };
+  }
+
+  async body() {
+    const logger = this.logger;
+    const {objectId, publicKey} = this.args;
+
+    const client = await this.concerns.Client.get();
+    const libraryId = await client.ContentObjectLibraryId({objectId});
+
+    const res =  await client.EditContentObject({libraryId, objectId});
+
+    await client.MigrateEncryptionConkForUserProvided({
+      libraryId,
+      objectId,
+      writeToken: res.writeToken,
+      newEncodedUserPublicKey: publicKey
+    });
+    logger.log("Migration of encryption conk is complete...");
+
+    const versionHash = await this.concerns.Edit.finalize({
+      libraryId,
+      objectId,
+      writeToken:res.writeToken
+    });
+    logger.log("version hash", versionHash);
+  }
+
+  header() {
+    return `Add encryption conk for user provided '${this.args.objectId}':`;
+  }
+}
+
+if(require.main === module) {
+  Utility.cmdLineInvoke(MigrateEncryptionConkForNewUser);
+} else {
+  module.exports = MigrateEncryptionConkForNewUser;
+}

package/utilities/ObjectAddFiles.js

@@ -1,6 +1,6 @@
 // Create new production master from specified file(s)
 
-const {ModOpt, StdOpt} = require("./lib/options");
+const {ModOpt, StdOpt, NewOpt} = require("./lib/options");
 const Utility = require("./lib/Utility");
 
 const ArgNoWait = require("./lib/concerns/ArgNoWait");
@@ -16,14 +16,19 @@ class ObjectAddFiles extends Utility {
       concerns: [Logger, ExistObj, Edit, ArgNoWait, LocalFile, CloudFile],
       options: [
         ModOpt("files", {X: "to add"}),
-        StdOpt("encrypt", {X: "uploaded files"})
+        StdOpt("encrypt", {X: "uploaded files"}),
+        NewOpt("resume", {
+          descTemplate: "If specified, resume jobs for the given write token (--resume 'write_token')",
+          group: "Main",
+          type: "string"
+        })
       ]
     };
   }
 
   async body() {
     const logger = this.logger;
-    const {encrypt, noWait} = this.args;
+    const {encrypt, noWait, resume: resumeArg} = this.args;
 
     let access;
     if(this.args.s3Reference || this.args.s3Copy) access = this.concerns.CloudFile.credentialSet();
@@ -35,10 +40,22 @@ class ObjectAddFiles extends Utility {
 
     const {libraryId, objectId} = await this.concerns.ExistObj.argsProc();
 
-    const writeToken = await this.concerns.Edit.getWriteToken({
-      libraryId,
-      objectId
-    });
+    let writeToken;
+    let resume=false;
+    if(resumeArg) {
+      writeToken = resumeArg;
+      resume=true;
+    } else {
+      // create new write token
+      writeToken = await this.concerns.Edit.getWriteToken({
+        libraryId,
+        objectId
+      });
+    }
+    logger.data("WRITE_TOKEN", writeToken);
+    // eslint-disable-next-line no-console
+    console.log("Using write token:", writeToken);
+
 
     if(access) {
       await this.concerns.CloudFile.add({
@@ -47,7 +64,8 @@ class ObjectAddFiles extends Utility {
         writeToken,
         access,
         fileInfo,
-        encrypt
+        encrypt,
+        resume,
       });
     } else {
       await this.concerns.LocalFile.add({
@@ -55,12 +73,14 @@ class ObjectAddFiles extends Utility {
         objectId,
         writeToken,
         fileInfo,
-        encrypt
+        encrypt,
+        resume,
       });
       // Close file handles
       this.concerns.LocalFile.closeFileHandles(fileHandles);
     }
 
+    // finalize the write token
     const hash = await this.concerns.Edit.finalize({
       libraryId,
       noWait,