@oreohq/ytdl-core 4.15.1

package/lib/index.js

@@ -0,0 +1,228 @@
+const PassThrough = require("stream").PassThrough;
+const getInfo = require("./info");
+const utils = require("./utils");
+const formatUtils = require("./format-utils");
+const urlUtils = require("./url-utils");
+const miniget = require("miniget");
+const m3u8stream = require("m3u8stream");
+const { parseTimestamp } = require("m3u8stream");
+const agent = require("./agent");
+
+/**
+ * @param {string} link
+ * @param {!Object} options
+ * @returns {ReadableStream}
+ */
+const ytdl = (link, options) => {
+  const stream = createStream(options);
+  ytdl.getInfo(link, options).then(
+    info => {
+      downloadFromInfoCallback(stream, info, options);
+    },
+    stream.emit.bind(stream, "error"),
+  );
+  return stream;
+};
+module.exports = ytdl;
+
+ytdl.getBasicInfo = getInfo.getBasicInfo;
+ytdl.getInfo = getInfo.getInfo;
+ytdl.chooseFormat = formatUtils.chooseFormat;
+ytdl.filterFormats = formatUtils.filterFormats;
+ytdl.validateID = urlUtils.validateID;
+ytdl.validateURL = urlUtils.validateURL;
+ytdl.getURLVideoID = urlUtils.getURLVideoID;
+ytdl.getVideoID = urlUtils.getVideoID;
+ytdl.createAgent = agent.createAgent;
+ytdl.createProxyAgent = agent.createProxyAgent;
+ytdl.cache = {
+  info: getInfo.cache,
+  watch: getInfo.watchPageCache,
+};
+ytdl.version = require("../package.json").version;
+
+const createStream = options => {
+  const stream = new PassThrough({
+    highWaterMark: (options && options.highWaterMark) || 1024 * 512,
+  });
+  stream._destroy = () => {
+    stream.destroyed = true;
+  };
+  return stream;
+};
+
+const pipeAndSetEvents = (req, stream, end) => {
+  // Forward events from the request to the stream.
+  ["abort", "request", "response", "error", "redirect", "retry", "reconnect"].forEach(event => {
+    req.prependListener(event, stream.emit.bind(stream, event));
+  });
+  req.pipe(stream, { end });
+};
+
+/**
+ * Chooses a format to download.
+ *
+ * @param {stream.Readable} stream
+ * @param {Object} info
+ * @param {Object} options
+ */
+const downloadFromInfoCallback = (stream, info, options) => {
+  options = options || {};
+
+  let err = utils.playError(info.player_response);
+  if (err) {
+    stream.emit("error", err);
+    return;
+  }
+
+  if (!info.formats.length) {
+    stream.emit("error", Error("This video is unavailable"));
+    return;
+  }
+
+  let format;
+  try {
+    format = formatUtils.chooseFormat(info.formats, options);
+  } catch (e) {
+    stream.emit("error", e);
+    return;
+  }
+  stream.emit("info", info, format);
+  if (stream.destroyed) {
+    return;
+  }
+
+  let contentLength,
+    downloaded = 0;
+  const ondata = chunk => {
+    downloaded += chunk.length;
+    stream.emit("progress", chunk.length, downloaded, contentLength);
+  };
+
+  utils.applyDefaultHeaders(options);
+  if (options.IPv6Block) {
+    options.requestOptions = Object.assign({}, options.requestOptions, {
+      localAddress: utils.getRandomIPv6(options.IPv6Block),
+    });
+  }
+  if (options.agent) {
+    if (options.agent.jar) {
+      utils.setPropInsensitive(
+        options.requestOptions.headers,
+        "cookie",
+        options.agent.jar.getCookieStringSync("https://www.youtube.com"),
+      );
+    }
+    if (options.agent.localAddress) {
+      options.requestOptions.localAddress = options.agent.localAddress;
+    }
+  }
+
+  // Download the file in chunks, in this case the default is 10MB,
+  // anything over this will cause youtube to throttle the download
+  const dlChunkSize = typeof options.dlChunkSize === "number" ? options.dlChunkSize : 1024 * 1024 * 10;
+  let req;
+  let shouldEnd = true;
+
+  if (format.isHLS || format.isDashMPD) {
+    req = m3u8stream(format.url, {
+      chunkReadahead: +info.live_chunk_readahead,
+      begin: options.begin || (format.isLive && Date.now()),
+      liveBuffer: options.liveBuffer,
+      requestOptions: options.requestOptions,
+      parser: format.isDashMPD ? "dash-mpd" : "m3u8",
+      id: format.itag,
+    });
+
+    req.on("progress", (segment, totalSegments) => {
+      stream.emit("progress", segment.size, segment.num, totalSegments);
+    });
+    pipeAndSetEvents(req, stream, shouldEnd);
+  } else {
+    const requestOptions = Object.assign({}, options.requestOptions, {
+      maxReconnects: 6,
+      maxRetries: 3,
+      backoff: { inc: 500, max: 10000 },
+    });
+
+    let shouldBeChunked = dlChunkSize !== 0 && (!format.hasAudio || !format.hasVideo);
+
+    if (shouldBeChunked) {
+      let start = (options.range && options.range.start) || 0;
+      let end = start + dlChunkSize;
+      const rangeEnd = options.range && options.range.end;
+
+      contentLength = options.range
+        ? (rangeEnd ? rangeEnd + 1 : parseInt(format.contentLength)) - start
+        : parseInt(format.contentLength);
+
+      const getNextChunk = () => {
+        if (stream.destroyed) return;
+        if (!rangeEnd && end >= contentLength) end = 0;
+        if (rangeEnd && end > rangeEnd) end = rangeEnd;
+        shouldEnd = !end || end === rangeEnd;
+
+        requestOptions.headers = Object.assign({}, requestOptions.headers, {
+          Range: `bytes=${start}-${end || ""}`,
+        });
+        req = miniget(format.url, requestOptions);
+        req.on("data", ondata);
+        req.on("end", () => {
+          if (stream.destroyed) return;
+          if (end && end !== rangeEnd) {
+            start = end + 1;
+            end += dlChunkSize;
+            getNextChunk();
+          }
+        });
+        pipeAndSetEvents(req, stream, shouldEnd);
+      };
+      getNextChunk();
+    } else {
+      // Audio only and video only formats don't support begin
+      if (options.begin) {
+        format.url += `&begin=${parseTimestamp(options.begin)}`;
+      }
+      if (options.range && (options.range.start || options.range.end)) {
+        requestOptions.headers = Object.assign({}, requestOptions.headers, {
+          Range: `bytes=${options.range.start || "0"}-${options.range.end || ""}`,
+        });
+      }
+      req = miniget(format.url, requestOptions);
+      req.on("response", res => {
+        if (stream.destroyed) return;
+        contentLength = contentLength || parseInt(res.headers["content-length"]);
+      });
+      req.on("data", ondata);
+      pipeAndSetEvents(req, stream, shouldEnd);
+    }
+  }
+
+  stream._destroy = () => {
+    stream.destroyed = true;
+    if (req) {
+      req.destroy();
+      req.end();
+    }
+  };
+};
+
+/**
+ * Can be used to download video after its `info` is gotten through
+ * `ytdl.getInfo()`. In case the user might want to look at the
+ * `info` object before deciding to download.
+ *
+ * @param {Object} info
+ * @param {!Object} options
+ * @returns {ReadableStream}
+ */
+ytdl.downloadFromInfo = (info, options) => {
+  const stream = createStream(options);
+  if (!info.full) {
+    throw Error("Cannot use `ytdl.downloadFromInfo()` when called with info from `ytdl.getBasicInfo()`");
+  }
+  setImmediate(() => {
+    downloadFromInfoCallback(stream, info, options);
+  });
+  return stream;
+};

package/lib/info-extras.js

@@ -0,0 +1,362 @@
+const utils = require("./utils");
+const qs = require("querystring");
+const { parseTimestamp } = require("m3u8stream");
+
+const BASE_URL = "https://www.youtube.com/watch?v=";
+const TITLE_TO_CATEGORY = {
+  song: { name: "Music", url: "https://music.youtube.com/" },
+};
+
+const getText = obj => (obj ? (obj.runs ? obj.runs[0].text : obj.simpleText) : null);
+
+/**
+ * Get video media.
+ *
+ * @param {Object} info
+ * @returns {Object}
+ */
+exports.getMedia = info => {
+  let media = {};
+  let results = [];
+  try {
+    results = info.response.contents.twoColumnWatchNextResults.results.results.contents;
+  } catch (err) {
+    // Do nothing
+  }
+
+  let result = results.find(v => v.videoSecondaryInfoRenderer);
+  if (!result) {
+    return {};
+  }
+
+  try {
+    let metadataRows = (result.metadataRowContainer || result.videoSecondaryInfoRenderer.metadataRowContainer)
+      .metadataRowContainerRenderer.rows;
+    for (let row of metadataRows) {
+      if (row.metadataRowRenderer) {
+        let title = getText(row.metadataRowRenderer.title).toLowerCase();
+        let contents = row.metadataRowRenderer.contents[0];
+        media[title] = getText(contents);
+        let runs = contents.runs;
+        if (runs && runs[0].navigationEndpoint) {
+          media[`${title}_url`] = new URL(
+            runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url,
+            BASE_URL,
+          ).toString();
+        }
+        if (title in TITLE_TO_CATEGORY) {
+          media.category = TITLE_TO_CATEGORY[title].name;
+          media.category_url = TITLE_TO_CATEGORY[title].url;
+        }
+      } else if (row.richMetadataRowRenderer) {
+        let contents = row.richMetadataRowRenderer.contents;
+        let boxArt = contents.filter(
+          meta => meta.richMetadataRenderer.style === "RICH_METADATA_RENDERER_STYLE_BOX_ART",
+        );
+        for (let { richMetadataRenderer } of boxArt) {
+          let meta = richMetadataRenderer;
+          media.year = getText(meta.subtitle);
+          let type = getText(meta.callToAction).split(" ")[1];
+          media[type] = getText(meta.title);
+          media[`${type}_url`] = new URL(meta.endpoint.commandMetadata.webCommandMetadata.url, BASE_URL).toString();
+          media.thumbnails = meta.thumbnail.thumbnails;
+        }
+        let topic = contents.filter(meta => meta.richMetadataRenderer.style === "RICH_METADATA_RENDERER_STYLE_TOPIC");
+        for (let { richMetadataRenderer } of topic) {
+          let meta = richMetadataRenderer;
+          media.category = getText(meta.title);
+          media.category_url = new URL(meta.endpoint.commandMetadata.webCommandMetadata.url, BASE_URL).toString();
+        }
+      }
+    }
+  } catch (err) {
+    // Do nothing.
+  }
+
+  return media;
+};
+
+const isVerified = badges => !!(badges && badges.find(b => b.metadataBadgeRenderer.tooltip === "Verified"));
+
+/**
+ * Get video author.
+ *
+ * @param {Object} info
+ * @returns {Object}
+ */
+exports.getAuthor = info => {
+  let channelId,
+    thumbnails = [],
+    subscriberCount,
+    verified = false;
+  try {
+    let results = info.response.contents.twoColumnWatchNextResults.results.results.contents;
+    let v = results.find(
+      v2 =>
+        v2.videoSecondaryInfoRenderer &&
+        v2.videoSecondaryInfoRenderer.owner &&
+        v2.videoSecondaryInfoRenderer.owner.videoOwnerRenderer,
+    );
+    let videoOwnerRenderer = v.videoSecondaryInfoRenderer.owner.videoOwnerRenderer;
+    channelId = videoOwnerRenderer.navigationEndpoint.browseEndpoint.browseId;
+    thumbnails = videoOwnerRenderer.thumbnail.thumbnails.map(thumbnail => {
+      thumbnail.url = new URL(thumbnail.url, BASE_URL).toString();
+      return thumbnail;
+    });
+    subscriberCount = utils.parseAbbreviatedNumber(getText(videoOwnerRenderer.subscriberCountText));
+    verified = isVerified(videoOwnerRenderer.badges);
+  } catch (err) {
+    // Do nothing.
+  }
+  try {
+    let videoDetails = info.player_response.microformat && info.player_response.microformat.playerMicroformatRenderer;
+    let id = (videoDetails && videoDetails.channelId) || channelId || info.player_response.videoDetails.channelId;
+    let author = {
+      id: id,
+      name: videoDetails ? videoDetails.ownerChannelName : info.player_response.videoDetails.author,
+      user: videoDetails ? videoDetails.ownerProfileUrl.split("/").slice(-1)[0] : null,
+      channel_url: `https://www.youtube.com/channel/${id}`,
+      external_channel_url: videoDetails ? `https://www.youtube.com/channel/${videoDetails.externalChannelId}` : "",
+      user_url: videoDetails ? new URL(videoDetails.ownerProfileUrl, BASE_URL).toString() : "",
+      thumbnails,
+      verified,
+      subscriber_count: subscriberCount,
+    };
+    if (thumbnails.length) {
+      utils.deprecate(author, "avatar", author.thumbnails[0].url, "author.avatar", "author.thumbnails[0].url");
+    }
+    return author;
+  } catch (err) {
+    return {};
+  }
+};
+
+const parseRelatedVideo = (details, rvsParams) => {
+  if (!details) return;
+  try {
+    let viewCount = getText(details.viewCountText);
+    let shortViewCount = getText(details.shortViewCountText);
+    let rvsDetails = rvsParams.find(elem => elem.id === details.videoId);
+    if (!/^\d/.test(shortViewCount)) {
+      shortViewCount = (rvsDetails && rvsDetails.short_view_count_text) || "";
+    }
+    viewCount = (/^\d/.test(viewCount) ? viewCount : shortViewCount).split(" ")[0];
+    let browseEndpoint = details.shortBylineText.runs[0].navigationEndpoint.browseEndpoint;
+    let channelId = browseEndpoint.browseId;
+    let name = getText(details.shortBylineText);
+    let user = (browseEndpoint.canonicalBaseUrl || "").split("/").slice(-1)[0];
+    let video = {
+      id: details.videoId,
+      title: getText(details.title),
+      published: getText(details.publishedTimeText),
+      author: {
+        id: channelId,
+        name,
+        user,
+        channel_url: `https://www.youtube.com/channel/${channelId}`,
+        user_url: `https://www.youtube.com/user/${user}`,
+        thumbnails: details.channelThumbnail.thumbnails.map(thumbnail => {
+          thumbnail.url = new URL(thumbnail.url, BASE_URL).toString();
+          return thumbnail;
+        }),
+        verified: isVerified(details.ownerBadges),
+
+        [Symbol.toPrimitive]() {
+          console.warn(
+            `\`relatedVideo.author\` will be removed in a near future release, ` +
+              `use \`relatedVideo.author.name\` instead.`,
+          );
+          return video.author.name;
+        },
+      },
+      short_view_count_text: shortViewCount.split(" ")[0],
+      view_count: viewCount.replace(/,/g, ""),
+      length_seconds: details.lengthText
+        ? Math.floor(parseTimestamp(getText(details.lengthText)) / 1000)
+        : rvsParams && `${rvsParams.length_seconds}`,
+      thumbnails: details.thumbnail.thumbnails,
+      richThumbnails: details.richThumbnail
+        ? details.richThumbnail.movingThumbnailRenderer.movingThumbnailDetails.thumbnails
+        : [],
+      isLive: !!(details.badges && details.badges.find(b => b.metadataBadgeRenderer.label === "LIVE NOW")),
+    };
+
+    utils.deprecate(
+      video,
+      "author_thumbnail",
+      video.author.thumbnails[0].url,
+      "relatedVideo.author_thumbnail",
+      "relatedVideo.author.thumbnails[0].url",
+    );
+    utils.deprecate(video, "ucid", video.author.id, "relatedVideo.ucid", "relatedVideo.author.id");
+    utils.deprecate(
+      video,
+      "video_thumbnail",
+      video.thumbnails[0].url,
+      "relatedVideo.video_thumbnail",
+      "relatedVideo.thumbnails[0].url",
+    );
+    return video;
+  } catch (err) {
+    // Skip.
+  }
+};
+
+/**
+ * Get related videos.
+ *
+ * @param {Object} info
+ * @returns {Array.<Object>}
+ */
+exports.getRelatedVideos = info => {
+  let rvsParams = [],
+    secondaryResults = [];
+  try {
+    rvsParams = info.response.webWatchNextResponseExtensionData.relatedVideoArgs.split(",").map(e => qs.parse(e));
+  } catch (err) {
+    // Do nothing.
+  }
+  try {
+    secondaryResults = info.response.contents.twoColumnWatchNextResults.secondaryResults.secondaryResults.results;
+  } catch (err) {
+    return [];
+  }
+  let videos = [];
+  for (let result of secondaryResults || []) {
+    let details = result.compactVideoRenderer;
+    if (details) {
+      let video = parseRelatedVideo(details, rvsParams);
+      if (video) videos.push(video);
+    } else {
+      let autoplay = result.compactAutoplayRenderer || result.itemSectionRenderer;
+      if (!autoplay || !Array.isArray(autoplay.contents)) continue;
+      for (let content of autoplay.contents) {
+        let video = parseRelatedVideo(content.compactVideoRenderer, rvsParams);
+        if (video) videos.push(video);
+      }
+    }
+  }
+  return videos;
+};
+
+/**
+ * Get like count.
+ *
+ * @param {Object} info
+ * @returns {number}
+ */
+exports.getLikes = info => {
+  try {
+    let contents = info.response.contents.twoColumnWatchNextResults.results.results.contents;
+    let video = contents.find(r => r.videoPrimaryInfoRenderer);
+    let buttons = video.videoPrimaryInfoRenderer.videoActions.menuRenderer.topLevelButtons;
+    let accessibilityText = buttons.find(b => b.segmentedLikeDislikeButtonViewModel).segmentedLikeDislikeButtonViewModel
+      .likeButtonViewModel.likeButtonViewModel.toggleButtonViewModel.toggleButtonViewModel.defaultButtonViewModel
+      .buttonViewModel.accessibilityText;
+    return parseInt(accessibilityText.match(/[\d,.]+/)[0].replace(/\D+/g, ""));
+  } catch (err) {
+    return null;
+  }
+};
+
+/**
+ * Cleans up a few fields on `videoDetails`.
+ *
+ * @param {Object} videoDetails
+ * @param {Object} info
+ * @returns {Object}
+ */
+exports.cleanVideoDetails = (videoDetails, info) => {
+  videoDetails.thumbnails = videoDetails.thumbnail.thumbnails;
+  delete videoDetails.thumbnail;
+  utils.deprecate(
+    videoDetails,
+    "thumbnail",
+    { thumbnails: videoDetails.thumbnails },
+    "videoDetails.thumbnail.thumbnails",
+    "videoDetails.thumbnails",
+  );
+  videoDetails.description = videoDetails.shortDescription || getText(videoDetails.description);
+  delete videoDetails.shortDescription;
+  utils.deprecate(
+    videoDetails,
+    "shortDescription",
+    videoDetails.description,
+    "videoDetails.shortDescription",
+    "videoDetails.description",
+  );
+
+  // Use more reliable `lengthSeconds` from `playerMicroformatRenderer`.
+  videoDetails.lengthSeconds =
+    (info.player_response.microformat && info.player_response.microformat.playerMicroformatRenderer.lengthSeconds) ||
+    info.player_response.videoDetails.lengthSeconds;
+  return videoDetails;
+};
+
+/**
+ * Get storyboards info.
+ *
+ * @param {Object} info
+ * @returns {Array.<Object>}
+ */
+exports.getStoryboards = info => {
+  const parts =
+    info.player_response.storyboards &&
+    info.player_response.storyboards.playerStoryboardSpecRenderer &&
+    info.player_response.storyboards.playerStoryboardSpecRenderer.spec &&
+    info.player_response.storyboards.playerStoryboardSpecRenderer.spec.split("|");
+
+  if (!parts) return [];
+
+  const url = new URL(parts.shift());
+
+  return parts.map((part, i) => {
+    let [thumbnailWidth, thumbnailHeight, thumbnailCount, columns, rows, interval, nameReplacement, sigh] =
+      part.split("#");
+
+    url.searchParams.set("sigh", sigh);
+
+    thumbnailCount = parseInt(thumbnailCount, 10);
+    columns = parseInt(columns, 10);
+    rows = parseInt(rows, 10);
+
+    const storyboardCount = Math.ceil(thumbnailCount / (columns * rows));
+
+    return {
+      templateUrl: url.toString().replace("$L", i).replace("$N", nameReplacement),
+      thumbnailWidth: parseInt(thumbnailWidth, 10),
+      thumbnailHeight: parseInt(thumbnailHeight, 10),
+      thumbnailCount,
+      interval: parseInt(interval, 10),
+      columns,
+      rows,
+      storyboardCount,
+    };
+  });
+};
+
+/**
+ * Get chapters info.
+ *
+ * @param {Object} info
+ * @returns {Array.<Object>}
+ */
+exports.getChapters = info => {
+  const playerOverlayRenderer =
+    info.response && info.response.playerOverlays && info.response.playerOverlays.playerOverlayRenderer;
+  const playerBar =
+    playerOverlayRenderer &&
+    playerOverlayRenderer.decoratedPlayerBarRenderer &&
+    playerOverlayRenderer.decoratedPlayerBarRenderer.decoratedPlayerBarRenderer &&
+    playerOverlayRenderer.decoratedPlayerBarRenderer.decoratedPlayerBarRenderer.playerBar;
+  const markersMap =
+    playerBar && playerBar.multiMarkersPlayerBarRenderer && playerBar.multiMarkersPlayerBarRenderer.markersMap;
+  const marker = Array.isArray(markersMap) && markersMap.find(m => m.value && Array.isArray(m.value.chapters));
+  if (!marker) return [];
+  const chapters = marker.value.chapters;
+
+  return chapters.map(chapter => ({
+    title: getText(chapter.chapterRenderer.title),
+    start_time: chapter.chapterRenderer.timeRangeStartMillis / 1000,
+  }));
+};