@oreohq/ytdl-core 4.15.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (C) 2012-present by fent
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE. 

package/README.md

@@ -0,0 +1,209 @@
+# @oreohq/ytdl-core
+
+Oreo HQ fork of `ytdl-core`. This fork is dedicated to fixing bugs and adding features that are not merged into the original repo as soon as possible.
+
+## Installation
+
+```bash
+npm install @oreohq/ytdl-core@latest
+```
+
+Make sure you're installing the latest version of `@oreohq/ytdl-core` to keep up with the latest fixes.
+
+## Usage
+
+```js
+const ytdl = require("@oreohq/ytdl-core");
+// TypeScript: import ytdl from '@oreohq/ytdl-core'; with --esModuleInterop
+// TypeScript: import * as ytdl from '@oreohq/ytdl-core'; with --allowSyntheticDefaultImports
+// TypeScript: import ytdl = require('@oreohq/ytdl-core'); with neither of the above
+
+// Download a video
+ytdl("https://www.youtube.com/watch?v=G9KVla9gwbY").pipe(require("fs").createWriteStream("video.mp4"));
+
+// Get video info
+ytdl.getBasicInfo("https://www.youtube.com/watch?v=G9KVla9gwbY").then(info => {
+  console.log(info.videoDetails.title);
+});
+
+// Get video info with download formats
+ytdl.getInfo("https://www.youtube.com/watch?v=G9KVla9gwbY").then(info => {
+  console.log(info.formats);
+});
+```
+
+### Cookies Support
+
+```js
+const ytdl = require("@oreohq/ytdl-core");
+
+// (Optional) Below are examples, NOT the recommended options
+const cookies = [
+  { name: "cookie1", value: "COOKIE1_HERE" },
+  { name: "cookie2", value: "COOKIE2_HERE" },
+];
+
+// (Optional) http-cookie-agent / undici agent options
+// Below are examples, NOT the recommended options
+const agentOptions = {
+  pipelining: 5,
+  maxRedirections: 0,
+  localAddress: "127.0.0.1",
+};
+
+// agent should be created once if you don't want to change your cookie
+const agent = ytdl.createAgent(cookies, agentOptions);
+
+ytdl.getBasicInfo("https://www.youtube.com/watch?v=G9KVla9gwbY", { agent });
+ytdl.getInfo("https://www.youtube.com/watch?v=G9KVla9gwbY", { agent });
+```
+
+#### How to get cookies
+
+- Install [EditThisCookie](http://www.editthiscookie.com/) extension for your browser.
+- Go to [YouTube](https://www.youtube.com/).
+- Log in to your account. (You should use a new account for this purpose)
+- Click on the extension icon and click "Export" icon.
+- Your cookies will be added to your clipboard and paste it into your code.
+
+> [!WARNING]
+> Don't logout it by clicking logout button on youtube/google account manager, it will expire your cookies.
+> You can delete your browser's cookies to log it out on your browser.
+> Or use incognito mode to get your cookies then close it.
+
+> [!WARNING]
+> Paste all the cookies array from clipboard into `createAgent` function. Don't remove/edit any cookies if you don't know what you're doing.
+
+> [!WARNING]
+> Make sure your account, which logged in when you getting your cookies, use 1 IP at the same time only. It will make your cookies alive longer.
+
+```js
+const ytdl = require("@oreohq/ytdl-core");
+const agent = ytdl.createAgent([
+  {
+    domain: ".youtube.com",
+    expirationDate: 1234567890,
+    hostOnly: false,
+    httpOnly: true,
+    name: "---xxx---",
+    path: "/",
+    sameSite: "no_restriction",
+    secure: true,
+    session: false,
+    value: "---xxx---",
+  },
+  {
+    "...": "...",
+  },
+]);
+```
+
+- Or you can paste your cookies array into a file and use `fs.readFileSync` to read it.
+
+```js
+const ytdl = require("@oreohq/ytdl-core");
+const fs = require("fs");
+const agent = ytdl.createAgent(JSON.parse(fs.readFileSync("cookies.json")));
+```
+
+### Proxy Support
+
+```js
+const ytdl = require("@oreohq/ytdl-core");
+
+const agent = ytdl.createProxyAgent({ uri: "my.proxy.server" });
+
+ytdl.getBasicInfo("https://www.youtube.com/watch?v=G9KVla9gwbY", { agent });
+ytdl.getInfo("https://www.youtube.com/watch?v=G9KVla9gwbY", { agent });
+```
+
+Use both proxy and cookies:
+
+```js
+const ytdl = require("@oreohq/ytdl-core");
+
+const agent = ytdl.createProxyAgent({ uri: "my.proxy.server" }, [{ name: "cookie", value: "COOKIE_HERE" }]);
+
+ytdl.getBasicInfo("https://www.youtube.com/watch?v=G9KVla9gwbY", { agent });
+ytdl.getInfo("https://www.youtube.com/watch?v=G9KVla9gwbY", { agent });
+```
+
+### IP Rotation
+
+_Built-in ip rotation (`getRandomIPv6`) won't be updated and will be removed in the future, create your own ip rotation instead._
+
+To implement IP rotation, you need to assign the desired IP address to the `localAddress` property within `undici.Agent.Options`.
+Therefore, you'll need to use a different `ytdl.Agent` for each IP address you want to use.
+
+```js
+const ytdl = require("@oreohq/ytdl-core");
+const { getRandomIPv6 } = require("@oreohq/ytdl-core/lib/utils");
+
+const agentForARandomIP = ytdl.createAgent(undefined, {
+  localAddress: getRandomIPv6("2001:2::/48"),
+});
+
+ytdl.getBasicInfo("https://www.youtube.com/watch?v=G9KVla9gwbY", { agent: agentForARandomIP });
+
+const agentForAnotherRandomIP = ytdl.createAgent(undefined, {
+  localAddress: getRandomIPv6("2001:2::/48"),
+});
+
+ytdl.getInfo("https://www.youtube.com/watch?v=G9KVla9gwbY", { agent: agentForAnotherRandomIP });
+```
+
+## API
+
+You can find the API documentation in the [original repo](https://github.com/fent/node-ytdl-core#api). Except a few changes:
+
+### `ytdl.getInfoOptions`
+
+- `requestOptions` is now `undici`'s [`RequestOptions`](https://github.com/nodejs/undici#undicirequesturl-options-promise).
+- `agent`: [`ytdl.Agent`](https://github.com/oreohq/ytdl-core/blob/master/typings/index.d.ts#L10-L14)
+- `playerClients`: An array of player clients to use. Accepts `WEB`, `WEB_CREATOR`, `IOS`, and `ANDROID`. Defaults to `["WEB_CREATOR", "IOS"]`.
+
+### `ytdl.createAgent([cookies]): ytdl.Agent`
+
+`cookies`: an array of json cookies exported with [EditThisCookie](http://www.editthiscookie.com/).
+
+### `ytdl.createProxyAgent(proxy[, cookies]): ytdl.Agent`
+
+`proxy`: [`ProxyAgentOptions`](https://github.com/nodejs/undici/blob/main/docs/api/ProxyAgent.md#parameter-proxyagentoptions) contains your proxy server information.
+
+#### How to implement `ytdl.Agent` with your own Dispatcher
+
+You can find the example [here](https://github.com/oreohq/ytdl-core/blob/master/lib/cookie.js#L73-L86)
+
+## Limitations
+
+ytdl cannot download videos that fall into the following
+
+- Regionally restricted (requires a [proxy](#proxy-support))
+- Private (if you have access, requires [cookies](#cookies-support))
+- Rentals (if you have access, requires [cookies](#cookies-support))
+- YouTube Premium content (if you have access, requires [cookies](#cookies-support))
+- Only [HLS Livestreams](https://en.wikipedia.org/wiki/HTTP_Live_Streaming) are currently supported. Other formats will get filtered out in ytdl.chooseFormats
+
+Generated download links are valid for 6 hours, and may only be downloadable from the same IP address.
+
+## Rate Limiting
+
+When doing too many requests YouTube might block. This will result in your requests getting denied with HTTP-StatusCode 429. The following steps might help you:
+
+- Update `@oreohq/ytdl-core` to the latest version
+- Use proxies (you can find an example [here](#proxy-support))
+- Extend the Proxy Idea by rotating (IPv6-)Addresses
+  - read [this](https://github.com/fent/node-ytdl-core#how-does-using-an-ipv6-block-help) for more information about this
+- Use cookies (you can find an example [here](#cookies-support))
+  - for this to take effect you have to FIRST wait for the current rate limit to expire
+- Wait it out (it usually goes away within a few days)
+
+## Update Checks
+
+The issue of using an outdated version of ytdl-core became so prevalent, that ytdl-core now checks for updates at run time, and every 12 hours. If it finds an update, it will print a warning to the console advising you to update. Due to the nature of this library, it is important to always use the latest version as YouTube continues to update.
+
+If you'd like to disable this update check, you can do so by providing the `YTDL_NO_UPDATE` env variable.
+
+```
+env YTDL_NO_UPDATE=1 node myapp.js
+```

package/lib/agent.js

@@ -0,0 +1,100 @@
+const { ProxyAgent } = require("undici");
+const { Cookie, CookieJar, canonicalDomain } = require("tough-cookie");
+const { CookieAgent, CookieClient } = require("http-cookie-agent/undici");
+
+const convertSameSite = sameSite => {
+  switch (sameSite) {
+    case "strict":
+      return "strict";
+    case "lax":
+      return "lax";
+    case "no_restriction":
+    case "unspecified":
+    default:
+      return "none";
+  }
+};
+
+const convertCookie = cookie =>
+  cookie instanceof Cookie
+    ? cookie
+    : new Cookie({
+        key: cookie.name,
+        value: cookie.value,
+        expires: typeof cookie.expirationDate === "number" ? new Date(cookie.expirationDate * 1000) : "Infinity",
+        domain: canonicalDomain(cookie.domain),
+        path: cookie.path,
+        secure: cookie.secure,
+        httpOnly: cookie.httpOnly,
+        sameSite: convertSameSite(cookie.sameSite),
+        hostOnly: cookie.hostOnly,
+      });
+
+const addCookies = (exports.addCookies = (jar, cookies) => {
+  if (!cookies || !Array.isArray(cookies)) {
+    throw new Error("cookies must be an array");
+  }
+  if (!cookies.some(c => c.name === "SOCS")) {
+    cookies.push({
+      domain: ".youtube.com",
+      hostOnly: false,
+      httpOnly: false,
+      name: "SOCS",
+      path: "/",
+      sameSite: "lax",
+      secure: true,
+      session: false,
+      value: "CAI",
+    });
+  }
+  for (const cookie of cookies) {
+    jar.setCookieSync(convertCookie(cookie), "https://www.youtube.com");
+  }
+});
+
+exports.addCookiesFromString = (jar, cookies) => {
+  if (!cookies || typeof cookies !== "string") {
+    throw new Error("cookies must be a string");
+  }
+  return addCookies(
+    jar,
+    cookies
+      .split(";")
+      .map(c => Cookie.parse(c))
+      .filter(Boolean),
+  );
+};
+
+const createAgent = (exports.createAgent = (cookies = [], opts = {}) => {
+  const options = Object.assign({}, opts);
+  if (!options.cookies) {
+    const jar = new CookieJar();
+    addCookies(jar, cookies);
+    options.cookies = { jar };
+  }
+  return {
+    dispatcher: new CookieAgent(options),
+    localAddress: options.localAddress,
+    jar: options.cookies.jar,
+  };
+});
+
+exports.createProxyAgent = (options, cookies = []) => {
+  if (!cookies) cookies = [];
+  if (typeof options === "string") options = { uri: options };
+  if (options.factory) throw new Error("Cannot use factory with createProxyAgent");
+  const jar = new CookieJar();
+  addCookies(jar, cookies);
+  const proxyOptions = Object.assign(
+    {
+      factory: (origin, opts) => {
+        const o = Object.assign({ cookies: { jar } }, opts);
+        return new CookieClient(origin, o);
+      },
+    },
+    options,
+  );
+  return { dispatcher: new ProxyAgent(proxyOptions), jar, localAddress: options.localAddress };
+};
+
+exports.defaultAgent = createAgent();

package/lib/cache.js

@@ -0,0 +1,54 @@
+const { setTimeout } = require("timers");
+
+// A cache that expires.
+module.exports = class Cache extends Map {
+  constructor(timeout = 1000) {
+    super();
+    this.timeout = timeout;
+  }
+  set(key, value) {
+    if (this.has(key)) {
+      clearTimeout(super.get(key).tid);
+    }
+    super.set(key, {
+      tid: setTimeout(this.delete.bind(this, key), this.timeout).unref(),
+      value,
+    });
+  }
+  get(key) {
+    let entry = super.get(key);
+    if (entry) {
+      return entry.value;
+    }
+    return null;
+  }
+  getOrSet(key, fn) {
+    if (this.has(key)) {
+      return this.get(key);
+    } else {
+      let value = fn();
+      this.set(key, value);
+      (async () => {
+        try {
+          await value;
+        } catch (err) {
+          this.delete(key);
+        }
+      })();
+      return value;
+    }
+  }
+  delete(key) {
+    let entry = super.get(key);
+    if (entry) {
+      clearTimeout(entry.tid);
+      super.delete(key);
+    }
+  }
+  clear() {
+    for (let entry of this.values()) {
+      clearTimeout(entry.tid);
+    }
+    super.clear();
+  }
+};

package/lib/format-utils.js

@@ -0,0 +1,218 @@
+const utils = require("./utils");
+const FORMATS = require("./formats");
+
+// Use these to help sort formats, higher index is better.
+const audioEncodingRanks = ["mp4a", "mp3", "vorbis", "aac", "opus", "flac"];
+const videoEncodingRanks = ["mp4v", "avc1", "Sorenson H.283", "MPEG-4 Visual", "VP8", "VP9", "H.264"];
+
+const getVideoBitrate = format => format.bitrate || 0;
+const getVideoEncodingRank = format =>
+  videoEncodingRanks.findIndex(enc => format.codecs && format.codecs.includes(enc));
+const getAudioBitrate = format => format.audioBitrate || 0;
+const getAudioEncodingRank = format =>
+  audioEncodingRanks.findIndex(enc => format.codecs && format.codecs.includes(enc));
+
+/**
+ * Sort formats by a list of functions.
+ *
+ * @param {Object} a
+ * @param {Object} b
+ * @param {Array.<Function>} sortBy
+ * @returns {number}
+ */
+const sortFormatsBy = (a, b, sortBy) => {
+  let res = 0;
+  for (let fn of sortBy) {
+    res = fn(b) - fn(a);
+    if (res !== 0) {
+      break;
+    }
+  }
+  return res;
+};
+
+const sortFormatsByVideo = (a, b) =>
+  sortFormatsBy(a, b, [format => parseInt(format.qualityLabel), getVideoBitrate, getVideoEncodingRank]);
+
+const sortFormatsByAudio = (a, b) => sortFormatsBy(a, b, [getAudioBitrate, getAudioEncodingRank]);
+
+/**
+ * Sort formats from highest quality to lowest.
+ *
+ * @param {Object} a
+ * @param {Object} b
+ * @returns {number}
+ */
+exports.sortFormats = (a, b) =>
+  sortFormatsBy(a, b, [
+    // Formats with both video and audio are ranked highest.
+    format => +!!format.isHLS,
+    format => +!!format.isDashMPD,
+    format => +(format.contentLength > 0),
+    format => +(format.hasVideo && format.hasAudio),
+    format => +format.hasVideo,
+    format => parseInt(format.qualityLabel) || 0,
+    getVideoBitrate,
+    getAudioBitrate,
+    getVideoEncodingRank,
+    getAudioEncodingRank,
+  ]);
+
+/**
+ * Choose a format depending on the given options.
+ *
+ * @param {Array.<Object>} formats
+ * @param {Object} options
+ * @returns {Object}
+ * @throws {Error} when no format matches the filter/format rules
+ */
+exports.chooseFormat = (formats, options) => {
+  if (typeof options.format === "object") {
+    if (!options.format.url) {
+      throw Error("Invalid format given, did you use `ytdl.getInfo()`?");
+    }
+    return options.format;
+  }
+
+  if (options.filter) {
+    formats = exports.filterFormats(formats, options.filter);
+  }
+
+  // We currently only support HLS-Formats for livestreams
+  // So we (now) remove all non-HLS streams
+  if (formats.some(fmt => fmt.isHLS)) {
+    formats = formats.filter(fmt => fmt.isHLS || !fmt.isLive);
+  }
+
+  let format;
+  const quality = options.quality || "highest";
+  switch (quality) {
+    case "highest":
+      format = formats[0];
+      break;
+
+    case "lowest":
+      format = formats[formats.length - 1];
+      break;
+
+    case "highestaudio": {
+      formats = exports.filterFormats(formats, "audio");
+      formats.sort(sortFormatsByAudio);
+      // Filter for only the best audio format
+      const bestAudioFormat = formats[0];
+      formats = formats.filter(f => sortFormatsByAudio(bestAudioFormat, f) === 0);
+      // Check for the worst video quality for the best audio quality and pick according
+      // This does not loose default sorting of video encoding and bitrate
+      const worstVideoQuality = formats.map(f => parseInt(f.qualityLabel) || 0).sort((a, b) => a - b)[0];
+      format = formats.find(f => (parseInt(f.qualityLabel) || 0) === worstVideoQuality);
+      break;
+    }
+
+    case "lowestaudio":
+      formats = exports.filterFormats(formats, "audio");
+      formats.sort(sortFormatsByAudio);
+      format = formats[formats.length - 1];
+      break;
+
+    case "highestvideo": {
+      formats = exports.filterFormats(formats, "video");
+      formats.sort(sortFormatsByVideo);
+      // Filter for only the best video format
+      const bestVideoFormat = formats[0];
+      formats = formats.filter(f => sortFormatsByVideo(bestVideoFormat, f) === 0);
+      // Check for the worst audio quality for the best video quality and pick according
+      // This does not loose default sorting of audio encoding and bitrate
+      const worstAudioQuality = formats.map(f => f.audioBitrate || 0).sort((a, b) => a - b)[0];
+      format = formats.find(f => (f.audioBitrate || 0) === worstAudioQuality);
+      break;
+    }
+
+    case "lowestvideo":
+      formats = exports.filterFormats(formats, "video");
+      formats.sort(sortFormatsByVideo);
+      format = formats[formats.length - 1];
+      break;
+
+    default:
+      format = getFormatByQuality(quality, formats);
+      break;
+  }
+
+  if (!format) {
+    throw Error(`No such format found: ${quality}`);
+  }
+  return format;
+};
+
+/**
+ * Gets a format based on quality or array of quality's
+ *
+ * @param {string|[string]} quality
+ * @param {[Object]} formats
+ * @returns {Object}
+ */
+const getFormatByQuality = (quality, formats) => {
+  let getFormat = itag => formats.find(format => `${format.itag}` === `${itag}`);
+  if (Array.isArray(quality)) {
+    return getFormat(quality.find(q => getFormat(q)));
+  } else {
+    return getFormat(quality);
+  }
+};
+
+/**
+ * @param {Array.<Object>} formats
+ * @param {Function} filter
+ * @returns {Array.<Object>}
+ */
+exports.filterFormats = (formats, filter) => {
+  let fn;
+  switch (filter) {
+    case "videoandaudio":
+    case "audioandvideo":
+      fn = format => format.hasVideo && format.hasAudio;
+      break;
+
+    case "video":
+      fn = format => format.hasVideo;
+      break;
+
+    case "videoonly":
+      fn = format => format.hasVideo && !format.hasAudio;
+      break;
+
+    case "audio":
+      fn = format => format.hasAudio;
+      break;
+
+    case "audioonly":
+      fn = format => !format.hasVideo && format.hasAudio;
+      break;
+
+    default:
+      if (typeof filter === "function") {
+        fn = filter;
+      } else {
+        throw TypeError(`Given filter (${filter}) is not supported`);
+      }
+  }
+  return formats.filter(format => !!format.url && fn(format));
+};
+
+/**
+ * @param {Object} format
+ * @returns {Object}
+ */
+exports.addFormatMeta = format => {
+  format = Object.assign({}, FORMATS[format.itag], format);
+  format.hasVideo = !!format.qualityLabel;
+  format.hasAudio = !!format.audioBitrate;
+  format.container = format.mimeType ? format.mimeType.split(";")[0].split("/")[1] : null;
+  format.codecs = format.mimeType ? utils.between(format.mimeType, 'codecs="', '"') : null;
+  format.videoCodec = format.hasVideo && format.codecs ? format.codecs.split(", ")[0] : null;
+  format.audioCodec = format.hasAudio && format.codecs ? format.codecs.split(", ").slice(-1)[0] : null;
+  format.isLive = /\bsource[/=]yt_live_broadcast\b/.test(format.url);
+  format.isHLS = /\/manifest\/hls_(variant|playlist)\//.test(format.url);
+  format.isDashMPD = /\/manifest\/dash\//.test(format.url);
+  return format;
+};