readability_js 0.0.1

data/lib/readability_js/node/node_modules/jsdom/lib/jsdom/living/xhr/XMLHttpRequestEventTarget-impl.js

@@ -0,0 +1,17 @@
+"use strict";
+const EventTargetImpl = require("../events/EventTarget-impl").implementation;
+const idlUtils = require("../generated/utils");
+const { setupForSimpleEventAccessors } = require("../helpers/create-event-accessor");
+
+const events = ["loadstart", "progress", "abort", "error", "load", "timeout", "loadend"];
+
+class XMLHttpRequestEventTargetImpl extends EventTargetImpl {
+  // TODO: remove this when we fix EventTargetImpl to use this._globalObject directly instead of using _ownerDocument.
+  // https://github.com/jsdom/jsdom/issues/2780
+  get _ownerDocument() {
+    return idlUtils.implForWrapper(this._globalObject._document);
+  }
+}
+setupForSimpleEventAccessors(XMLHttpRequestEventTargetImpl.prototype, events);
+
+exports.implementation = XMLHttpRequestEventTargetImpl;

data/lib/readability_js/node/node_modules/jsdom/lib/jsdom/living/xhr/XMLHttpRequestUpload-impl.js

@@ -0,0 +1,4 @@
+"use strict";
+const XMLHttpRequestEventTargetImpl = require("./XMLHttpRequestEventTarget-impl").implementation;
+
+exports.implementation = class XMLHttpRequestUploadImpl extends XMLHttpRequestEventTargetImpl {};

data/lib/readability_js/node/node_modules/jsdom/lib/jsdom/living/xhr/multipart-form-data.js

@@ -0,0 +1,105 @@
+"use strict";
+const conversions = require("webidl-conversions");
+// https://fetch.spec.whatwg.org/#concept-bodyinit-extract (note: always UTF-8)
+// https://html.spec.whatwg.org/multipage/form-control-infrastructure.html#multipart%2Fform-data-encoding-algorithm
+// https://andreubotella.github.io/multipart-form-data/
+
+const utf8Encoder = new TextEncoder();
+
+const contentDispositionPrefix = utf8Encoder.encode(`Content-Disposition: form-data; name="`);
+const filenamePrefix = utf8Encoder.encode(`; filename="`);
+const contentType = utf8Encoder.encode(`Content-Type: `);
+
+// https://andreubotella.github.io/multipart-form-data/#multipart-form-data-boundary
+function generateBoundary() {
+  let boundary = "--------------------------";
+  for (let i = 0; i < 24; ++i) {
+    boundary += Math.floor(Math.random() * 10).toString(16);
+  }
+  return utf8Encoder.encode(boundary);
+}
+
+// https://andreubotella.github.io/multipart-form-data/#escape-a-multipart-form-data-name
+function escapeName(name, isFilename = false) {
+  if (isFilename) {
+    name = conversions.USVString(name);
+  } else {
+    name = name.replace(/\r(?!\n)|(?<!\r)\n/g, "\r\n");
+  }
+
+  const encoded = utf8Encoder.encode(name);
+  const encodedWithSubs = [];
+  for (const originalByte of encoded) {
+    if (originalByte === 0x0A) {
+      encodedWithSubs.push(37, 48, 65); // `%0A`
+    } else if (originalByte === 0x0D) {
+      encodedWithSubs.push(37, 48, 68); // `%0D`
+    } else if (originalByte === 0x22) {
+      encodedWithSubs.push(37, 50, 50); // `%22`
+    } else {
+      encodedWithSubs.push(originalByte);
+    }
+  }
+
+  return new Uint8Array(encodedWithSubs);
+}
+
+// https://andreubotella.github.io/multipart-form-data/#multipart-form-data-chunk-serializer
+exports.serializeEntryList = entries => {
+  const boundary = generateBoundary();
+  const outputChunks = [];
+
+  for (const entry of entries) {
+    const chunkBytes = [
+      45, 45, // `--`
+      ...boundary,
+      0x0D, 0x0A
+    ];
+
+    chunkBytes.push(...contentDispositionPrefix, ...escapeName(entry.name), 0x22 /* `"` */);
+
+    let { value } = entry;
+    if (typeof value === "string") {
+      chunkBytes.push(0x0D, 0x0A, 0x0D, 0x0A);
+
+      value = value.replace(/\r(?!\n)|(?<!\r)\n/g, "\r\n");
+
+      chunkBytes.push(...utf8Encoder.encode(value));
+
+      chunkBytes.push(0x0D, 0x0A);
+
+      outputChunks.push(new Uint8Array(chunkBytes));
+    } else {
+      // value is a FileImpl object
+
+      chunkBytes.push(...filenamePrefix, ...escapeName(value.name, true), 0x22, 0x0D, 0x0A);
+
+      const type = value.type !== "" ? value.type : "application/octet-stream";
+      chunkBytes.push(...contentType, ...utf8Encoder.encode(type));
+
+      chunkBytes.push(0x0D, 0x0A, 0x0D, 0x0A);
+
+      outputChunks.push(
+        new Uint8Array(chunkBytes),
+        // The spec returns the File object here but for our purposes the bytes (as a `Buffer`) are more convenient.
+        value._buffer,
+        new Uint8Array([0x0D, 0x0A])
+      );
+    }
+  }
+
+  outputChunks.push(
+    new Uint8Array([45, 45]), // `--`
+    boundary,
+    new Uint8Array([45, 45]), // `--`
+    new Uint8Array([0x0D, 0x0A])
+  );
+
+  return { boundary, outputChunks };
+};
+
+// Inspired by https://andreubotella.github.io/multipart-form-data/#create-a-multipart-form-data-readable-stream
+// (But we don't stream in jsdom, at least for now.)
+exports.chunksToBuffer = chunks => {
+  return Buffer.concat(chunks);
+};

data/lib/readability_js/node/node_modules/jsdom/lib/jsdom/living/xhr/xhr-sync-worker.js

@@ -0,0 +1,59 @@
+"use strict";
+const util = require("util");
+const { JSDOM } = require("../../../..");
+const { READY_STATES } = require("./xhr-utils");
+const idlUtils = require("../generated/utils");
+const tough = require("tough-cookie");
+
+const dom = new JSDOM();
+const xhr = new dom.window.XMLHttpRequest();
+const xhrImpl = idlUtils.implForWrapper(xhr);
+
+const chunks = [];
+
+process.stdin.on("data", chunk => {
+  chunks.push(chunk);
+});
+
+process.stdin.on("end", () => {
+  const buffer = Buffer.concat(chunks);
+
+  const flag = JSON.parse(buffer.toString());
+  if (flag.body && flag.body.type === "Buffer" && flag.body.data) {
+    flag.body = Buffer.from(flag.body.data);
+  }
+  if (flag.cookieJar) {
+    flag.cookieJar = tough.CookieJar.fromJSON(flag.cookieJar);
+  }
+
+  flag.synchronous = false;
+  Object.assign(xhrImpl.flag, flag);
+  const { properties } = xhrImpl;
+  xhrImpl.readyState = READY_STATES.OPENED;
+  try {
+    xhr.addEventListener("loadend", () => {
+      if (properties.error) {
+        properties.error = properties.error.stack || util.inspect(properties.error);
+      }
+      process.stdout.write(JSON.stringify({
+        responseURL: xhrImpl.responseURL,
+        status: xhrImpl.status,
+        statusText: xhrImpl.statusText,
+        properties
+      }), () => {
+        process.exit(0);
+      });
+    }, false);
+    xhr.send(flag.body);
+  } catch (error) {
+    properties.error += error.stack || util.inspect(error);
+    process.stdout.write(JSON.stringify({
+      responseURL: xhrImpl.responseURL,
+      status: xhrImpl.status,
+      statusText: xhrImpl.statusText,
+      properties
+    }), () => {
+      process.exit(0);
+    });
+  }
+});

data/lib/readability_js/node/node_modules/jsdom/lib/jsdom/living/xhr/xhr-utils.js

@@ -0,0 +1,410 @@
+"use strict";
+const fs = require("fs");
+const { EventEmitter } = require("events");
+const { URL } = require("whatwg-url");
+const parseDataURL = require("data-urls");
+const DOMException = require("../generated/DOMException");
+
+const ProgressEvent = require("../generated/ProgressEvent");
+
+const agentFactory = require("../helpers/agent-factory");
+const Request = require("../helpers/http-request");
+const { fireAnEvent } = require("../helpers/events");
+
+const headerListSeparatorRegexp = /,[ \t]*/;
+const simpleMethods = new Set(["GET", "HEAD", "POST"]);
+const simpleHeaders = new Set(["accept", "accept-language", "content-language", "content-type"]);
+const preflightHeaders = new Set([
+  "access-control-expose-headers",
+  "access-control-allow-headers",
+  "access-control-allow-credentials",
+  "access-control-allow-origin"
+]);
+
+const READY_STATES = exports.READY_STATES = Object.freeze({
+  UNSENT: 0,
+  OPENED: 1,
+  HEADERS_RECEIVED: 2,
+  LOADING: 3,
+  DONE: 4
+});
+
+function getRequestHeader(requestHeaders, header) {
+  const lcHeader = header.toLowerCase();
+  const keys = Object.keys(requestHeaders);
+  let n = keys.length;
+  while (n--) {
+    const key = keys[n];
+    if (key.toLowerCase() === lcHeader) {
+      return requestHeaders[key];
+    }
+  }
+  return null;
+}
+
+function updateRequestHeader(requestHeaders, header, newValue) {
+  const lcHeader = header.toLowerCase();
+  const keys = Object.keys(requestHeaders);
+  let n = keys.length;
+  while (n--) {
+    const key = keys[n];
+    if (key.toLowerCase() === lcHeader) {
+      requestHeaders[key] = newValue;
+    }
+  }
+}
+
+function dispatchError(xhr) {
+  const errMessage = xhr.properties.error;
+  requestErrorSteps(xhr, "error", DOMException.create(xhr._globalObject, [errMessage, "NetworkError"]));
+}
+
+function validCORSHeaders(xhr, response, flag, properties, origin) {
+  const acaoStr = response.headers["access-control-allow-origin"];
+  const acao = acaoStr ? acaoStr.trim() : null;
+  if (acao !== "*" && acao !== origin) {
+    properties.error = "Cross origin " + origin + " forbidden";
+    dispatchError(xhr);
+    return false;
+  }
+  const acacStr = response.headers["access-control-allow-credentials"];
+  const acac = acacStr ? acacStr.trim() : null;
+  if (flag.withCredentials && acac !== "true") {
+    properties.error = "Credentials forbidden";
+    dispatchError(xhr);
+    return false;
+  }
+  return true;
+}
+
+function validCORSPreflightHeaders(xhr, response, flag, properties) {
+  if (!validCORSHeaders(xhr, response, flag, properties, properties.origin)) {
+    return false;
+  }
+  const acahStr = response.headers["access-control-allow-headers"];
+  const acah = new Set(acahStr ? acahStr.trim().toLowerCase().split(headerListSeparatorRegexp) : []);
+  const forbiddenHeaders = acah.has("*") ?
+  [] :
+  Object.keys(flag.requestHeaders).filter(header => {
+    const lcHeader = header.toLowerCase();
+    return !simpleHeaders.has(lcHeader) && !acah.has(lcHeader);
+  });
+  if (forbiddenHeaders.length > 0) {
+    properties.error = "Headers " + forbiddenHeaders + " forbidden";
+    dispatchError(xhr);
+    return false;
+  }
+  return true;
+}
+
+function requestErrorSteps(xhr, event, exception) {
+  const { flag, properties, upload } = xhr;
+
+  xhr.readyState = READY_STATES.DONE;
+  properties.send = false;
+
+  setResponseToNetworkError(xhr);
+
+  if (flag.synchronous) {
+    throw exception;
+  }
+
+  fireAnEvent("readystatechange", xhr);
+
+  if (!properties.uploadComplete) {
+    properties.uploadComplete = true;
+
+    if (properties.uploadListener) {
+      fireAnEvent(event, upload, ProgressEvent, { loaded: 0, total: 0, lengthComputable: false });
+      fireAnEvent("loadend", upload, ProgressEvent, { loaded: 0, total: 0, lengthComputable: false });
+    }
+  }
+
+  fireAnEvent(event, xhr, ProgressEvent, { loaded: 0, total: 0, lengthComputable: false });
+  fireAnEvent("loadend", xhr, ProgressEvent, { loaded: 0, total: 0, lengthComputable: false });
+}
+
+function setResponseToNetworkError(xhr) {
+  const { properties } = xhr;
+
+  properties.responseBuffer =
+    properties.responseCache =
+    properties.responseTextCache =
+    properties.responseXMLCache = null;
+
+  properties.responseHeaders = {};
+  xhr.status = 0;
+  xhr.statusText = "";
+}
+
+// return a "request" client object or an event emitter matching the same behaviour for unsupported protocols
+// the callback should be called with a "request" response object or an event emitter matching the same behaviour too
+function createClient(xhr) {
+  const { flag, properties } = xhr;
+  const urlObj = new URL(flag.uri);
+  const uri = urlObj.href;
+  const ucMethod = flag.method.toUpperCase();
+
+  const { requestManager } = flag;
+
+  if (urlObj.protocol === "file:") {
+    const response = new EventEmitter();
+    response.statusCode = 200;
+    response.rawHeaders = [];
+    response.headers = {};
+    const filePath = urlObj.pathname
+      .replace(/^file:\/\//, "")
+      .replace(/^\/([a-z]):\//i, "$1:/")
+      .replace(/%20/g, " ");
+
+    const client = new EventEmitter();
+
+    const readableStream = fs.createReadStream(filePath, { encoding: null });
+
+    readableStream.on("data", chunk => {
+      response.emit("data", chunk);
+      client.emit("data", chunk);
+    });
+
+    readableStream.on("end", () => {
+      response.emit("end");
+      client.emit("end");
+    });
+
+    readableStream.on("error", err => {
+      client.emit("error", err);
+    });
+
+    client.abort = function () {
+      readableStream.destroy();
+      client.emit("abort");
+    };
+
+    if (requestManager) {
+      const req = {
+        abort() {
+          properties.abortError = true;
+          xhr.abort();
+        }
+      };
+      requestManager.add(req);
+      const rmReq = requestManager.remove.bind(requestManager, req);
+      client.on("abort", rmReq);
+      client.on("error", rmReq);
+      client.on("end", rmReq);
+    }
+
+    process.nextTick(() => client.emit("response", response, urlObj.href));
+
+    return client;
+  }
+
+  if (urlObj.protocol === "data:") {
+    const response = new EventEmitter();
+
+    const client = new EventEmitter();
+
+    let buffer;
+    try {
+      const parsed = parseDataURL(uri);
+      const contentType = parsed.mimeType.toString();
+      buffer = Buffer.from(parsed.body);
+      response.statusCode = 200;
+      response.rawHeaders = ["Content-Type", contentType];
+      response.headers = { "content-type": contentType };
+    } catch (err) {
+      process.nextTick(() => client.emit("error", err));
+      return client;
+    }
+
+    client.abort = () => {
+      // do nothing
+    };
+
+    process.nextTick(() => {
+      client.emit("response", response, urlObj.href);
+      process.nextTick(() => {
+        response.emit("data", buffer);
+        client.emit("data", buffer);
+        response.emit("end");
+        client.emit("end");
+      });
+    });
+
+    return client;
+  }
+  const agents = agentFactory(flag.proxy, flag.strictSSL);
+  const requestHeaders = {};
+
+  for (const header in flag.requestHeaders) {
+    requestHeaders[header] = flag.requestHeaders[header];
+  }
+
+  if (getRequestHeader(flag.requestHeaders, "referer") === null) {
+    requestHeaders.Referer = flag.referrer;
+  }
+  if (getRequestHeader(flag.requestHeaders, "user-agent") === null) {
+    requestHeaders["User-Agent"] = flag.userAgent;
+  }
+  if (getRequestHeader(flag.requestHeaders, "accept-language") === null) {
+    requestHeaders["Accept-Language"] = "en";
+  }
+  if (getRequestHeader(flag.requestHeaders, "accept") === null) {
+    requestHeaders.Accept = "*/*";
+  }
+
+  const crossOrigin = flag.origin !== urlObj.origin;
+  if (crossOrigin) {
+    requestHeaders.Origin = flag.origin;
+  }
+
+  const options = { rejectUnauthorized: flag.strictSSL, agents, followRedirects: true };
+  if (flag.auth) {
+    options.user = flag.auth.user || "";
+    options.pass = flag.auth.pass || "";
+  }
+  if (flag.cookieJar && (!crossOrigin || flag.withCredentials)) {
+    options.cookieJar = flag.cookieJar;
+  }
+
+  const { body } = flag;
+  const hasBody = body !== undefined &&
+                  body !== null &&
+                  body !== "" &&
+                  !(ucMethod === "HEAD" || ucMethod === "GET");
+
+  if (hasBody && getRequestHeader(flag.requestHeaders, "content-type") === null) {
+    requestHeaders["Content-Type"] = "text/plain;charset=UTF-8";
+  }
+
+  function doRequest() {
+    try {
+      requestHeaders["Accept-Encoding"] = "gzip, deflate";
+
+      let len = 0;
+      if (hasBody) {
+        len = body.byteLength;
+        requestHeaders["Content-Length"] = len;
+      }
+
+      const requestClient = new Request(uri, options, { method: flag.method, headers: requestHeaders });
+      if (hasBody) {
+        requestClient.write(body);
+      }
+      return requestClient;
+    } catch (e) {
+      const eventEmitterclient = new EventEmitter();
+      process.nextTick(() => eventEmitterclient.emit("error", e));
+      eventEmitterclient.end = () => {};
+      eventEmitterclient.abort = () => {
+        // do nothing
+      };
+      return eventEmitterclient;
+    }
+  }
+
+  let client;
+
+  const nonSimpleHeaders = Object.keys(flag.requestHeaders)
+    .filter(header => !simpleHeaders.has(header.toLowerCase()));
+
+  if (crossOrigin && (!simpleMethods.has(ucMethod) || nonSimpleHeaders.length > 0 || properties.uploadListener)) {
+    client = new EventEmitter();
+
+    const preflightRequestHeaders = {};
+    for (const header in requestHeaders) {
+      // the only existing request headers the cors spec allows on the preflight request are Origin and Referer
+      const lcHeader = header.toLowerCase();
+      if (lcHeader === "origin" || lcHeader === "referer") {
+        preflightRequestHeaders[header] = requestHeaders[header];
+      }
+    }
+
+    preflightRequestHeaders["Access-Control-Request-Method"] = flag.method;
+    if (nonSimpleHeaders.length > 0) {
+      preflightRequestHeaders["Access-Control-Request-Headers"] = nonSimpleHeaders.join(", ");
+    }
+
+    preflightRequestHeaders["User-Agent"] = flag.userAgent;
+
+    flag.preflight = true;
+
+    const rejectUnauthorized = flag.strictSSL;
+    const preflightClient = new Request(
+      uri,
+      { agents, followRedirects: false },
+      { method: "OPTIONS", headers: preflightRequestHeaders, rejectUnauthorized }
+    );
+
+    preflightClient.on("response", resp => {
+      // don't send the real request if the preflight request returned an error
+      if (resp.statusCode < 200 || resp.statusCode > 299) {
+        client.emit("error", new Error("Response for preflight has invalid HTTP status code " + resp.statusCode));
+        return;
+      }
+      // don't send the real request if we aren't allowed to use the headers
+      if (!validCORSPreflightHeaders(xhr, resp, flag, properties)) {
+        setResponseToNetworkError(xhr);
+        return;
+      }
+      // Set request gzip option right before headers are set
+      const realClient = doRequest();
+      realClient.on("response", (...args) => client.emit("response", ...args));
+      realClient.on("data", chunk => client.emit("data", chunk));
+      realClient.on("end", () => client.emit("end"));
+      realClient.on("abort", () => client.emit("abort"));
+      realClient.on("request", req => {
+        client.headers = realClient.headers;
+        client.emit("request", req);
+      });
+      realClient.on("redirect", (...args) => {
+        client.emit("redirect", ...args);
+      });
+      realClient.on("error", err => {
+        client.emit("error", err);
+      });
+      client.abort = () => {
+        realClient.abort();
+      };
+      setImmediate(() => realClient.end());
+    });
+
+    preflightClient.on("error", err => {
+      client.emit("error", err);
+    });
+
+    client.abort = () => {
+      preflightClient.abort();
+    };
+    setImmediate(() => preflightClient.end());
+  } else {
+    client = doRequest();
+    setImmediate(() => client.end());
+  }
+
+  if (requestManager) {
+    const req = {
+      abort() {
+        properties.abortError = true;
+        xhr.abort();
+      }
+    };
+    requestManager.add(req);
+    const rmReq = requestManager.remove.bind(requestManager, req);
+    client.on("abort", rmReq);
+    client.on("error", rmReq);
+    client.on("end", rmReq);
+  }
+  return client;
+}
+
+exports.headerListSeparatorRegexp = headerListSeparatorRegexp;
+exports.simpleHeaders = simpleHeaders;
+exports.preflightHeaders = preflightHeaders;
+exports.getRequestHeader = getRequestHeader;
+exports.updateRequestHeader = updateRequestHeader;
+exports.dispatchError = dispatchError;
+exports.validCORSHeaders = validCORSHeaders;
+exports.requestErrorSteps = requestErrorSteps;
+exports.setResponseToNetworkError = setResponseToNetworkError;
+exports.createClient = createClient;

data/lib/readability_js/node/node_modules/jsdom/lib/jsdom/utils.js

@@ -0,0 +1,105 @@
+"use strict";
+const { domSymbolTree } = require("./living/helpers/internal-constants");
+const SYMBOL_TREE_POSITION = require("symbol-tree").TreePosition;
+
+/**
+ * Define a set of properties on an object, by copying the property descriptors
+ * from the original object.
+ *
+ * - `object` {Object} the target object
+ * - `properties` {Object} the source from which to copy property descriptors
+ */
+exports.define = function define(object, properties) {
+  for (const name of Object.getOwnPropertyNames(properties)) {
+    const propDesc = Object.getOwnPropertyDescriptor(properties, name);
+    Object.defineProperty(object, name, propDesc);
+  }
+};
+
+exports.mixin = (target, source) => {
+  const keys = Reflect.ownKeys(source);
+  for (let i = 0; i < keys.length; ++i) {
+    if (keys[i] in target) {
+      continue;
+    }
+
+    Object.defineProperty(target, keys[i], Object.getOwnPropertyDescriptor(source, keys[i]));
+  }
+};
+
+let memoizeQueryTypeCounter = 0;
+
+/**
+ * Returns a version of a method that memoizes specific types of calls on the object
+ *
+ * - `fn` {Function} the method to be memozied
+ */
+exports.memoizeQuery = function memoizeQuery(fn) {
+  // Only memoize query functions with arity <= 2
+  if (fn.length > 2) {
+    return fn;
+  }
+
+  const type = memoizeQueryTypeCounter++;
+
+  return function (...args) {
+    if (!this._memoizedQueries) {
+      return fn.apply(this, args);
+    }
+
+    if (!this._memoizedQueries[type]) {
+      this._memoizedQueries[type] = Object.create(null);
+    }
+
+    let key;
+    if (args.length === 1 && typeof args[0] === "string") {
+      key = args[0];
+    } else if (args.length === 2 && typeof args[0] === "string" && typeof args[1] === "string") {
+      key = args[0] + "::" + args[1];
+    } else {
+      return fn.apply(this, args);
+    }
+
+    if (!(key in this._memoizedQueries[type])) {
+      this._memoizedQueries[type][key] = fn.apply(this, args);
+    }
+    return this._memoizedQueries[type][key];
+  };
+};
+
+exports.simultaneousIterators = function* (first, second) {
+  for (;;) {
+    const firstResult = first.next();
+    const secondResult = second.next();
+
+    if (firstResult.done && secondResult.done) {
+      return;
+    }
+
+    yield [
+      firstResult.done ? null : firstResult.value,
+      secondResult.done ? null : secondResult.value
+    ];
+  }
+};
+
+exports.treeOrderSorter = function (a, b) {
+  const compare = domSymbolTree.compareTreePosition(a, b);
+
+  if (compare & SYMBOL_TREE_POSITION.PRECEDING) { // b is preceding a
+    return 1;
+  }
+
+  if (compare & SYMBOL_TREE_POSITION.FOLLOWING) {
+    return -1;
+  }
+
+  // disconnected or equal:
+  return 0;
+};
+
+try {
+  exports.Canvas = require("canvas");
+} catch {
+  exports.Canvas = null;
+}