postdoc 0.1.2 → 0.2.0.beta4

data/node_modules/puppeteer/lib/Browser.js

@@ -0,0 +1,186 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { helper } = require('./helper');
+const Target = require('./Target');
+const EventEmitter = require('events');
+const TaskQueue = require('./TaskQueue');
+
+class Browser extends EventEmitter {
+  /**
+   * @param {!Puppeteer.Connection} connection
+   * @param {!BrowserOptions=} options
+   * @param {?Puppeteer.ChildProcess} process
+   * @param {(function():Promise)=} closeCallback
+   */
+  constructor(connection, options = {}, process, closeCallback) {
+    super();
+    this._ignoreHTTPSErrors = !!options.ignoreHTTPSErrors;
+    this._appMode = !!options.appMode;
+    this._process = process;
+    this._screenshotTaskQueue = new TaskQueue();
+    this._connection = connection;
+    this._closeCallback = closeCallback || new Function();
+    /** @type {Map<string, Target>} */
+    this._targets = new Map();
+    this._connection.setClosedCallback(() => {
+      this.emit(Browser.Events.Disconnected);
+    });
+    this._connection.on('Target.targetCreated', this._targetCreated.bind(this));
+    this._connection.on('Target.targetDestroyed', this._targetDestroyed.bind(this));
+    this._connection.on('Target.targetInfoChanged', this._targetInfoChanged.bind(this));
+  }
+
+  /**
+   * @return {?Puppeteer.ChildProcess}
+   */
+  process() {
+    return this._process;
+  }
+
+  /**
+   * @param {!Puppeteer.Connection} connection
+   * @param {!BrowserOptions=} options
+   * @param {?Puppeteer.ChildProcess} process
+   * @param {function()=} closeCallback
+   */
+  static async create(connection, options, process, closeCallback) {
+    const browser = new Browser(connection, options, process, closeCallback);
+    await connection.send('Target.setDiscoverTargets', {discover: true});
+    return browser;
+  }
+
+  /**
+   * @param {{targetInfo: !Puppeteer.TargetInfo}} event
+   */
+  async _targetCreated(event) {
+    const targetInfo = event.targetInfo;
+    const target = new Target(targetInfo, this, () => this._connection.createSession(targetInfo.targetId), this._ignoreHTTPSErrors, !this._appMode, this._screenshotTaskQueue);
+    console.assert(!this._targets.has(event.targetInfo.targetId), 'Target should not exist before targetCreated');
+    this._targets.set(event.targetInfo.targetId, target);
+
+    if (await target._initializedPromise)
+      this.emit(Browser.Events.TargetCreated, target);
+  }
+
+  /**
+   * @param {{targetId: string}} event
+   */
+  async _targetDestroyed(event) {
+    const target = this._targets.get(event.targetId);
+    target._initializedCallback(false);
+    this._targets.delete(event.targetId);
+    target._closedCallback();
+    if (await target._initializedPromise)
+      this.emit(Browser.Events.TargetDestroyed, target);
+  }
+
+  /**
+   * @param {{targetInfo: !Puppeteer.TargetInfo}} event
+   */
+  _targetInfoChanged(event) {
+    const target = this._targets.get(event.targetInfo.targetId);
+    console.assert(target, 'target should exist before targetInfoChanged');
+    const previousURL = target.url();
+    const wasInitialized = target._isInitialized;
+    target._targetInfoChanged(event.targetInfo);
+    if (wasInitialized && previousURL !== target.url())
+      this.emit(Browser.Events.TargetChanged, target);
+  }
+
+  /**
+   * @return {string}
+   */
+  wsEndpoint() {
+    return this._connection.url();
+  }
+
+  /**
+   * @return {!Promise<!Puppeteer.Page>}
+   */
+  async newPage() {
+    const {targetId} = await this._connection.send('Target.createTarget', {url: 'about:blank'});
+    const target = await this._targets.get(targetId);
+    console.assert(await target._initializedPromise, 'Failed to create target for page');
+    const page = await target.page();
+    return page;
+  }
+
+  /**
+   * @return {!Array<!Target>}
+   */
+  targets() {
+    return Array.from(this._targets.values()).filter(target => target._isInitialized);
+  }
+
+  /**
+   * @return {!Promise<!Array<!Puppeteer.Page>>}
+   */
+  async pages() {
+    const pages = await Promise.all(this.targets().map(target => target.page()));
+    return pages.filter(page => !!page);
+  }
+
+  /**
+   * @return {!Promise<string>}
+   */
+  async version() {
+    const version = await this._getVersion();
+    return version.product;
+  }
+
+  /**
+   * @return {!Promise<string>}
+   */
+  async userAgent() {
+    const version = await this._getVersion();
+    return version.userAgent;
+  }
+
+  async close() {
+    await this._closeCallback.call(null);
+    this.disconnect();
+  }
+
+  disconnect() {
+    this._connection.dispose();
+  }
+
+  /**
+   * @return {!Promise<!Object>}
+   */
+  _getVersion() {
+    return this._connection.send('Browser.getVersion');
+  }
+}
+
+/** @enum {string} */
+Browser.Events = {
+  TargetCreated: 'targetcreated',
+  TargetDestroyed: 'targetdestroyed',
+  TargetChanged: 'targetchanged',
+  Disconnected: 'disconnected'
+};
+
+helper.tracePublicAPI(Browser);
+
+module.exports = Browser;
+
+/**
+ * @typedef {Object} BrowserOptions
+ * @property {boolean=} appMode
+ * @property {boolean=} ignoreHTTPSErrors
+ */

data/node_modules/puppeteer/lib/BrowserFetcher.js

@@ -0,0 +1,279 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const os = require('os');
+const fs = require('fs');
+const path = require('path');
+const extract = require('extract-zip');
+const util = require('util');
+const URL = require('url');
+const {helper} = require('./helper');
+const removeRecursive = require('rimraf');
+// @ts-ignore
+const ProxyAgent = require('https-proxy-agent');
+// @ts-ignore
+const getProxyForUrl = require('proxy-from-env').getProxyForUrl;
+
+const DEFAULT_DOWNLOAD_HOST = 'https://storage.googleapis.com';
+const downloadURLs = {
+  linux: '%s/chromium-browser-snapshots/Linux_x64/%d/chrome-linux.zip',
+  mac: '%s/chromium-browser-snapshots/Mac/%d/chrome-mac.zip',
+  win32: '%s/chromium-browser-snapshots/Win/%d/chrome-win32.zip',
+  win64: '%s/chromium-browser-snapshots/Win_x64/%d/chrome-win32.zip',
+};
+
+const readdirAsync = helper.promisify(fs.readdir.bind(fs));
+const mkdirAsync = helper.promisify(fs.mkdir.bind(fs));
+const unlinkAsync = helper.promisify(fs.unlink.bind(fs));
+const chmodAsync = helper.promisify(fs.chmod.bind(fs));
+
+function existsAsync(filePath) {
+  let fulfill = null;
+  const promise = new Promise(x => fulfill = x);
+  fs.access(filePath, err => fulfill(!err));
+  return promise;
+}
+
+class BrowserFetcher {
+  /**
+   * @param {!BrowserFetcher.Options=} options
+   */
+  constructor(options = {}) {
+    this._downloadsFolder = options.path || path.join(helper.projectRoot(), '.local-chromium');
+    this._downloadHost = options.host || DEFAULT_DOWNLOAD_HOST;
+    this._platform = options.platform || '';
+    if (!this._platform) {
+      const platform = os.platform();
+      if (platform === 'darwin')
+        this._platform = 'mac';
+      else if (platform === 'linux')
+        this._platform = 'linux';
+      else if (platform === 'win32')
+        this._platform = os.arch() === 'x64' ? 'win64' : 'win32';
+      console.assert(this._platform, 'Unsupported platform: ' + os.platform());
+    }
+    const supportedPlatforms = ['mac', 'linux', 'win32', 'win64'];
+    console.assert(supportedPlatforms.includes(this._platform), 'Unsupported platform: ' + this._platform);
+  }
+
+  /**
+   * @return {string}
+   */
+  platform() {
+    return this._platform;
+  }
+
+  /**
+   * @param {string} revision
+   * @return {!Promise<boolean>}
+   */
+  canDownload(revision) {
+    const url = util.format(downloadURLs[this._platform], this._downloadHost, revision);
+
+    let resolve;
+    const promise = new Promise(x => resolve = x);
+    const request = httpRequest(url, 'HEAD', response => {
+      resolve(response.statusCode === 200);
+    });
+    request.on('error', error => {
+      console.error(error);
+      resolve(false);
+    });
+    return promise;
+  }
+
+  /**
+   * @param {string} revision
+   * @param {?function(number, number)} progressCallback
+   * @return {!Promise<!BrowserFetcher.RevisionInfo>}
+   */
+  async download(revision, progressCallback) {
+    let url = downloadURLs[this._platform];
+    url = util.format(url, this._downloadHost, revision);
+    const zipPath = path.join(this._downloadsFolder, `download-${this._platform}-${revision}.zip`);
+    const folderPath = this._getFolderPath(revision);
+    if (await existsAsync(folderPath))
+      return this.revisionInfo(revision);
+    if (!(await existsAsync(this._downloadsFolder)))
+      await mkdirAsync(this._downloadsFolder);
+    try {
+      await downloadFile(url, zipPath, progressCallback);
+      await extractZip(zipPath, folderPath);
+    } finally {
+      if (await existsAsync(zipPath))
+        await unlinkAsync(zipPath);
+    }
+    const revisionInfo = this.revisionInfo(revision);
+    if (revisionInfo)
+      await chmodAsync(revisionInfo.executablePath, 0o755);
+    return revisionInfo;
+  }
+
+  /**
+   * @return {!Promise<!Array<string>>}
+   */
+  async localRevisions() {
+    if (!await existsAsync(this._downloadsFolder))
+      return [];
+    const fileNames = await readdirAsync(this._downloadsFolder);
+    return fileNames.map(fileName => parseFolderPath(fileName)).filter(entry => entry && entry.platform === this._platform).map(entry => entry.revision);
+  }
+
+  /**
+   * @param {string} revision
+   * @return {!Promise}
+   */
+  async remove(revision) {
+    const folderPath = this._getFolderPath(revision);
+    console.assert(await existsAsync(folderPath), `Failed to remove: revision ${revision} is not downloaded`);
+    await new Promise(fulfill => removeRecursive(folderPath, fulfill));
+  }
+
+  /**
+   * @param {string} revision
+   * @return {!BrowserFetcher.RevisionInfo}
+   */
+  revisionInfo(revision) {
+    const folderPath = this._getFolderPath(revision);
+    let executablePath = '';
+    if (this._platform === 'mac')
+      executablePath = path.join(folderPath, 'chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium');
+    else if (this._platform === 'linux')
+      executablePath = path.join(folderPath, 'chrome-linux', 'chrome');
+    else if (this._platform === 'win32' || this._platform === 'win64')
+      executablePath = path.join(folderPath, 'chrome-win32', 'chrome.exe');
+    else
+      throw 'Unsupported platform: ' + this._platform;
+    let url = downloadURLs[this._platform];
+    url = util.format(url, this._downloadHost, revision);
+    const local = fs.existsSync(folderPath);
+    return {revision, executablePath, folderPath, local, url};
+  }
+
+  /**
+   * @param {string} revision
+   * @return {string}
+   */
+  _getFolderPath(revision) {
+    return path.join(this._downloadsFolder, this._platform + '-' + revision);
+  }
+}
+
+module.exports = BrowserFetcher;
+
+/**
+ * @param {string} folderPath
+ * @return {?{platform: string, revision: string}}
+ */
+function parseFolderPath(folderPath) {
+  const name = path.basename(folderPath);
+  const splits = name.split('-');
+  if (splits.length !== 2)
+    return null;
+  const [platform, revision] = splits;
+  if (!downloadURLs[platform])
+    return null;
+  return {platform, revision};
+}
+
+/**
+ * @param {string} url
+ * @param {string} destinationPath
+ * @param {?function(number, number)} progressCallback
+ * @return {!Promise}
+ */
+function downloadFile(url, destinationPath, progressCallback) {
+  let fulfill, reject;
+  let downloadedBytes = 0;
+  let totalBytes = 0;
+
+  const promise = new Promise((x, y) => { fulfill = x; reject = y; });
+
+  const request = httpRequest(url, 'GET', response => {
+    if (response.statusCode !== 200) {
+      const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
+      // consume response data to free up memory
+      response.resume();
+      reject(error);
+      return;
+    }
+    const file = fs.createWriteStream(destinationPath);
+    file.on('finish', () => fulfill());
+    file.on('error', error => reject(error));
+    response.pipe(file);
+    totalBytes = parseInt(/** @type {string} */ (response.headers['content-length']), 10);
+    if (progressCallback)
+      response.on('data', onData);
+  });
+  request.on('error', error => reject(error));
+  return promise;
+
+  function onData(chunk) {
+    downloadedBytes += chunk.length;
+    progressCallback(downloadedBytes, totalBytes);
+  }
+}
+
+/**
+ * @param {string} zipPath
+ * @param {string} folderPath
+ * @return {!Promise<?Error>}
+ */
+function extractZip(zipPath, folderPath) {
+  return new Promise(fulfill => extract(zipPath, {dir: folderPath}, fulfill));
+}
+
+function httpRequest(url, method, response) {
+  /** @type {Object} */
+  const options = URL.parse(url);
+  options.method = method;
+
+  const proxyURL = getProxyForUrl(url);
+  if (proxyURL) {
+    /** @type {Object} */
+    const parsedProxyURL = URL.parse(proxyURL);
+    parsedProxyURL.secureProxy = parsedProxyURL.protocol === 'https:';
+
+    options.agent = new ProxyAgent(parsedProxyURL);
+    options.rejectUnauthorized = false;
+  }
+
+  const driver = options.protocol === 'https:' ? 'https' : 'http';
+  const request = require(driver).request(options, res => {
+    if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location)
+      httpRequest(res.headers.location, method, response);
+    else
+      response(res);
+  });
+  request.end();
+  return request;
+}
+
+/**
+ * @typedef {Object} BrowserFetcher.Options
+ * @property {string=} platform
+ * @property {string=} path
+ * @property {string=} host
+ */
+
+/**
+ * @typedef {Object} BrowserFetcher.RevisionInfo
+ * @property {string} folderPath
+ * @property {string} executablePath
+ * @property {string} url
+ * @property {boolean} local
+ * @property {string} revision
+ */

data/node_modules/puppeteer/lib/Connection.js

@@ -0,0 +1,246 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const {helper} = require('./helper');
+const debugProtocol = require('debug')('puppeteer:protocol');
+const debugSession = require('debug')('puppeteer:session');
+
+const EventEmitter = require('events');
+const WebSocket = require('ws');
+const Pipe = require('./Pipe');
+
+class Connection extends EventEmitter {
+  /**
+   * @param {string} url
+   * @param {number=} delay
+   * @return {!Promise<!Connection>}
+   */
+  static async createForWebSocket(url, delay = 0) {
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(url, { perMessageDeflate: false });
+      ws.on('open', () => resolve(new Connection(url, ws, delay)));
+      ws.on('error', reject);
+    });
+  }
+
+  /**
+   * @param {!NodeJS.WritableStream} pipeWrite
+   * @param {!NodeJS.ReadableStream} pipeRead
+   * @param {number=} delay
+   * @return {!Connection}
+   */
+  static createForPipe(pipeWrite, pipeRead, delay = 0) {
+    return new Connection('', new Pipe(pipeWrite, pipeRead), delay);
+  }
+
+  /**
+   * @param {string} url
+   * @param {!Puppeteer.ConnectionTransport} transport
+   * @param {number=} delay
+   */
+  constructor(url, transport, delay = 0) {
+    super();
+    this._url = url;
+    this._lastId = 0;
+    /** @type {!Map<number, {resolve: function, reject: function, error: !Error, method: string}>}*/
+    this._callbacks = new Map();
+    this._delay = delay;
+
+    this._transport = transport;
+    this._transport.on('message', this._onMessage.bind(this));
+    this._transport.on('close', this._onClose.bind(this));
+    /** @type {!Map<string, !CDPSession>}*/
+    this._sessions = new Map();
+  }
+
+  /**
+   * @return {string}
+   */
+  url() {
+    return this._url;
+  }
+
+  /**
+   * @param {string} method
+   * @param {!Object=} params
+   * @return {!Promise<?Object>}
+   */
+  send(method, params = {}) {
+    const id = ++this._lastId;
+    const message = JSON.stringify({id, method, params});
+    debugProtocol('SEND ► ' + message);
+    this._transport.send(message);
+    return new Promise((resolve, reject) => {
+      this._callbacks.set(id, {resolve, reject, error: new Error(), method});
+    });
+  }
+
+  /**
+   * @param {function()} callback
+   */
+  setClosedCallback(callback) {
+    this._closeCallback = callback;
+  }
+
+  /**
+   * @param {string} message
+   */
+  async _onMessage(message) {
+    if (this._delay)
+      await new Promise(f => setTimeout(f, this._delay));
+    debugProtocol('◀ RECV ' + message);
+    const object = JSON.parse(message);
+    if (object.id) {
+      const callback = this._callbacks.get(object.id);
+      // Callbacks could be all rejected if someone has called `.dispose()`.
+      if (callback) {
+        this._callbacks.delete(object.id);
+        if (object.error)
+          callback.reject(rewriteError(callback.error, `Protocol error (${callback.method}): ${object.error.message} ${object.error.data}`));
+        else
+          callback.resolve(object.result);
+      }
+    } else {
+      if (object.method === 'Target.receivedMessageFromTarget') {
+        const session = this._sessions.get(object.params.sessionId);
+        if (session)
+          session._onMessage(object.params.message);
+      } else if (object.method === 'Target.detachedFromTarget') {
+        const session = this._sessions.get(object.params.sessionId);
+        if (session)
+          session._onClosed();
+        this._sessions.delete(object.params.sessionId);
+      } else {
+        this.emit(object.method, object.params);
+      }
+    }
+  }
+
+  _onClose() {
+    if (this._closeCallback) {
+      this._closeCallback();
+      this._closeCallback = null;
+    }
+    this._transport.removeAllListeners();
+    // If transport throws any error at this point of time, we don't care and should swallow it.
+    this._transport.on('error', () => {});
+    for (const callback of this._callbacks.values())
+      callback.reject(rewriteError(callback.error, `Protocol error (${callback.method}): Target closed.`));
+    this._callbacks.clear();
+    for (const session of this._sessions.values())
+      session._onClosed();
+    this._sessions.clear();
+  }
+
+  dispose() {
+    this._onClose();
+    this._transport.close();
+  }
+
+  /**
+   * @param {string} targetId
+   * @return {!Promise<!CDPSession>}
+   */
+  async createSession(targetId) {
+    const {sessionId} = await this.send('Target.attachToTarget', {targetId});
+    const session = new CDPSession(this, targetId, sessionId);
+    this._sessions.set(sessionId, session);
+    return session;
+  }
+}
+
+class CDPSession extends EventEmitter {
+  /**
+   * @param {!Connection} connection
+   * @param {string} targetId
+   * @param {string} sessionId
+   */
+  constructor(connection, targetId, sessionId) {
+    super();
+    this._lastId = 0;
+    /** @type {!Map<number, {resolve: function, reject: function, error: !Error, method: string}>}*/
+    this._callbacks = new Map();
+    this._connection = connection;
+    this._targetId = targetId;
+    this._sessionId = sessionId;
+  }
+
+  /**
+   * @param {string} method
+   * @param {!Object=} params
+   * @return {!Promise<?Object>}
+   */
+  send(method, params = {}) {
+    if (!this._connection)
+      return Promise.reject(new Error(`Protocol error (${method}): Session closed. Most likely the page has been closed.`));
+    const id = ++this._lastId;
+    const message = JSON.stringify({id, method, params});
+    debugSession('SEND ► ' + message);
+    this._connection.send('Target.sendMessageToTarget', {sessionId: this._sessionId, message}).catch(e => {
+      // The response from target might have been already dispatched.
+      if (!this._callbacks.has(id))
+        return;
+      const callback = this._callbacks.get(id);
+      this._callbacks.delete(id);
+      callback.reject(rewriteError(callback.error, e && e.message));
+    });
+    return new Promise((resolve, reject) => {
+      this._callbacks.set(id, {resolve, reject, error: new Error(), method});
+    });
+  }
+
+  /**
+   * @param {string} message
+   */
+  _onMessage(message) {
+    debugSession('◀ RECV ' + message);
+    const object = JSON.parse(message);
+    if (object.id && this._callbacks.has(object.id)) {
+      const callback = this._callbacks.get(object.id);
+      this._callbacks.delete(object.id);
+      if (object.error)
+        callback.reject(rewriteError(callback.error, `Protocol error (${callback.method}): ${object.error.message} ${object.error.data}`));
+      else
+        callback.resolve(object.result);
+    } else {
+      console.assert(!object.id);
+      this.emit(object.method, object.params);
+    }
+  }
+
+  async detach() {
+    await this._connection.send('Target.detachFromTarget',  {sessionId: this._sessionId});
+  }
+
+  _onClosed() {
+    for (const callback of this._callbacks.values())
+      callback.reject(rewriteError(callback.error, `Protocol error (${callback.method}): Target closed.`));
+    this._callbacks.clear();
+    this._connection = null;
+  }
+}
+helper.tracePublicAPI(CDPSession);
+
+/**
+ * @param {!Error} error
+ * @param {string} message
+ * @return {!Error}
+ */
+function rewriteError(error, message) {
+  error.message = message;
+  return error;
+}
+
+module.exports = {Connection, CDPSession};