@loaders.gl/polyfills 3.3.0-alpha.13 → 3.3.0-alpha.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/polyfills",
-  "version": "3.3.0-alpha.13",
+  "version": "3.3.0-alpha.14",
   "description": "Polyfills for TextEncoder/TextDecoder",
   "license": "MIT",
   "publishConfig": {
@@ -106,5 +106,5 @@
     "through": "^2.3.8",
     "web-streams-polyfill": "^3.0.0"
   },
-  "gitHead": "429a5667c9903cc2de4d1f3e55e1c2f2e58a3ff6"
+  "gitHead": "cc91201ca3c0581a5c9edf7a8bc0fc230212bf3d"
 }

package/src/index.ts

@@ -8,9 +8,9 @@ import {allSettled} from './promise/all-settled';
 // Node specific
 import * as base64 from './node/buffer/btoa.node';
 
-import HeadersNode from './node/fetch/headers.node';
-import ResponseNode from './node/fetch/response.node';
-import fetchNode from './node/fetch/fetch.node';
+import {Headers as HeadersNode} from './node/fetch/headers.node';
+import {Response as ResponseNode} from './node/fetch/response.node';
+import {fetchNode as fetchNode} from './node/fetch/fetch.node';
 
 import {encodeImageNode} from './node/images/encode-image.node';
 import {parseImageNode} from './node/images/parse-image.node';
@@ -20,7 +20,8 @@ export {BlobPolyfill} from './node/file/blob';
 export {FileReaderPolyfill} from './node/file/file-reader';
 export {FilePolyfill} from './node/file/file';
 export {installFilePolyfills} from './node/file/install-file-polyfills';
-export {default as _fetchNode} from './node/fetch/fetch.node';
+export {fetchNode as _fetchNode} from './node/fetch/fetch.node';
+export {fetchFileNode as _fetchFileNode} from './node/fetch/fetch-file.node';
 
 // POLYFILLS: TextEncoder, TextDecoder
 // - Recent Node versions have these classes but virtually no encodings unless special build.

package/src/node/fetch/fetch-file.node.ts

@@ -0,0 +1,51 @@
+// loaders.gl, MIT license
+
+import fs from 'fs'; // `fs` will be empty object in browsers (see package.json "browser" field).
+import {Response} from './response.node';
+import {Headers} from './headers.node';
+
+export function isRequestURL(url: string): boolean {
+  return url.startsWith('http:') || url.startsWith('https:');
+}
+
+export async function fetchFileNode(url: string, options): Promise<Response> {
+  const noqueryUrl = url.split('?')[0];
+
+  try {
+    // Now open the stream
+    const body = await new Promise((resolve, reject) => {
+      // @ts-ignore
+      const stream = fs.createReadStream(noqueryUrl, {encoding: null});
+      stream.once('readable', () => resolve(stream));
+      stream.on('error', (error) => reject(error));
+    });
+
+    const status = 200;
+    const statusText = 'OK';
+    const headers = getHeadersForFile(noqueryUrl);
+    return new Response(body, {headers, status, statusText, url});
+  } catch (error) {
+    const status = 400;
+    const statusText = (error as Error).message;
+    const headers = {};
+    return new Response((error as Error).message, {headers, status, statusText, url});
+  }
+}
+
+function getHeadersForFile(noqueryUrl: string): Headers {
+  const headers = {};
+
+  // Fix up content length if we can for best progress experience
+  if (!headers['content-length']) {
+    const stats = fs.statSync(noqueryUrl);
+    headers['content-length'] = stats.size;
+  }
+
+  // Automatically decompress gzipped files with .gz extension
+  if (noqueryUrl.endsWith('.gz')) {
+    noqueryUrl = noqueryUrl.slice(0, -3);
+    headers['content-encoding'] = 'gzip';
+  }
+
+  return new Headers(headers);
+}

package/src/node/fetch/fetch.node.ts

@@ -1,20 +1,29 @@
-import fs from 'fs'; // `fs` will be empty object in browsers (see package.json "browser" field).
-import Response from './response.node';
-import Headers from './headers.node';
+// loaders.gl, MIT license
 
+import http from 'http';
+import https from 'https';
+import {Response} from './response.node';
+import {Headers} from './headers.node';
 import {decodeDataUri} from './utils/decode-data-uri.node';
-import {createReadStream} from './utils/stream-utils.node';
 
-const isDataURL = (url) => url.startsWith('data:');
-const isRequestURL = (url) => url.startsWith('http:') || url.startsWith('https:');
+import {fetchFileNode} from './fetch-file.node';
+
+const isDataURL = (url: string): boolean => url.startsWith('data:');
+const isRequestURL = (url: string): boolean => url.startsWith('http:') || url.startsWith('https:');
 
 /**
  * Emulation of Browser fetch for Node.js
  * @param url
  * @param options
  */
-export default async function fetchNode(url, options) {
+// eslint-disable-next-line complexity
+export async function fetchNode(url: string, options): Promise<Response> {
   try {
+    // Handle file streams in node
+    if (!isRequestURL(url) && !isDataURL(url)) {
+      return await fetchFileNode(url, options);
+    }
+
     // Handle data urls in node, to match `fetch``
     // Note - this loses the MIME type, data URIs are handled directly in fetch
     if (isDataURL(url)) {
@@ -35,7 +44,7 @@ export default async function fetchNode(url, options) {
     }
 
     // Need to create the stream in advance since Response constructor needs to be sync
-    const body = await createReadStream(originalUrl, options);
+    const body = await createHTTPRequestReadStream(originalUrl, options);
     const headers = getHeaders(url, body, syntheticResponseHeaders);
     const {status, statusText} = getStatus(body);
 
@@ -55,6 +64,23 @@ export default async function fetchNode(url, options) {
   }
 }
 
+/** Returns a promise that resolves to a readable stream */
+export async function createHTTPRequestReadStream(
+  url: string,
+  options
+): Promise<http.IncomingMessage> {
+  // HANDLE HTTP/HTTPS REQUESTS IN NODE
+  // TODO: THIS IS BAD SINCE WE RETURN A PROMISE INSTEAD OF A STREAM
+  return await new Promise((resolve, reject) => {
+    const requestOptions = getRequestOptions(url, options);
+    const req = url.startsWith('https:')
+      ? https.request(requestOptions, (res) => resolve(res))
+      : http.request(requestOptions, (res) => resolve(res));
+    req.on('error', (error) => reject(error));
+    req.end();
+  });
+}
+
 /**
  * Generate redirect url from location without origin and protocol.
  * @param originalUrl
@@ -72,9 +98,33 @@ function generateRedirectUrl(originalUrl: string, location: string): string {
 }
 
 // HELPER FUNCTIONS
-// PRIVATE
 
-function getStatus(httpResponse) {
+function getRequestOptions(url: string, options?: {fetch?: typeof fetch; headers?}) {
+  // Ensure header keys are lower case so that we can merge without duplicates
+  const originalHeaders = options?.headers || {};
+  const headers = {};
+  for (const key of Object.keys(originalHeaders)) {
+    headers[key.toLowerCase()] = originalHeaders[key];
+  }
+
+  // Add default accept-encoding to headers
+  headers['accept-encoding'] = headers['accept-encoding'] || 'gzip,br,deflate';
+
+  const urlObject = new URL(url);
+  return {
+    hostname: urlObject.hostname,
+    path: urlObject.pathname,
+    method: 'GET',
+    // Add options and user provided 'options.fetch' overrides if available
+    ...options,
+    ...options?.fetch,
+    // Override with updated headers with accepted encodings:
+    headers,
+    port: urlObject.port
+  };
+}
+
+function getStatus(httpResponse: http.IncomingMessage): {status: number; statusText: string} {
   if (httpResponse.statusCode) {
     return {status: httpResponse.statusCode, statusText: httpResponse.statusMessage || 'NA'};
   }
@@ -105,24 +155,8 @@ function getHeaders(url, httpResponse, additionalHeaders = {}) {
   return new Headers(headers);
 }
 
-function getContentLength(url) {
-  if (isRequestURL(url)) {
-    // Needs to be read from actual headers
-    return null;
-  } else if (isDataURL(url)) {
-    // TODO - remove media type etc
-    return url.length - 'data:'.length;
-  }
-  // File URL
-  // TODO - how to handle non-existing file, this presumably just throws
-  try {
-    // strip query params from URL
-    const noqueryUrl = url.split('?')[0];
-    const stats = fs.statSync(noqueryUrl);
-    return stats.size;
-  } catch (error) {
-    // ignore for now
-  }
-
-  return null;
+/** Needs to be read from actual headers */
+function getContentLength(url: string): number | null {
+  // TODO - remove media type etc
+  return isDataURL(url) ? url.length - 'data:'.length : null;
 }

package/src/node/fetch/headers.node.ts

@@ -2,7 +2,7 @@
  * Polyfill for Browser Headers
  * Based on https://github.com/github/fetch under MIT license
  */
-export default class Headers {
+export class Headers {
   map: {};
 
   constructor(headers) {

package/src/node/fetch/response.node.ts

@@ -1,6 +1,8 @@
+// loaders.gl, MIT license
+
 import {assert} from '../../utils/assert';
 import {decompressReadStream, concatenateReadStream} from './utils/stream-utils.node';
-import Headers from './headers.node';
+import {Headers} from './headers.node';
 
 const isBoolean = (x) => typeof x === 'boolean';
 const isFunction = (x) => typeof x === 'function';
@@ -20,7 +22,7 @@ const isReadableNodeStream = (x) =>
  */
 import {Readable} from 'stream';
 
-export default class Response {
+export class Response {
   readonly ok: boolean;
   readonly status: number;
   readonly statusText: string;

package/src/node/fetch/utils/stream-utils.node.ts

@@ -1,39 +1,12 @@
-import fs from 'fs'; // `fs` will be empty object in browsers (see package.json "browser" field).
-import http from 'http';
-import https from 'https';
+// loaders.gl, MIT license
+
 import zlib from 'zlib';
 
 import {toArrayBuffer} from './decode-data-uri.node';
 
-function isRequestURL(url: string): boolean {
-  return url.startsWith('http:') || url.startsWith('https:');
-}
-
-// Returns a promise that resolves to a readable stream
-export async function createReadStream(url, options): Promise<any> {
-  // Handle file streams in node
-  if (!isRequestURL(url)) {
-    const noqueryUrl = url.split('?')[0];
-    // Now open the stream
-    return await new Promise((resolve, reject) => {
-      // @ts-ignore
-      const stream = fs.createReadStream(noqueryUrl, {encoding: null});
-      stream.once('readable', () => resolve(stream));
-      stream.on('error', (error) => reject(error));
-    });
-  }
-
-  // HANDLE HTTP/HTTPS REQUESTS IN NODE
-  // TODO: THIS IS BAD SINCE WE RETURN A PROMISE INSTEAD OF A STREAM
-  return await new Promise((resolve, reject) => {
-    const requestFunction = url.startsWith('https:') ? https.request : http.request;
-    const requestOptions = getRequestOptions(url, options);
-    const req = requestFunction(requestOptions, (res) => resolve(res));
-    req.on('error', (error) => reject(error));
-    req.end();
-  });
-}
-
+/**
+ *
+ */
 export function decompressReadStream(readStream, headers) {
   switch (headers.get('content-encoding')) {
     case 'br':
@@ -48,6 +21,11 @@ export function decompressReadStream(readStream, headers) {
   }
 }
 
+/**
+ *
+ * @param readStream
+ * @returns
+ */
 export async function concatenateReadStream(readStream): Promise<ArrayBuffer> {
   const arrayBufferChunks: ArrayBuffer[] = [];
 
@@ -72,33 +50,6 @@ export async function concatenateReadStream(readStream): Promise<ArrayBuffer> {
   });
 }
 
-// HELPERS
-
-function getRequestOptions(url: string, options?: {fetch?: typeof fetch; headers?}) {
-  // Ensure header keys are lower case so that we can merge without duplicates
-  const originalHeaders = options?.headers || {};
-  const headers = {};
-  for (const key of Object.keys(originalHeaders)) {
-    headers[key.toLowerCase()] = originalHeaders[key];
-  }
-
-  // Add default accept-encoding to headers
-  headers['accept-encoding'] = headers['accept-encoding'] || 'gzip,br,deflate';
-
-  const urlObject = new URL(url);
-  return {
-    hostname: urlObject.hostname,
-    path: urlObject.pathname,
-    method: 'GET',
-    // Add options and user provided 'options.fetch' overrides if available
-    ...options,
-    ...options?.fetch,
-    // Override with updated headers with accepted encodings:
-    headers,
-    port: urlObject.port
-  };
-}
-
 /**
  * Concatenate a sequence of ArrayBuffers
  * @return A concatenated ArrayBuffer