@gjsify/fetch 0.0.2

package/src/headers.ts

@@ -0,0 +1,279 @@
+/**
+ * Headers.js
+ *
+ * Headers class offers convenient helpers
+ */
+
+import Soup from '@girs/soup-3.0';
+import { URLSearchParams } from '@gjsify/deno-runtime/ext/url/00_url';
+
+import { types } from 'util';
+import * as http from 'http';
+import type { IncomingMessage } from 'http';
+
+
+/* c8 ignore next 9 */
+
+const validateHeaderName = http.validateHeaderName;
+const validateHeaderValue = http.validateHeaderValue;
+
+/**
+ * This Fetch API interface allows you to perform various actions on HTTP request and response headers.
+ * These actions include retrieving, setting, adding to, and removing.
+ * A Headers object has an associated header list, which is initially empty and consists of zero or more name and value pairs.
+ * You can add to this using methods like append() (see Examples.)
+ * In all methods of this interface, header names are matched by case-insensitive byte sequence.
+ *
+ */
+export default class Headers extends URLSearchParams implements globalThis.Headers, Iterable<[string, string]> {
+    /**
+     * Headers class
+     *
+     * @constructor
+     * @param init Response headers
+     */
+    constructor(init?: HeadersInit) {
+        // Validate and normalize init object in [name, value(s)][]
+        let result: string[][] = [];
+        if (init instanceof Headers) {
+            const raw = init.raw();
+            for (const [name, values] of Object.entries(raw)) {
+                result.push(...values.map(value => [name, value]));
+            }
+        } else if (init == null) { // eslint-disable-line no-eq-null, eqeqeq
+            // No op
+        } else if (typeof init === 'object' && !types.isBoxedPrimitive(init)) {
+            const method = init[Symbol.iterator];
+            // eslint-disable-next-line no-eq-null, eqeqeq
+            if (method == null) {
+                // Record<ByteString, ByteString>
+                result.push(...Object.entries(init));
+            } else {
+                if (typeof method !== 'function') {
+                    throw new TypeError('Header pairs must be iterable');
+                }
+
+                // Sequence<sequence<ByteString>>
+                // Note: per spec we have to first exhaust the lists then process them
+                result = [...(init as  string[][])] // TODO check if this works with Objects
+                    .map(pair => {
+                        if (
+                            typeof pair !== 'object' || types.isBoxedPrimitive(pair)
+                        ) {
+                            throw new TypeError('Each header pair must be an iterable object');
+                        }
+
+                        return [...pair];
+                    }).map(pair => {
+                        if (pair.length !== 2) {
+                            throw new TypeError('Each header pair must be a name/value tuple');
+                        }
+
+                        return [...pair];
+                    });
+            }
+        } else {
+            throw new TypeError('Failed to construct \'Headers\': The provided value is not of type \'(sequence<sequence<ByteString>> or record<ByteString, ByteString>)');
+        }
+
+        // Validate and lowercase
+        result =
+            result.length > 0 ?
+                result.map(([name, value]) => {
+                    validateHeaderName(name);
+                    validateHeaderValue(name, String(value));
+                    return [String(name).toLowerCase(), String(value)];
+                }) :
+                undefined;
+
+        super(result);
+
+        // Returning a Proxy that will lowercase key names, validate parameters and sort keys
+        // eslint-disable-next-line no-constructor-return
+        return new Proxy(this, {
+            get(target, p, receiver) {
+                switch (p) {
+                    case 'append':
+                    case 'set':
+                        return (name: string, value: any) => {
+                            validateHeaderName(name);
+                            validateHeaderValue(name, String(value));
+                            return URLSearchParams.prototype[p].call(
+                                target,
+                                String(name).toLowerCase(),
+                                String(value)
+                            );
+                        };
+
+                    case 'delete':
+                    case 'has':
+                    case 'getAll':
+                        return (name: string) => {
+                            validateHeaderName(name);
+                            return URLSearchParams.prototype[p].call(
+                                target,
+                                String(name).toLowerCase()
+                            );
+                        };
+
+                    case 'keys':
+                        return () => {
+                            target.sort();
+                            return new Set(URLSearchParams.prototype.keys.call(target)).keys();
+                        };
+
+                    default:
+                        return Reflect.get(target, p, receiver);
+                }
+            }
+        });
+        /* c8 ignore next */
+    }
+
+    get [Symbol.toStringTag]() {
+        return this.constructor.name;
+    }
+
+    toString() {
+        return Object.prototype.toString.call(this);
+    }
+
+    _appendToSoupMessage(message?: Soup.Message, type = Soup.MessageHeadersType.REQUEST) {
+        const soupHeaders = message ? message.get_request_headers() : new Soup.MessageHeaders(type);
+        for (const header in this.entries()) {
+            soupHeaders.append(header, this.get(header));
+        }
+        return soupHeaders;
+    }
+
+
+    static _newFromSoupMessage(message: Soup.Message, type: Soup.MessageHeadersType = Soup.MessageHeadersType.RESPONSE) {
+        let soupHeaders: Soup.MessageHeaders;
+        const headers = new Headers();
+        
+        if (type === Soup.MessageHeadersType.RESPONSE) {
+            soupHeaders = message.get_response_headers();
+        } else if(type === Soup.MessageHeadersType.REQUEST)  {
+            soupHeaders = message.get_request_headers();
+        } else {
+            for (const header in message.get_request_headers()) {
+                headers.append(header, soupHeaders[header]);
+            }
+            soupHeaders = message.get_response_headers();
+        }
+
+        for (const header in soupHeaders) {
+            headers.append(header, soupHeaders[header]);
+        }
+        return headers;
+    }
+
+    get(name: string) {
+        const values = this.getAll(name);
+        if (values.length === 0) {
+            return null;
+        }
+
+        let value = values.join(', ');
+        if (/^content-encoding$/i.test(name)) {
+            value = value.toLowerCase();
+        }
+
+        return value;
+    }
+
+    forEach(callback, thisArg = undefined) {
+        for (const name of this.keys()) {
+            Reflect.apply(callback, thisArg, [this.get(name), name, this]);
+        }
+    }
+
+    * values() {
+        for (const name of this.keys()) {
+            yield this.get(name);
+        }
+    }
+
+    /**
+     * 
+     */
+    * entries(): IterableIterator<[string, string]> {
+        for (const name of this.keys()) {
+            yield [name, this.get(name)];
+        }
+    }
+
+	[Symbol.iterator]() {
+		return this.entries();
+	}
+
+    /**
+     * Node-fetch non-spec method
+     * returning all headers and their values as array
+     */
+    raw(): Record<string, string[]> {
+        return [...this.keys()].reduce((result, key) => {
+            result[key] = this.getAll(key);
+            return result;
+        }, {});
+    }
+
+    /**
+     * For better console.log(headers) and also to convert Headers into Node.js Request compatible format
+     */
+    [Symbol.for('nodejs.util.inspect.custom')]() {
+        return [...this.keys()].reduce((result, key) => {
+            const values = this.getAll(key);
+            // Http.request() only supports string as Host header.
+            // This hack makes specifying custom Host header possible.
+            if (key === 'host') {
+                result[key] = values[0];
+            } else {
+                result[key] = values.length > 1 ? values : values[0];
+            }
+
+            return result;
+        }, {});
+    }
+}
+
+/**
+ * Re-shaping object for Web IDL tests
+ * Only need to do it for overridden methods
+ */
+Object.defineProperties(
+    Headers.prototype,
+    ['get', 'entries', 'forEach', 'values'].reduce((result, property) => {
+        result[property] = { enumerable: true };
+        return result;
+    }, {})
+);
+
+/**
+ * Create a Headers object from an http.IncomingMessage.rawHeaders, ignoring those that do
+ * not conform to HTTP grammar productions.
+ * @param headers
+ */
+export function fromRawHeaders(headers: IncomingMessage['rawHeaders'] = []) {
+    return new Headers(
+        headers
+            // Split into pairs
+            .reduce((result, value, index, array) => {
+                if (index % 2 === 0) {
+                    result.push(array.slice(index, index + 2));
+                }
+
+                return result;
+            }, [])
+            .filter(([name, value]) => {
+                try {
+                    validateHeaderName(name);
+                    validateHeaderValue(name, String(value));
+                    return true;
+                } catch {
+                    return false;
+                }
+            })
+
+    );
+}

package/src/index.spec.ts

@@ -0,0 +1,13 @@
+import { describe, it, expect } from '@gjsify/unit';
+
+import fetch from 'node-fetch';
+
+export default async () => {
+
+	await describe('fetch', async () => {
+		await it('fetch should be a function', async () => {
+			expect(typeof fetch).toBe("function");
+		});
+	});
+
+}

package/src/index.ts

@@ -0,0 +1,367 @@
+import Soup from '@girs/soup-3.0';
+import Gio from '@girs/gio-2.0';
+import Request from './request.js';
+
+/**
+ * Index.js
+ *
+ * a request API compatible with window.fetch
+ *
+ * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/.
+ */
+
+import zlib from 'zlib';
+import Stream, { PassThrough, pipeline as pump } from 'stream';
+
+import dataUriToBuffer from 'data-uri-to-buffer';
+
+import { writeToStream, clone } from './body.js';
+import Response from './response.js';
+import Headers from './headers.js';
+import { getSoupRequestOptions } from './request.js';
+import { FetchError } from './errors/fetch-error.js';
+import { AbortError } from './errors/abort-error.js';
+import { isRedirect } from './utils/is-redirect.js';
+import { FormData } from 'formdata-polyfill/esm.min.js';
+import { isDomainOrSubdomain, isSameProtocol } from './utils/is.js';
+import { parseReferrerPolicyFromHeader } from './utils/referrer.js';
+import {
+  Blob,
+  File,
+  fileFromSync,
+  fileFrom,
+  blobFromSync,
+  blobFrom
+} from './utils/blob-from.js';
+
+import { URL } from '@gjsify/deno-runtime/ext/url/00_url';
+
+export { FormData, Headers, Request, Response, FetchError, AbortError, isRedirect };
+export { Blob, File, fileFromSync, fileFrom, blobFromSync, blobFrom };
+
+import type { SystemError } from './types/index.js';
+
+const supportedSchemas = new Set(['data:', 'http:', 'https:']);
+
+/**
+ * Fetch function
+ *
+ * @param url Absolute url or Request instance
+ * @param init Fetch options
+ */
+export default async function fetch(url: RequestInfo | URL, init: RequestInit = {}): Promise<Response> {
+  return new Promise(async (resolve, reject) => {
+    // Build request object
+    const request = new Request(url, init);
+    const { parsedURL, options } = getSoupRequestOptions(request);
+    if (!supportedSchemas.has(parsedURL.protocol)) {
+      throw new TypeError(`@gjsify/fetch cannot load ${url}. URL scheme "${parsedURL.protocol.replace(/:$/, '')}" is not supported.`);
+    }
+
+    if (parsedURL.protocol === 'data:') {
+      const data = dataUriToBuffer(request.url);
+      const response = new Response(data, { headers: { 'Content-Type': data.typeFull } });
+      resolve(response);
+      return;
+    }
+
+    const { signal } = request;
+    let response = null;
+
+    const abort = () => {
+      const error = new AbortError('The operation was aborted.');
+      reject(error);
+      if (request.body && request.body instanceof Stream.Readable) {
+        request.body.destroy(error);
+      }
+
+      if (!response || !response.body) {
+        return;
+      }
+
+      response.body.emit('error', error);
+    };
+
+    if (signal && signal.aborted) {
+      abort();
+      return;
+    }
+
+    const abortAndFinalize = () => {
+      abort();
+      finalize();
+    };
+
+    let readable: Stream.Readable;
+    let cancellable: Gio.Cancellable;
+
+    // Send request
+    try {
+      const sendRes = await request._send(options);
+      readable = sendRes.readable;
+      cancellable = sendRes.cancellable;
+    } catch (error) {
+      reject(error);
+    }
+
+    if (signal) {
+      signal.addEventListener('abort', abortAndFinalize);
+    }
+
+    const cancelledSignalId = cancellable.connect('cancelled', () => {
+      abortAndFinalize();
+    });
+
+    const finalize = () => {
+      cancellable.cancel()
+      if (signal) {
+        signal.removeEventListener('abort', abortAndFinalize);
+      }
+      cancellable.disconnect(cancelledSignalId);
+    };
+
+    const message = request._message;
+
+    readable.on('error', (error: SystemError) => {
+      reject(new FetchError(`request to ${request.url} failed, reason: ${error.message}`, 'system', error));
+      finalize();
+    });
+
+    message.connect('finished', (message) => {
+
+      const headers = Headers._newFromSoupMessage( request._message, Soup.MessageHeadersType.RESPONSE);
+      const statusCode = message.status_code;
+      const statusMessage = message.get_reason_phrase() ;
+
+      // HTTP fetch step 5
+      if (isRedirect(statusCode)) {
+        // HTTP fetch step 5.2
+        const location = headers.get('Location');
+
+        // HTTP fetch step 5.3
+        let locationURL = null;
+        try {
+          locationURL = location === null ? null : new URL(location, request.url);
+        } catch {
+          // error here can only be invalid URL in Location: header
+          // do not throw when options.redirect == manual
+          // let the user extract the errorneous redirect URL
+          if (request.redirect !== 'manual') {
+            reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect'));
+            finalize();
+            return;
+          }
+        }
+
+        // HTTP fetch step 5.5
+        switch (request.redirect) {
+          case 'error':
+            reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${request.url}`, 'no-redirect'));
+            finalize();
+            return;
+          case 'manual':
+            // Nothing to do
+            break;
+          case 'follow': {
+            // HTTP-redirect fetch step 2
+            if (locationURL === null) {
+              break;
+            }
+
+            // HTTP-redirect fetch step 5
+            if (request.counter >= request.follow) {
+              reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect'));
+              finalize();
+              return;
+            }
+
+            // HTTP-redirect fetch step 6 (counter increment)
+            // Create a new Request object.
+            const requestOptions = {
+              headers: new Headers(request.headers),
+              follow: request.follow,
+              counter: request.counter + 1,
+              agent: request.agent,
+              compress: request.compress,
+              method: request.method,
+              body: clone(request),
+              signal: request.signal,
+              size: request.size,
+              referrer: request.referrer,
+              referrerPolicy: request.referrerPolicy
+            };
+
+            // when forwarding sensitive headers like "Authorization",
+            // "WWW-Authenticate", and "Cookie" to untrusted targets,
+            // headers will be ignored when following a redirect to a domain
+            // that is not a subdomain match or exact match of the initial domain.
+            // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
+            // will forward the sensitive headers, but a redirect to "bar.com" will not.
+            // headers will also be ignored when following a redirect to a domain using
+            // a different protocol. For example, a redirect from "https://foo.com" to "http://foo.com"
+            // will not forward the sensitive headers
+            if (!isDomainOrSubdomain(request.url, locationURL) || !isSameProtocol(request.url, locationURL)) {
+              for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) {
+                requestOptions.headers.delete(name);
+              }
+            }
+
+            // HTTP-redirect fetch step 9
+            if (statusCode !== 303 && request.body && init.body instanceof Stream.Readable) {
+              reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect'));
+              finalize();
+              return;
+            }
+
+            // HTTP-redirect fetch step 11
+            if (statusCode === 303 || ((statusCode === 301 || statusCode === 302) && request.method === 'POST')) {
+              requestOptions.method = 'GET';
+              requestOptions.body = undefined;
+              requestOptions.headers.delete('content-length');
+            }
+
+            // HTTP-redirect fetch step 14
+            const responseReferrerPolicy = parseReferrerPolicyFromHeader(headers);
+            if (responseReferrerPolicy) {
+              requestOptions.referrerPolicy = responseReferrerPolicy;
+            }
+
+            // HTTP-redirect fetch step 15
+            resolve(fetch(new Request(locationURL, requestOptions)));
+            finalize();
+            return;
+          }
+
+          default:
+            return reject(new TypeError(`Redirect option '${request.redirect}' is not a valid value of RequestRedirect`));
+        }
+      }
+
+      // Prepare response
+      // if (signal) {
+      //   response_.once('end', () => {
+      //     signal.removeEventListener('abort', abortAndFinalize);
+      //   });
+      // }
+
+      let body = pump(response_, new PassThrough(), error => {
+        if (error) {
+          reject(error);
+        }
+      });
+      // see https://github.com/nodejs/node/pull/29376
+      /* c8 ignore next 3 */
+      // if (process.version < 'v12.10') {
+      //   response_.on('aborted', abortAndFinalize);
+      // }
+
+      const responseOptions = {
+        url: request.url,
+        status: statusCode,
+        statusText: statusMessage,
+        headers,
+        size: request.size,
+        counter: request.counter,
+        highWaterMark: request.highWaterMark
+      };
+
+      // HTTP-network fetch step 12.1.1.3
+      const codings = headers.get('Content-Encoding');
+
+      // HTTP-network fetch step 12.1.1.4: handle content codings
+
+      // in following scenarios we ignore compression support
+      // 1. compression support is disabled
+      // 2. HEAD request
+      // 3. no Content-Encoding header
+      // 4. no content response (204)
+      // 5. content not modified response (304)
+      if (!request.compress || request.method === 'HEAD' || codings === null || statusCode === 204 || statusCode === 304) {
+        response = new Response(body, responseOptions);
+        resolve(response);
+        return;
+      }
+
+      // For Node v6+
+      // Be less strict when decoding compressed responses, since sometimes
+      // servers send slightly invalid responses that are still accepted
+      // by common browsers.
+      // Always using Z_SYNC_FLUSH is what cURL does.
+      const zlibOptions = {
+        flush: zlib.Z_SYNC_FLUSH,
+        finishFlush: zlib.Z_SYNC_FLUSH
+      };
+
+      // For gzip
+      if (codings === 'gzip' || codings === 'x-gzip') {
+        body = pump(body, zlib.createGunzip(zlibOptions), error => {
+          if (error) {
+            reject(error);
+          }
+        });
+        response = new Response(body, responseOptions);
+        resolve(response);
+        return;
+      }
+
+      // For deflate
+      // if (codings === 'deflate' || codings === 'x-deflate') {
+      //   // Handle the infamous raw deflate response from old servers
+      //   // a hack for old IIS and Apache servers
+      //   const raw = pump(response_, new PassThrough(), error => {
+      //     if (error) {
+      //       reject(error);
+      //     }
+      //   });
+      
+      //   raw.once('data', chunk => {
+      //     // See http://stackoverflow.com/questions/37519828
+      //     if ((chunk[0] & 0x0F) === 0x08) {
+      //       body = pump(body, zlib.createInflate(), error => {
+      //         if (error) {
+      //           reject(error);
+      //         }
+      //       });
+      //     } else {
+      //       body = pump(body, zlib.createInflateRaw(), error => {
+      //         if (error) {
+      //           reject(error);
+      //         }
+      //       });
+      //     }
+
+      //     response = new Response(body, responseOptions);
+      //     resolve(response);
+      //   });
+
+      //   raw.once('end', () => {
+      //     // Some old IIS servers return zero-length OK deflate responses, so
+      //     // 'data' is never emitted. See https://github.com/node-fetch/node-fetch/pull/903
+      //     if (!response) {
+      //       response = new Response(body, responseOptions);
+      //       resolve(response);
+      //     }
+      //   });
+      //   return;
+      // }
+
+      // For br
+      if (codings === 'br') {
+        body = pump(body, zlib.createBrotliDecompress(), error => {
+          if (error) {
+            reject(error);
+          }
+        });
+        response = new Response(body, responseOptions);
+        resolve(response);
+        return;
+      }
+
+      // Otherwise, use response as-is
+      response = new Response(body, responseOptions);
+      resolve(response);
+    });
+
+    writeToStream(inputStream, request).catch(reject);
+  });
+}