@loaders.gl/loader-utils 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/lib/worker-loader-utils/create-loader-worker.js

@@ -1,102 +1,98 @@
-import { WorkerBody } from '@loaders.gl/worker-utils';
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLoaderWorker = void 0;
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+// import {validateLoaderVersion} from './validate-loader-version.js';
 let requestId = 0;
-export function createLoaderWorker(loader) {
-  if (typeof self === 'undefined') {
-    return;
-  }
-
-  WorkerBody.onmessage = async (type, payload) => {
-    switch (type) {
-      case 'process':
-        try {
-          const {
-            input,
-            options = {}
-          } = payload;
-          const result = await parseData({
-            loader,
-            arrayBuffer: input,
-            options,
-            context: {
-              parse: parseOnMainThread
-            }
-          });
-          WorkerBody.postMessage('done', {
-            result
-          });
-        } catch (error) {
-          const message = error instanceof Error ? error.message : '';
-          WorkerBody.postMessage('error', {
-            error: message
-          });
-        }
-
-        break;
-
-      default:
+/**
+ * Set up a WebWorkerGlobalScope to talk with the main thread
+ * @param loader
+ */
+function createLoaderWorker(loader) {
+    // Check that we are actually in a worker thread
+    if (!worker_utils_1.WorkerBody.inWorkerThread()) {
+        return;
     }
-  };
+    worker_utils_1.WorkerBody.onmessage = async (type, payload) => {
+        switch (type) {
+            case 'process':
+                try {
+                    // validateLoaderVersion(loader, data.source.split('@')[1]);
+                    const { input, options = {}, context = {} } = payload;
+                    const result = await parseData({
+                        loader,
+                        arrayBuffer: input,
+                        options,
+                        context: {
+                            ...context,
+                            parse: parseOnMainThread
+                        }
+                    });
+                    worker_utils_1.WorkerBody.postMessage('done', { result });
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : '';
+                    worker_utils_1.WorkerBody.postMessage('error', { error: message });
+                }
+                break;
+            default:
+        }
+    };
 }
-
+exports.createLoaderWorker = createLoaderWorker;
 function parseOnMainThread(arrayBuffer, options) {
-  return new Promise((resolve, reject) => {
-    const id = requestId++;
-
-    const onMessage = (type, payload) => {
-      if (payload.id !== id) {
-        return;
-      }
-
-      switch (type) {
-        case 'done':
-          WorkerBody.removeEventListener(onMessage);
-          resolve(payload.result);
-          break;
-
-        case 'error':
-          WorkerBody.removeEventListener(onMessage);
-          reject(payload.error);
-          break;
-
-        default:
-      }
-    };
-
-    WorkerBody.addEventListener(onMessage);
-    const payload = {
-      id,
-      input: arrayBuffer,
-      options
-    };
-    WorkerBody.postMessage('process', payload);
-  });
+    return new Promise((resolve, reject) => {
+        const id = requestId++;
+        /**
+         */
+        const onMessage = (type, payload) => {
+            if (payload.id !== id) {
+                // not ours
+                return;
+            }
+            switch (type) {
+                case 'done':
+                    worker_utils_1.WorkerBody.removeEventListener(onMessage);
+                    resolve(payload.result);
+                    break;
+                case 'error':
+                    worker_utils_1.WorkerBody.removeEventListener(onMessage);
+                    reject(payload.error);
+                    break;
+                default:
+                // ignore
+            }
+        };
+        worker_utils_1.WorkerBody.addEventListener(onMessage);
+        // Ask the main thread to decode data
+        const payload = { id, input: arrayBuffer, options };
+        worker_utils_1.WorkerBody.postMessage('process', payload);
+    });
 }
-
-async function parseData({
-  loader,
-  arrayBuffer,
-  options,
-  context
-}) {
-  let data;
-  let parser;
-
-  if (loader.parseSync || loader.parse) {
-    data = arrayBuffer;
-    parser = loader.parseSync || loader.parse;
-  } else if (loader.parseTextSync) {
-    const textDecoder = new TextDecoder();
-    data = textDecoder.decode(arrayBuffer);
-    parser = loader.parseTextSync;
-  } else {
-    throw new Error("Could not load data with ".concat(loader.name, " loader"));
-  }
-
-  options = { ...options,
-    modules: loader && loader.options && loader.options.modules || {},
-    worker: false
-  };
-  return await parser(data, { ...options
-  }, context, loader);
+// TODO - Support byteOffset and byteLength (enabling parsing of embedded binaries without copies)
+// TODO - Why not support async loader.parse* funcs here?
+// TODO - Why not reuse a common function instead of reimplementing loader.parse* selection logic? Keeping loader small?
+// TODO - Lack of appropriate parser functions can be detected when we create worker, no need to wait until parse
+async function parseData({ loader, arrayBuffer, options, context }) {
+    let data;
+    let parser;
+    if (loader.parseSync || loader.parse) {
+        data = arrayBuffer;
+        parser = loader.parseSync || loader.parse;
+    }
+    else if (loader.parseTextSync) {
+        const textDecoder = new TextDecoder();
+        data = textDecoder.decode(arrayBuffer);
+        parser = loader.parseTextSync;
+    }
+    else {
+        throw new Error(`Could not load data with ${loader.name} loader`);
+    }
+    // TODO - proper merge in of loader options...
+    options = {
+        ...options,
+        modules: (loader && loader.options && loader.options.modules) || {},
+        worker: false
+    };
+    return await parser(data, { ...options }, context, loader);
 }
-//# sourceMappingURL=create-loader-worker.js.map

package/dist/lib/worker-loader-utils/encode-with-worker.d.ts

@@ -0,0 +1,8 @@
+import { Writer, WriterOptions } from '../../types';
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+export declare function canEncodeWithWorker(writer: Writer, options?: WriterOptions): any;
+//# sourceMappingURL=encode-with-worker.d.ts.map

package/dist/lib/worker-loader-utils/encode-with-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encode-with-worker.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-loader-utils/encode-with-worker.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,aAAa,CAAC;AAGlD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,OAW1E"}

package/dist/lib/worker-loader-utils/encode-with-worker.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.canEncodeWithWorker = void 0;
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+const globals_1 = require("../env-utils/globals");
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+function canEncodeWithWorker(writer, options) {
+    if (!worker_utils_1.WorkerFarm.isSupported()) {
+        return false;
+    }
+    // Node workers are still experimental
+    if (!globals_1.isBrowser && !options?._nodeWorkers) {
+        return false;
+    }
+    return writer.worker && options?.worker;
+}
+exports.canEncodeWithWorker = canEncodeWithWorker;

package/dist/lib/worker-loader-utils/parse-with-worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-with-worker.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-loader-utils/parse-with-worker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAE,aAAa,EAAC,MAAM,aAAa,CAAC;AAGtE;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,4BAMzE;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,GAAG,EACT,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,EACvB,iBAAiB,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,EAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,gBA2B/F"}
+{"version":3,"file":"parse-with-worker.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-loader-utils/parse-with-worker.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAE,aAAa,EAAC,MAAM,aAAa,CAAC;AAGtE;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,4BAWzE;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,GAAG,EACT,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,EACvB,iBAAiB,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,EAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,gBA8B/F"}

package/dist/lib/worker-loader-utils/parse-with-worker.js

@@ -1,64 +1,81 @@
-import { WorkerFarm, getWorkerURL } from '@loaders.gl/worker-utils';
-export function canParseWithWorker(loader, options) {
-  if (!WorkerFarm.isSupported()) {
-    return false;
-  }
-
-  return loader.worker && (options === null || options === void 0 ? void 0 : options.worker);
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseWithWorker = exports.canParseWithWorker = void 0;
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+const worker_utils_2 = require("@loaders.gl/worker-utils");
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+function canParseWithWorker(loader, options) {
+    if (!worker_utils_2.WorkerFarm.isSupported()) {
+        return false;
+    }
+    // Node workers are still experimental
+    if (!worker_utils_1.isBrowser && !options?._nodeWorkers) {
+        return false;
+    }
+    return loader.worker && options?.worker;
 }
-export async function parseWithWorker(loader, data, options, context, parseOnMainThread) {
-  const name = loader.id;
-  const url = getWorkerURL(loader, options);
-  const workerFarm = WorkerFarm.getWorkerFarm(options);
-  const workerPool = workerFarm.getWorkerPool({
-    name,
-    url
-  });
-  options = JSON.parse(JSON.stringify(options));
-  const job = await workerPool.startJob('process-on-worker', onMessage.bind(null, parseOnMainThread));
-  job.postMessage('process', {
-    input: data,
-    options
-  });
-  const result = await job.result;
-  return await result.result;
+exports.canParseWithWorker = canParseWithWorker;
+/**
+ * this function expects that the worker function sends certain messages,
+ * this can be automated if the worker is wrapper by a call to createLoaderWorker in @loaders.gl/loader-utils.
+ */
+async function parseWithWorker(loader, data, options, context, parseOnMainThread) {
+    const name = loader.id; // TODO
+    const url = (0, worker_utils_2.getWorkerURL)(loader, options);
+    const workerFarm = worker_utils_2.WorkerFarm.getWorkerFarm(options);
+    const workerPool = workerFarm.getWorkerPool({ name, url });
+    // options.log object contains functions which cannot be transferred
+    // context.fetch & context.parse functions cannot be transferred
+    // TODO - decide how to handle logging on workers
+    options = JSON.parse(JSON.stringify(options));
+    context = JSON.parse(JSON.stringify(context || {}));
+    const job = await workerPool.startJob('process-on-worker', 
+    // @ts-expect-error
+    onMessage.bind(null, parseOnMainThread) // eslint-disable-line @typescript-eslint/no-misused-promises
+    );
+    job.postMessage('process', {
+        // @ts-ignore
+        input: data,
+        options,
+        context
+    });
+    const result = await job.result;
+    // TODO - what is going on here?
+    return await result.result;
 }
-
+exports.parseWithWorker = parseWithWorker;
+/**
+ * Handle worker's responses to the main thread
+ * @param job
+ * @param type
+ * @param payload
+ */
 async function onMessage(parseOnMainThread, job, type, payload) {
-  switch (type) {
-    case 'done':
-      job.done(payload);
-      break;
-
-    case 'error':
-      job.error(new Error(payload.error));
-      break;
-
-    case 'process':
-      const {
-        id,
-        input,
-        options
-      } = payload;
-
-      try {
-        const result = await parseOnMainThread(input, options);
-        job.postMessage('done', {
-          id,
-          result
-        });
-      } catch (error) {
-        const message = error instanceof Error ? error.message : 'unknown error';
-        job.postMessage('error', {
-          id,
-          error: message
-        });
-      }
-
-      break;
-
-    default:
-      console.warn("parse-with-worker unknown message ".concat(type));
-  }
+    switch (type) {
+        case 'done':
+            job.done(payload);
+            break;
+        case 'error':
+            job.error(new Error(payload.error));
+            break;
+        case 'process':
+            // Worker is asking for main thread to parseO
+            const { id, input, options } = payload;
+            try {
+                const result = await parseOnMainThread(input, options);
+                job.postMessage('done', { id, result });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : 'unknown error';
+                job.postMessage('error', { id, error: message });
+            }
+            break;
+        default:
+            // eslint-disable-next-line
+            console.warn(`parse-with-worker unknown message ${type}`);
+    }
 }
-//# sourceMappingURL=parse-with-worker.js.map

package/dist/types.d.ts

@@ -1,10 +1,13 @@
 /// <reference types="node" />
-export declare type TypedIntArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Int32Array | Uint32Array;
-export declare type TypedFloatArray = Uint16Array | Float32Array | Float64Array;
-export declare type TypedArray = TypedIntArray | TypedFloatArray;
-export declare type NumericArray = Array<number> | TypedIntArray | TypedFloatArray;
-declare type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
-export declare type LoaderOptions = {
+export type TypedIntArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Int32Array | Uint32Array;
+export type TypedFloatArray = Uint16Array | Float32Array | Float64Array;
+export type TypedArray = TypedIntArray | TypedFloatArray;
+export type NumericArray = Array<number> | TypedIntArray | TypedFloatArray;
+type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
+/**
+ * Core Loader Options
+ */
+export type LoaderOptions = {
     /** fetch options or a custom fetch function */
     fetch?: typeof fetch | FetchLike | RequestInit | null;
     /** Do not throw on errors */
@@ -30,7 +33,7 @@ export declare type LoaderOptions = {
     /** Transforms to run on incoming batches */
     transforms?: TransformBatches[];
     /** CDN load workers from */
-    CDN?: string;
+    CDN?: string | null;
     /** Set to `false` to disable workers */
     worker?: boolean;
     /** Number of concurrent workers (per loader) on desktop browser */
@@ -39,128 +42,138 @@ export declare type LoaderOptions = {
     maxMobileConcurrency?: number;
     /** Set to `false` to prevent reuse workers */
     reuseWorkers?: boolean;
-    /** set to 'test' to run worker type */
+    /** Whether to use workers under Node.js (experimental) */
+    _nodeWorkers?: boolean;
+    /** set to 'test' to run local worker */
     _workerType?: string;
     /** @deprecated `options.batchType` removed, Use `options.<loader>.type` instead */
     batchType?: 'row' | 'columnar' | 'arrow';
     /** @deprecated `options.throw removed`, Use `options.nothrow` instead */
     throws?: boolean;
     /** @deprecated `options.dataType` no longer used */
-    dataType?: any;
+    dataType?: never;
     /** @deprecated `options.uri` no longer used */
-    uri?: any;
+    uri?: never;
     /** @deprecated `options.method` removed. Use `options.fetch.method` */
-    method?: any;
+    method?: never;
     /** @deprecated `options.headers` removed. Use `options.fetch.headers` */
-    headers?: any;
+    headers?: never;
     /** @deprecated `options.body` removed. Use `options.fetch.body` */
-    body?: any;
+    body?: never;
     /** @deprecated `options.mode` removed. Use `options.fetch.mode` */
-    mode?: any;
+    mode?: never;
     /** @deprecated `options.credentials` removed. Use `options.fetch.credentials` */
-    credentials?: any;
+    credentials?: never;
     /** @deprecated `options.cache` removed. Use `options.fetch.cache` */
-    cache?: any;
+    cache?: never;
     /** @deprecated `options.redirect` removed. Use `options.fetch.redirect` */
-    redirect?: any;
+    redirect?: never;
     /** @deprecated `options.referrer` removed. Use `options.fetch.referrer` */
-    referrer?: any;
+    referrer?: never;
     /** @deprecated `options.referrerPolicy` removed. Use `options.fetch.referrerPolicy` */
-    referrerPolicy?: any;
+    referrerPolicy?: never;
     /** @deprecated `options.integrity` removed. Use `options.fetch.integrity` */
-    integrity?: any;
+    integrity?: never;
     /** @deprecated `options.keepalive` removed. Use `options.fetch.keepalive` */
-    keepalive?: any;
+    keepalive?: never;
     /** @deprecated `options.signal` removed. Use `options.fetch.signal` */
-    signal?: any;
-    [loaderId: string]: any;
+    signal?: never;
+    [loaderId: string]: unknown;
 };
-declare type PreloadOptions = {
-    [key: string]: any;
+type PreloadOptions = {
+    [key: string]: unknown;
 };
 /**
  * A worker loader definition that can be used with `@loaders.gl/core` functions
  */
-export declare type Loader = {
+export type Loader<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> = {
+    dataType?: DataT;
+    batchType?: BatchT;
+    options: LoaderOptionsT;
+    deprecatedOptions?: Record<string, string | Record<string, string>>;
     name: string;
     id: string;
     module: string;
     version: string;
     worker?: string | boolean;
-    options: object;
-    deprecatedOptions?: object;
     category?: string;
     extensions: string[];
     mimeTypes: string[];
     binary?: boolean;
     text?: boolean;
-    tests?: (((ArrayBuffer: any) => boolean) | ArrayBuffer | string)[];
+    tests?: (((ArrayBuffer: ArrayBuffer) => boolean) | ArrayBuffer | string)[];
     supported?: boolean;
-    testText?: (string: any) => boolean;
+    testText?: (string: string) => boolean;
 };
 /**
  * A "bundled" loader definition that can be used with `@loaders.gl/core` functions
  * If a worker loader is supported it will also be supported.
  */
-export declare type LoaderWithParser = Loader & {
-    testText?: (string: any) => boolean;
-    parse: Parse;
+export type LoaderWithParser<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> = Loader<DataT, BatchT, LoaderOptionsT> & {
+    testText?: (string: string) => boolean;
+    parse: (arrayBuffer: ArrayBuffer, options?: LoaderOptionsT, context?: LoaderContext) => Promise<DataT>;
     preload?: Preload;
-    parseSync?: ParseSync;
-    parseText?: ParseText;
-    parseTextSync?: ParseTextSync;
-    parseInBatches?: ParseInBatches;
-    parseFileInBatches?: ParseFileInBatches;
+    parseSync?: (arrayBuffer: ArrayBuffer, options?: LoaderOptionsT, context?: LoaderContext) => DataT;
+    parseText?: (text: string, options?: LoaderOptionsT) => Promise<DataT>;
+    parseTextSync?: (text: string, options?: LoaderOptionsT) => DataT;
+    parseInBatches?: (iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: LoaderOptionsT, context?: LoaderContext) => AsyncIterable<BatchT>;
+    parseFileInBatches?: (file: Blob, options?: LoaderOptionsT, context?: LoaderContext) => AsyncIterable<BatchT>;
+};
+export type LoaderContext = {
+    loaders?: Loader[] | null;
+    url?: string;
+    fetch: typeof fetch | FetchLike;
+    response?: Response;
+    parse: (arrayBuffer: ArrayBuffer, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext) => Promise<any>;
+    parseSync?: (arrayBuffer: ArrayBuffer, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext) => any;
+    parseInBatches?: (iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext) => AsyncIterable<any> | Promise<AsyncIterable<any>>;
 };
+type Preload = (url: string, options?: PreloadOptions) => any;
+/** Typescript helper to extract options type from a generic loader type */
+export type LoaderOptionsType<T = Loader> = T extends Loader<any, any, infer Options> ? Options : never;
+/** Typescript helper to extract data type from a generic loader type */
+export type LoaderReturnType<T = Loader> = T extends Loader<infer Return, any, any> ? Return : never;
+/** Typescript helper to extract batch type from a generic loader type */
+export type LoaderBatchType<T = Loader> = T extends Loader<any, infer Batch, any> ? Batch : never;
 /** Options for writers */
-export declare type WriterOptions = {};
+export type WriterOptions = {
+    /** worker source. If is set will be used instead of loading worker from the Internet */
+    souce?: string | null;
+    /** writer-specific options */
+    [writerId: string]: any;
+};
 /**
  * A writer definition that can be used with `@loaders.gl/core` functions
  */
-export declare type Writer = {
+export type Writer<DataT = unknown, BatchT = unknown, WriterOptionsT = WriterOptions> = {
     name: string;
     id: string;
     module: string;
     version: string;
-    options: object;
-    deprecatedOptions?: object;
-    binary?: boolean;
+    worker?: string | boolean;
     extensions?: string[];
     mimeTypes?: string[];
+    binary?: boolean;
     text?: boolean;
-    encode?: Encode;
+    options: WriterOptionsT;
+    deprecatedOptions?: Record<string, string>;
     encodeSync?: EncodeSync;
-    encodeInBatches?: EncodeInBatches;
     encodeURLtoURL?: EncodeURLtoURL;
-    encodeText?: EncodeText;
-};
-export declare type LoaderContext = {
-    loaders?: Loader[] | null;
-    url?: string;
-    fetch: typeof fetch;
-    parse: (arrayBuffer: ArrayBuffer, loaders?: any, options?: LoaderOptions, context?: LoaderContext) => Promise<any>;
-    parseSync?: (arrayBuffer: ArrayBuffer, loaders?: any, options?: LoaderOptions, context?: LoaderContext) => any;
-    parseInBatches?: (iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, loaders?: any, options?: LoaderOptions, context?: LoaderContext) => AsyncIterable<any> | Promise<AsyncIterable<any>>;
+    encode?(data: DataT, options?: WriterOptionsT): Promise<ArrayBuffer>;
+    encodeText?(table: DataT, options?: WriterOptionsT): Promise<string> | string;
+    encodeInBatches?(data: AsyncIterable<any>, options?: WriterOptionsT): AsyncIterable<ArrayBuffer>;
 };
-declare type Parse = (arrayBuffer: ArrayBuffer, options?: LoaderOptions, context?: LoaderContext) => Promise<any>;
-declare type ParseSync = (arrayBuffer: ArrayBuffer, options?: LoaderOptions, context?: LoaderContext) => any;
-declare type ParseText = (text: string, options?: LoaderOptions) => Promise<any>;
-declare type ParseTextSync = (text: string, options?: LoaderOptions) => any;
-declare type ParseInBatches = (iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: LoaderOptions, context?: LoaderContext) => AsyncIterable<any>;
-declare type ParseFileInBatches = (file: Blob, options?: LoaderOptions, context?: LoaderContext) => AsyncIterable<any>;
-declare type Encode = (data: any, options?: WriterOptions) => Promise<ArrayBuffer>;
-declare type EncodeSync = (data: any, options?: WriterOptions) => ArrayBuffer;
-declare type EncodeText = Function;
-declare type EncodeInBatches = Function;
-declare type EncodeURLtoURL = (inputUrl: string, outputUrl: string, options?: WriterOptions) => Promise<string>;
-declare type Preload = (url: string, options?: PreloadOptions) => any;
-export declare type TransformBatches = (asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>) => AsyncIterable<ArrayBuffer>;
+type EncodeSync = (data: any, options?: WriterOptions) => ArrayBuffer;
+type EncodeURLtoURL = (inputUrl: string, outputUrl: string, options?: WriterOptions) => Promise<string>;
+/** Typescript helper to extract the writer options type from a generic writer type */
+export type WriterOptionsType<T = Writer> = T extends Writer<unknown, unknown, infer Options> ? Options : never;
+export type TransformBatches = (asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>) => AsyncIterable<ArrayBuffer>;
 /** Types that can be synchronously parsed */
-export declare type SyncDataType = string | ArrayBuffer;
+export type SyncDataType = string | ArrayBuffer;
 /** Types that can be parsed async */
-export declare type DataType = string | ArrayBuffer | File | Blob | Response | ReadableStream | Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>;
+export type DataType = string | ArrayBuffer | File | Blob | Response | ReadableStream | Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>;
 /** Types that can be parsed in batches */
-export declare type BatchableDataType = DataType | Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer> | Promise<AsyncIterable<ArrayBuffer>>;
+export type BatchableDataType = DataType | Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer> | Promise<AsyncIterable<ArrayBuffer>>;
 /**
  * A FileSystem interface can encapsulate a FileList, a ZipFile, a GoogleDrive etc.
  */
@@ -188,14 +201,14 @@ export interface IFileSystem {
      */
     fetch(filename: string, options?: object): Promise<Response>;
 }
-declare type ReadOptions = {
+type ReadOptions = {
     buffer?: ArrayBuffer;
     offset?: number;
     length?: number;
     position?: number;
 };
 export interface IRandomAccessReadFileSystem extends IFileSystem {
-    open(path: string, flags: any, mode?: any): Promise<any>;
+    open(path: string, flags: string | number, mode?: any): Promise<any>;
     close(fd: any): Promise<void>;
     fstat(fd: any): Promise<object>;
     read(fd: any, options?: ReadOptions): Promise<{