@loaders.gl/loader-utils 3.1.0-alpha.4 → 3.1.0-alpha.5

package/dist/lib/filesystems/node-filesystem.d.ts

@@ -0,0 +1,37 @@
+/// <reference types="node" />
+import * as fs from '../node/fs';
+import { IFileSystem, IRandomAccessReadFileSystem } from '../../types';
+declare type Stat = {
+    size: number;
+    isDirectory: () => boolean;
+    info?: fs.Stats;
+};
+declare type ReadOptions = {
+    buffer?: Buffer;
+    offset?: number;
+    length?: number;
+    position?: number;
+};
+/**
+ * FileSystem pass-through for Node.js
+ * Compatible with BrowserFileSystem.
+ * @param options
+ */
+export default class NodeFileSystem implements IFileSystem, IRandomAccessReadFileSystem {
+    constructor(options: {
+        [key: string]: any;
+    });
+    readdir(dirname?: string, options?: {}): Promise<any[]>;
+    stat(path: string, options?: {}): Promise<Stat>;
+    fetch(path: string, options: {
+        [key: string]: any;
+    }): Promise<any>;
+    open(path: string, flags: string | number, mode?: any): Promise<number>;
+    close(fd: number): Promise<void>;
+    fstat(fd: number): Promise<Stat>;
+    read(fd: number, { buffer, offset, length, position }: ReadOptions): Promise<{
+        bytesRead: number;
+        buffer: Buffer;
+    }>;
+}
+export {};

package/dist/lib/iterators/async-iteration.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Iterate over async iterator, without resetting iterator if end is not reached
+ * - forEach intentionally does not reset iterator if exiting loop prematurely
+ *   so that iteration can continue in a second loop
+ * - It is recommended to use a standard for-await as last loop to ensure
+ *   iterator gets properly reset
+ *
+ * TODO - optimize using sync iteration if argument is an Iterable?
+ *
+ * @param iterator
+ * @param visitor
+ */
+export declare function forEach(iterator: any, visitor: any): Promise<void>;
+/**
+ * Concatenates all data chunks yielded by an (async) iterator
+ * This function can e.g. be used to enable atomic parsers to work on (async) iterator inputs
+ */
+export declare function concatenateArrayBuffersAsync(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>): Promise<ArrayBuffer>;
+export declare function concatenateStringsAsync(asyncIterator: AsyncIterable<string> | Iterable<string>): Promise<string>;

package/dist/lib/iterators/text-iterators.d.ts

@@ -0,0 +1,18 @@
+export declare function makeTextDecoderIterator(arrayBufferIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: TextDecoderOptions): AsyncIterable<string>;
+export declare function makeTextEncoderIterator(textIterator: AsyncIterable<string> | Iterable<ArrayBuffer>): AsyncIterable<ArrayBuffer>;
+/**
+ * @param textIterator async iterable yielding strings
+ * @returns an async iterable over lines
+ * See http://2ality.com/2018/04/async-iter-nodejs.html
+ */
+export declare function makeLineIterator(textIterator: AsyncIterable<string>): AsyncIterable<string>;
+/**
+ * @param lineIterator async iterable yielding lines
+ * @returns async iterable yielding numbered lines
+ *
+ * See http://2ality.com/2018/04/async-iter-nodejs.html
+ */
+export declare function makeNumberedLineIterator(lineIterator: AsyncIterable<string>): AsyncIterable<{
+    counter: number;
+    line: string;
+}>;

package/dist/lib/node/buffer.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+/**
+ * Convert Buffer to ArrayBuffer
+ */
+export declare function toArrayBuffer(buffer: any): any;
+/**
+ * Convert (copy) ArrayBuffer to Buffer
+ */
+export declare function toBuffer(binaryData: any): void | Buffer;

package/dist/lib/node/fs.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+import fs from 'fs';
+export type { Stats } from 'fs';
+/** Wrapper for Node.js fs method */
+export declare const readdir: typeof fs.readdir.__promisify__;
+/** Wrapper for Node.js fs method */
+export declare const stat: typeof fs.stat.__promisify__;
+/** Wrapper for Node.js fs method */
+export declare const readFile: typeof fs.readFile.__promisify__;
+/** Wrapper for Node.js fs method */
+export declare const readFileSync: typeof fs.readFileSync;
+/** Wrapper for Node.js fs method */
+export declare const writeFile: typeof fs.writeFile.__promisify__;
+/** Wrapper for Node.js fs method */
+export declare const writeFileSync: typeof fs.writeFileSync;
+/** Wrapper for Node.js fs method */
+export declare const open: typeof fs.open.__promisify__;
+/** Wrapper for Node.js fs method */
+export declare const close: typeof fs.close.__promisify__;
+/** Wrapper for Node.js fs method */
+export declare const read: typeof fs.read.__promisify__;
+/** Wrapper for Node.js fs method */
+export declare const fstat: typeof fs.fstat.__promisify__;
+export declare const isSupported: boolean;
+export declare function _readToArrayBuffer(fd: number, start: number, length: number): Promise<any>;

package/dist/lib/node/util.d.ts

@@ -0,0 +1,4 @@
+/// <reference types="node" />
+import * as util from 'util';
+/** Wrapper for Node.js promisify */
+export declare const promisify: typeof util.promisify;

package/dist/lib/parser-utils/parse-json.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Minimal JSON parser that throws more meaningful error messages
+ */
+export declare function parseJSON(string: string): any;

package/dist/lib/path-utils/file-aliases.d.ts

@@ -0,0 +1,16 @@
+export declare function setPathPrefix(prefix: string): void;
+export declare function getPathPrefix(): string;
+/**
+ *
+ * @param aliases
+ *
+ * Note: addAliases are an experimental export, they are only for testing of loaders.gl loaders
+ * not intended as a generic aliasing mechanism
+ */
+export declare function addAliases(aliases: {
+    [aliasPath: string]: string;
+}): void;
+/**
+ * Resolves aliases and adds path-prefix to paths
+ */
+export declare function resolvePath(filename: string): string;

package/dist/lib/path-utils/path.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Replacement for Node.js path.filename
+ * @param url
+ */
+export declare function filename(url: string): string;
+/**
+ * Replacement for Node.js path.dirname
+ * @param url
+ */
+export declare function dirname(url: string): string;
+/**
+ * Replacement for Node.js path.join
+ * @param parts
+ */
+export declare function join(...parts: string[]): string;

package/dist/lib/request-utils/request-scheduler.d.ts

@@ -0,0 +1,61 @@
+import { Stats } from '@probe.gl/stats';
+declare type Handle = any;
+declare type DoneFunction = () => any;
+declare type GetPriorityFunction = () => number;
+declare type RequestResult = {
+    done: DoneFunction;
+} | null;
+/** RequestScheduler Options */
+export declare type RequestSchedulerProps = {
+    id?: string;
+    throttleRequests?: boolean;
+    maxRequests?: number;
+};
+/** Tracks one request */
+declare type Request = {
+    handle: Handle;
+    priority: number;
+    getPriority: GetPriorityFunction;
+    resolve?: (value: any) => any;
+};
+/**
+ * Used to issue a request, without having them "deeply queued" by the browser.
+ * @todo - Track requests globally, across multiple servers
+ */
+export default class RequestScheduler {
+    readonly props: Required<RequestSchedulerProps>;
+    readonly stats: Stats;
+    activeRequestCount: number;
+    /** Tracks the number of active requests and prioritizes/cancels queued requests. */
+    private requestQueue;
+    private requestMap;
+    private deferredUpdate;
+    constructor(props?: RequestSchedulerProps);
+    /**
+     * Called by an application that wants to issue a request, without having it deeply queued by the browser
+     *
+     * When the returned promise resolved, it is OK for the application to issue a request.
+     * The promise resolves to an object that contains a `done` method.
+     * When the application's request has completed (or failed), the application must call the `done` function
+     *
+     * @param handle
+     * @param getPriority will be called when request "slots" open up,
+     *    allowing the caller to update priority or cancel the request
+     *    Highest priority executes first, priority < 0 cancels the request
+     * @returns a promise
+     *   - resolves to a object (with a `done` field) when the request can be issued without queueing,
+     *   - resolves to `null` if the request has been cancelled (by the callback return < 0).
+     *     In this case the application should not issue the request
+     */
+    scheduleRequest(handle: Handle, getPriority?: GetPriorityFunction): Promise<RequestResult>;
+    _issueRequest(request: Request): Promise<any>;
+    /** We check requests asynchronously, to prevent multiple updates */
+    _issueNewRequests(): void;
+    /** Refresh all requests  */
+    _issueNewRequestsAsync(): void;
+    /** Ensure all requests have updated priorities, and that no longer valid requests are cancelled */
+    _updateAllRequests(): void;
+    /** Update a single request by calling the callback */
+    _updateRequest(request: any): boolean;
+}
+export {};

package/dist/lib/worker-loader-utils/create-loader-worker.d.ts

@@ -0,0 +1,6 @@
+import type { LoaderWithParser } from '../../types';
+/**
+ * Set up a WebWorkerGlobalScope to talk with the main thread
+ * @param loader
+ */
+export declare function createLoaderWorker(loader: LoaderWithParser): void;

package/dist/lib/worker-loader-utils/parse-with-worker.d.ts

@@ -0,0 +1,14 @@
+import type { Loader, LoaderOptions, LoaderContext } from '../../types';
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+export declare function canParseWithWorker(loader: Loader, options?: LoaderOptions): boolean | "" | undefined;
+/**
+ * this function expects that the worker function sends certain messages,
+ * this can be automated if the worker is wrapper by a call to createLoaderWorker in @loaders.gl/loader-utils.
+ */
+export declare function parseWithWorker(loader: Loader, data: any, options?: LoaderOptions, context?: LoaderContext, parseOnMainThread?: (arrayBuffer: ArrayBuffer, options: {
+    [key: string]: any;
+}) => Promise<void>): Promise<any>;

package/dist/types.d.ts

@@ -0,0 +1,206 @@
+/// <reference types="node" />
+export declare type TypedIntArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Int32Array | Uint32Array;
+export declare type TypedFloatArray = Uint16Array | Float32Array | Float64Array;
+export declare type TypedArray = TypedIntArray | TypedFloatArray;
+export declare type NumericArray = Array<number> | TypedIntArray | TypedFloatArray;
+declare type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
+export declare type LoaderOptions = {
+    /** fetch options or a custom fetch function */
+    fetch?: typeof fetch | FetchLike | RequestInit | null;
+    /** Do not throw on errors */
+    nothrow?: boolean;
+    /** loader selection, search first for supplied mimeType */
+    mimeType?: string;
+    /** loader selection, provide fallback mimeType is server does not provide */
+    fallbackMimeType?: string;
+    /** loader selection, avoid searching registered loaders */
+    ignoreRegisteredLoaders?: boolean;
+    /** Experimental: Supply a logger to the parser */
+    log?: any;
+    /** Size of each batch. `auto` matches batches to size of incoming chunks */
+    batchSize?: number | 'auto';
+    /** Minimal amount of time between batches */
+    batchDebounceMs?: number;
+    /** Stop loading after a given number of rows (compare SQL limit clause) */
+    limit?: 0;
+    /** Experimental: Stop loading after reaching */
+    _limitMB?: 0;
+    /** Generate metadata batches */
+    metadata?: boolean;
+    /** Transforms to run on incoming batches */
+    transforms?: TransformBatches[];
+    /** CDN load workers from */
+    CDN?: string;
+    /** Set to `false` to disable workers */
+    worker?: boolean;
+    /** Number of concurrent workers (per loader) on desktop browser */
+    maxConcurrency?: number;
+    /** Number of concurrent workers (per loader) on mobile browsers */
+    maxMobileConcurrency?: number;
+    /** Set to `false` to prevent reuse workers */
+    reuseWorkers?: boolean;
+    /** set to 'test' to run worker type */
+    _workerType?: string;
+    /** @deprecated `options.batchType` removed, Use `options.<loader>.type` instead */
+    batchType?: 'row' | 'columnar' | 'arrow';
+    /** @deprecated `options.throw removed`, Use `options.nothrow` instead */
+    throws?: boolean;
+    /** @deprecated `options.dataType` no longer used */
+    dataType?: any;
+    /** @deprecated `options.uri` no longer used */
+    uri?: any;
+    /** @deprecated `options.method` removed. Use `options.fetch.method` */
+    method?: any;
+    /** @deprecated `options.headers` removed. Use `options.fetch.headers` */
+    headers?: any;
+    /** @deprecated `options.body` removed. Use `options.fetch.body` */
+    body?: any;
+    /** @deprecated `options.mode` removed. Use `options.fetch.mode` */
+    mode?: any;
+    /** @deprecated `options.credentials` removed. Use `options.fetch.credentials` */
+    credentials?: any;
+    /** @deprecated `options.cache` removed. Use `options.fetch.cache` */
+    cache?: any;
+    /** @deprecated `options.redirect` removed. Use `options.fetch.redirect` */
+    redirect?: any;
+    /** @deprecated `options.referrer` removed. Use `options.fetch.referrer` */
+    referrer?: any;
+    /** @deprecated `options.referrerPolicy` removed. Use `options.fetch.referrerPolicy` */
+    referrerPolicy?: any;
+    /** @deprecated `options.integrity` removed. Use `options.fetch.integrity` */
+    integrity?: any;
+    /** @deprecated `options.keepalive` removed. Use `options.fetch.keepalive` */
+    keepalive?: any;
+    /** @deprecated `options.signal` removed. Use `options.fetch.signal` */
+    signal?: any;
+    [loaderId: string]: any;
+};
+declare type PreloadOptions = {
+    [key: string]: any;
+};
+/**
+ * A worker loader definition that can be used with `@loaders.gl/core` functions
+ */
+export declare type Loader = {
+    name: string;
+    id: string;
+    module: string;
+    version: string;
+    worker?: string | boolean;
+    options: object;
+    deprecatedOptions?: object;
+    category?: string;
+    extensions: string[];
+    mimeTypes: string[];
+    binary?: boolean;
+    text?: boolean;
+    tests?: (((ArrayBuffer: any) => boolean) | ArrayBuffer | string)[];
+    supported?: boolean;
+    testText?: (string: any) => boolean;
+};
+/**
+ * A "bundled" loader definition that can be used with `@loaders.gl/core` functions
+ * If a worker loader is supported it will also be supported.
+ */
+export declare type LoaderWithParser = Loader & {
+    testText?: (string: any) => boolean;
+    parse: Parse;
+    preload?: Preload;
+    parseSync?: ParseSync;
+    parseText?: ParseText;
+    parseTextSync?: ParseTextSync;
+    parseInBatches?: ParseInBatches;
+    parseFileInBatches?: ParseFileInBatches;
+};
+/** Options for writers */
+export declare type WriterOptions = {};
+/**
+ * A writer definition that can be used with `@loaders.gl/core` functions
+ */
+export declare type Writer = {
+    name: string;
+    id: string;
+    module: string;
+    version: string;
+    options: object;
+    deprecatedOptions?: object;
+    binary?: boolean;
+    extensions?: string[];
+    mimeTypes?: string[];
+    text?: boolean;
+    encode?: Encode;
+    encodeSync?: EncodeSync;
+    encodeInBatches?: EncodeInBatches;
+    encodeURLtoURL?: EncodeURLtoURL;
+    encodeText?: EncodeText;
+};
+export declare type LoaderContext = {
+    loaders?: Loader[] | null;
+    url?: string;
+    fetch: typeof fetch;
+    parse: (arrayBuffer: ArrayBuffer, loaders?: any, options?: LoaderOptions, context?: LoaderContext) => Promise<any>;
+    parseSync?: (arrayBuffer: ArrayBuffer, loaders?: any, options?: LoaderOptions, context?: LoaderContext) => any;
+    parseInBatches?: (iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, loaders?: any, options?: LoaderOptions, context?: LoaderContext) => AsyncIterable<any> | Promise<AsyncIterable<any>>;
+};
+declare type Parse = (arrayBuffer: ArrayBuffer, options?: LoaderOptions, context?: LoaderContext) => Promise<any>;
+declare type ParseSync = (arrayBuffer: ArrayBuffer, options?: LoaderOptions, context?: LoaderContext) => any;
+declare type ParseText = (text: string, options?: LoaderOptions) => Promise<any>;
+declare type ParseTextSync = (text: string, options?: LoaderOptions) => any;
+declare type ParseInBatches = (iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: LoaderOptions, context?: LoaderContext) => AsyncIterable<any>;
+declare type ParseFileInBatches = (file: Blob, options?: LoaderOptions, context?: LoaderContext) => AsyncIterable<any>;
+declare type Encode = (data: any, options?: WriterOptions) => Promise<ArrayBuffer>;
+declare type EncodeSync = (data: any, options?: WriterOptions) => ArrayBuffer;
+declare type EncodeText = Function;
+declare type EncodeInBatches = Function;
+declare type EncodeURLtoURL = (inputUrl: string, outputUrl: string, options?: WriterOptions) => Promise<string>;
+declare type Preload = (url: string, options?: PreloadOptions) => any;
+export declare type TransformBatches = (asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>) => AsyncIterable<ArrayBuffer>;
+/** Types that can be synchronously parsed */
+export declare type SyncDataType = string | ArrayBuffer;
+/** Types that can be parsed async */
+export declare type DataType = string | ArrayBuffer | File | Blob | Response | ReadableStream | Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>;
+/** Types that can be parsed in batches */
+export declare type BatchableDataType = DataType | Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer> | Promise<AsyncIterable<ArrayBuffer>>;
+/**
+ * A FileSystem interface can encapsulate a FileList, a ZipFile, a GoogleDrive etc.
+ */
+export interface IFileSystem {
+    /**
+     * Return a list of file names
+     * @param dirname directory name. file system root directory if omitted
+     */
+    readdir(dirname?: string, options?: {
+        recursive?: boolean;
+    }): Promise<string[]>;
+    /**
+     * Gets information from a local file from the filesystem
+     * @param filename file name to stat
+     * @param options currently unused
+     * @throws if filename is not in local filesystem
+     */
+    stat(filename: string, options?: object): Promise<{
+        size: number;
+    }>;
+    /**
+     * Fetches a local file from the filesystem (or a URL)
+     * @param filename
+     * @param options
+     */
+    fetch(filename: string, options?: object): Promise<Response>;
+}
+declare type ReadOptions = {
+    buffer?: ArrayBuffer;
+    offset?: number;
+    length?: number;
+    position?: number;
+};
+export interface IRandomAccessReadFileSystem extends IFileSystem {
+    open(path: string, flags: any, mode?: any): Promise<any>;
+    close(fd: any): Promise<void>;
+    fstat(fd: any): Promise<object>;
+    read(fd: any, options?: ReadOptions): Promise<{
+        bytesRead: number;
+        buffer: Buffer;
+    }>;
+}
+export {};

package/dist/workers/json-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/loader-utils",
-  "version": "3.1.0-alpha.4",
+  "version": "3.1.0-alpha.5",
   "description": "Framework-independent loaders for 3D graphics formats",
   "license": "MIT",
   "publishConfig": {
@@ -17,7 +17,7 @@
     "mesh",
     "point cloud"
   ],
-  "types": "src/index.ts",
+  "types": "dist/index.d.ts",
   "main": "dist/es5/index.js",
   "module": "dist/esm/index.js",
   "sideEffects": false,
@@ -44,12 +44,13 @@
   },
   "scripts": {
     "pre-build-disabled": "npm run build-bundle",
+    "post-build": "tsc",
     "build-bundle": "webpack --config ../../scripts/webpack/bundle.js"
   },
   "dependencies": {
     "@babel/runtime": "^7.3.1",
-    "@loaders.gl/worker-utils": "3.1.0-alpha.4",
+    "@loaders.gl/worker-utils": "3.1.0-alpha.5",
     "@probe.gl/stats": "^3.4.0"
   },
-  "gitHead": "e309784af37ef9f3640c7733c7851124c72e1fa3"
+  "gitHead": "352241dd910a8c6307a235dadbe154ca915b885b"
 }