@loaders.gl/core 5.0.0-alpha.1 → 5.0.0-alpha.3

package/src/lib/api/load.ts

@@ -13,7 +13,8 @@ import type {
   LoaderReturnType,
   LoaderArrayOptionsType,
   LoaderArrayReturnType,
-  SourceLoader
+  SourceLoader,
+  LoaderWithParser
 } from '@loaders.gl/loader-utils';
 import {isBlob, isSourceLoader} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
@@ -27,6 +28,7 @@ import {parseSync} from './parse-sync';
 import {parseInBatches} from './parse-in-batches';
 import {loadInBatches} from './load-in-batches';
 import {selectLoader} from './select-loader';
+import {getLoaderImplementation} from './load-loader';
 
 /**
  * Parses `data` using a specified loader
@@ -94,6 +96,7 @@ export async function load(
   }
 
   if (!Array.isArray(resolvedLoaders) && isSourceLoader(resolvedLoaders)) {
+    const sourceLoader = await resolveSourceLoader(resolvedLoaders, url, resolvedOptions);
     const runtimeCoreApi = {
       fetchFile,
       parse,
@@ -103,7 +106,7 @@ export async function load(
       load,
       loadInBatches
     };
-    return resolvedLoaders.createDataSource(
+    return sourceLoader.createDataSource(
       url as string | Blob,
       (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
       runtimeCoreApi
@@ -115,6 +118,7 @@ export async function load(
     resolvedLoaders.length === 1 &&
     isSourceLoader(resolvedLoaders[0])
   ) {
+    const sourceLoader = await resolveSourceLoader(resolvedLoaders[0], url, resolvedOptions);
     const runtimeCoreApi = {
       fetchFile,
       parse,
@@ -124,7 +128,7 @@ export async function load(
       load,
       loadInBatches
     };
-    return resolvedLoaders[0].createDataSource(
+    return sourceLoader.createDataSource(
       url as string | Blob,
       (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
       runtimeCoreApi
@@ -138,7 +142,8 @@ export async function load(
     });
 
     if (selectedLoader && isSourceLoader(selectedLoader)) {
-      return selectedLoader.createDataSource(
+      const sourceLoader = await resolveSourceLoader(selectedLoader, url, resolvedOptions);
+      return sourceLoader.createDataSource(
         url,
         (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
         {
@@ -152,6 +157,24 @@ export async function load(
         }
       );
     }
+
+    if (selectedLoader && typeof url === 'string') {
+      const loaderImplementation = await getLoaderImplementation(
+        selectedLoader,
+        resolvedOptions,
+        url
+      );
+      const parseUrl = (
+        loaderImplementation as LoaderWithParser & {
+          parseUrl?: (
+            url: string,
+            options?: LoaderOptions,
+            context?: LoaderContext
+          ) => Promise<unknown>;
+        }
+      ).parseUrl;
+      if (parseUrl) return await parseUrl(url, resolvedOptions, context);
+    }
   }
 
   // Select fetch function
@@ -190,3 +213,20 @@ export async function load(
     ? await parse(data, resolvedLoaders, resolvedOptions, context) // loader array overload
     : await parse(data, resolvedLoaders, resolvedOptions, context); // single loader overload
 }
+
+/** Resolves a lightweight source loader into its runtime implementation when necessary. */
+async function resolveSourceLoader(
+  loader: SourceLoader,
+  url: string | DataType,
+  options?: LoaderOptions
+): Promise<SourceLoader> {
+  if (!loader.preload) {
+    return loader;
+  }
+
+  const sourceLoader = await loader.preload(typeof url === 'string' ? url : '', options);
+  if (!isSourceLoader(sourceLoader)) {
+    throw new Error(`${loader.id} source loader preload() did not return a runtime source loader`);
+  }
+  return sourceLoader;
+}

package/src/lib/api/parse.ts

@@ -21,6 +21,7 @@ import {
   canParseWithWorker,
   mergeOptions,
   isResponse,
+  isBlob,
   isSourceLoader
 } from '@loaders.gl/loader-utils';
 import {validateWorkerVersion} from '@loaders.gl/worker-utils';
@@ -160,6 +161,10 @@ async function parseWithLoader(
     return await parseWithWorker(loaderWithParser, data, options, context, parse);
   }
 
+  if (isBlob(data) && loaderWithParser.parseBlob) {
+    return await loaderWithParser.parseBlob(data, options, context);
+  }
+
   data = await getArrayBufferOrStringFromData(data, loader, options);
 
   return await parseWithLoaderImplementation(loaderWithParser, data, options, context);

package/src/lib/filesystems/browser-filesystem.ts

@@ -50,23 +50,34 @@ export class BrowserFileSystem implements FileSystem {
     }
 
     // Local fetches are served from the list of files
-    const file = this.files[path];
+    const file = this._getFile(path, true);
     if (!file) {
       return new Response(path, {status: 400, statusText: 'NOT FOUND'});
     }
 
     const headers = new Headers(options?.headers);
     const range = headers.get('Range');
-    const bytes = range && /bytes=($1)-($2)/.exec(range);
+    const bytes = range && /bytes=(\d+)-(\d+)/.exec(range);
 
     if (bytes) {
       const start = parseInt(bytes[1]);
-      const end = parseInt(bytes[2]);
+      const end = Math.min(parseInt(bytes[2]), file.size - 1);
+      if (start > end) {
+        const response = new Response(null, {
+          status: 416,
+          headers: {'Content-Range': `bytes */${file.size}`}
+        });
+        Object.defineProperty(response, 'url', {value: path});
+        return response;
+      }
       // The trick when reading File objects is to read successive "slices" of the File
       // Per spec https://w3c.github.io/FileAPI/, slicing a File should only update the start and end fields
       // Actually reading from file should happen in `readAsArrayBuffer` (and as far we can tell it does)
-      const data = await file.slice(start, end).arrayBuffer();
-      const response = new Response(data);
+      const data = await file.slice(start, end + 1).arrayBuffer();
+      const response = new Response(data, {
+        status: 206,
+        headers: {'Content-Range': `bytes ${start}-${end}/${file.size}`}
+      });
       Object.defineProperty(response, 'url', {value: path});
       return response;
     }
@@ -95,7 +106,7 @@ export class BrowserFileSystem implements FileSystem {
    * Return information (size) about files in this file system
    */
   async stat(path: string, options?: object): Promise<{size: number}> {
-    const file = this.files[path];
+    const file = this._getFile(path, false);
     if (!file) {
       throw new Error(path);
     }
@@ -106,16 +117,19 @@ export class BrowserFileSystem implements FileSystem {
    * Just removes the file from the list
    */
   async unlink(path: string): Promise<void> {
-    delete this.files[path];
-    delete this.lowerCaseFiles[path];
-    this.usedFiles[path] = true;
+    const file = this._getFile(path, false);
+    if (file) {
+      delete this.files[file.name];
+      delete this.lowerCaseFiles[file.name.toLowerCase()];
+      this.usedFiles[file.name] = true;
+    }
   }
 
   // implements IRandomAccessFileSystem
 
   // RANDOM ACCESS
   async openReadableFile(pathname: string, flags: unknown): Promise<ReadableFile> {
-    return new BlobFile(this.files[pathname]);
+    return new BlobFile(this._getFile(pathname, true));
   }
 
   // PRIVATE
@@ -123,7 +137,7 @@ export class BrowserFileSystem implements FileSystem {
   // Supports case independent paths, and file usage tracking
   _getFile(path: string, used: boolean): File {
     // Prefer case match, but fall back to case independent.
-    const file = this.files[path] || this.lowerCaseFiles[path];
+    const file = this.files[path] || this.lowerCaseFiles[path.toLowerCase()];
     if (file && used) {
       this.usedFiles[path] = true;
     }

package/src/lib/loader-utils/get-fetch-function.ts

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 
 import type {LoaderContext, LoaderOptions, FetchLike} from '@loaders.gl/loader-utils';
-import {isObject} from '@loaders.gl/loader-utils';
+import {createAuthenticatedFetch, isObject} from '@loaders.gl/loader-utils';
 import {fetchFile} from '../fetch/fetch-file';
 import {getGlobalLoaderOptions} from './option-utils';
 
@@ -21,22 +21,33 @@ export function getFetchFunction(
 
   const loaderOptions = options || globalOptions;
   const fetchOption = loaderOptions.fetch ?? loaderOptions.core?.fetch;
+  let fetchFunction: FetchLike;
 
   // options.fetch can be a function
   if (typeof fetchOption === 'function') {
-    return fetchOption;
+    fetchFunction = fetchOption;
+  } else if (isObject(fetchOption)) {
+    fetchFunction = (url, requestOptions) =>
+      fetchFile(url, mergeFetchOptions(fetchOption as RequestInit, requestOptions));
+  } else if (context?.fetch) {
+    fetchFunction = context.fetch;
+  } else {
+    fetchFunction = fetchFile;
   }
 
-  // options.fetch can be an options object
-  if (isObject(fetchOption)) {
-    return url => fetchFile(url, fetchOption as RequestInit);
-  }
+  return createAuthenticatedFetch({
+    fetch: fetchFunction,
+    credentials: loaderOptions.core?.credentials || []
+  });
+}
 
-  // else refer to context (from parent loader) if available
-  if (context?.fetch) {
-    return context?.fetch;
+/** Combines static and per-request fetch options without replacing either header collection. */
+function mergeFetchOptions(defaultOptions: RequestInit, requestOptions?: RequestInit): RequestInit {
+  const options = {...defaultOptions, ...requestOptions};
+  if (defaultOptions.headers || requestOptions?.headers) {
+    const headers = new Headers(defaultOptions.headers);
+    new Headers(requestOptions?.headers).forEach((value, key) => headers.set(key, value));
+    options.headers = headers;
   }
-
-  // else return the default fetch function
-  return fetchFile;
+  return options;
 }