@loaders.gl/core 3.4.0-alpha.3 → 3.4.0-alpha.5

package/src/lib/api/parse-in-batches.ts

@@ -11,7 +11,7 @@ import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getLoaderContext} from '../loader-utils/loader-context';
 import {getAsyncIterableFromData} from '../loader-utils/get-data';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 import {selectLoader} from './select-loader';
 
 // Ensure `parse` is available in context if loader falls back to `parse`
@@ -32,6 +32,8 @@ export async function parseInBatches(
 ): Promise<AsyncIterable<any>> {
   assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url
 
+  const loaderArray = Array.isArray(loaders) ? loaders : undefined;
+
   // Signature: parseInBatches(data, options, url) - Uses registered loaders
   if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
     context = undefined; // context not supported in short signature
@@ -43,11 +45,11 @@ export async function parseInBatches(
   options = options || {};
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   // Chooses a loader and normalizes it
   // Note - only uses URL and contentType for streams and iterator inputs
-  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader[], options);
+  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader | Loader[], options);
   // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
   if (!loader) {
     // @ts-ignore
@@ -55,14 +57,11 @@ export async function parseInBatches(
   }
 
   // Normalize options
-  // @ts-ignore
-  options = normalizeOptions(options, loader, loaders, url);
-  // @ts-ignore
+  options = normalizeOptions(options, loader, loaderArray, url);
   context = getLoaderContext(
-    // @ts-ignore
-    {url, parseInBatches, parse, loaders: loaders as Loader[]},
+    {url, parseInBatches, parse, loaders: loaderArray},
     options,
-    context
+    context || null
   );
 
   return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);

package/src/lib/api/parse-sync.ts

@@ -11,7 +11,7 @@ import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 
 /**
  * Parses `data` synchronously using a specified loader
@@ -52,12 +52,16 @@ export function parseSync(
   options = normalizeOptions(options, loader, candidateLoaders);
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   const parse = () => {
-    throw new Error('parseSync called parse');
+    throw new Error('parseSync called parse (which is async');
   };
-  context = getLoaderContext({url, parseSync, parse, loaders: loaders as Loader[]}, options);
+  context = getLoaderContext(
+    {url, parseSync, parse, loaders: loaders as Loader[]},
+    options,
+    context || null
+  );
 
   return parseWithLoaderSync(loader as LoaderWithParser, data, options, context);
 }

package/src/lib/api/parse.ts

@@ -6,7 +6,7 @@ import {isResponse} from '../../javascript-utils/is-type';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 import {selectLoader} from './select-loader';
 
 /**
@@ -36,7 +36,7 @@ export async function parse(
   options = options || {};
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   // Chooses a loader (and normalizes it)
   // Also use any loaders in the context, new loaders take priority
@@ -53,7 +53,7 @@ export async function parse(
   options = normalizeOptions(options, loader, candidateLoaders, url);
 
   // Get a context (if already present, will be unchanged)
-  context = getLoaderContext({url, parse, loaders: candidateLoaders}, options, context);
+  context = getLoaderContext({url, parse, loaders: candidateLoaders}, options, context || null);
 
   return await parseWithLoader(loader, data, options, context);
 }

package/src/lib/api/select-loader.ts

@@ -2,9 +2,10 @@ import type {LoaderContext, LoaderOptions, Loader} from '@loaders.gl/loader-util
 import {compareArrayBuffers, path} from '@loaders.gl/loader-utils';
 import {normalizeLoader} from '../loader-utils/normalize-loader';
 import {log} from '../utils/log';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl, getResourceMIMEType} from '../utils/resource-utils';
 import {getRegisteredLoaders} from './register-loaders';
 import {isBlob} from '../../javascript-utils/is-type';
+import {stripQueryString} from '../utils/url-utils';
 
 const EXT_PATTERN = /\.([^.]+)$/;
 
@@ -111,9 +112,10 @@ function selectLoaderInternal(
   options?: LoaderOptions,
   context?: LoaderContext
 ) {
-  const {url, type} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
+  const type = getResourceMIMEType(data);
 
-  const testUrl = url || context?.url;
+  const testUrl = stripQueryString(url) || context?.url;
 
   let loader: Loader | null = null;
   let reason: string = '';
@@ -161,7 +163,8 @@ function validHTTPResponse(data: any): boolean {
 
 /** Generate a helpful message to help explain why loader selection failed. */
 function getNoValidLoaderMessage(data): string {
-  const {url, type} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
+  const type = getResourceMIMEType(data);
 
   let message = 'No valid loader found (';
   message += url ? `${path.filename(url)}, ` : 'no url provided, ';

package/src/lib/loader-utils/loader-context.ts

@@ -1,5 +1,7 @@
 import type {Loader, LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
 import {getFetchFunction} from './get-fetch-function';
+import {extractQueryString, stripQueryString} from '../utils/url-utils';
+import {path} from '@loaders.gl/loader-utils';
 
 /**
  * "sub" loaders invoked by other loaders get a "context" injected on `this`
@@ -12,26 +14,35 @@ import {getFetchFunction} from './get-fetch-function';
  */
 export function getLoaderContext(
   context: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>,
-  options?: LoaderOptions,
-  previousContext: LoaderContext | null = null
+  options: LoaderOptions,
+  parentContext: LoaderContext | null
 ): LoaderContext {
   // For recursive calls, we already have a context
   // TODO - add any additional loaders to context?
-  if (previousContext) {
-    return previousContext;
+  if (parentContext) {
+    return parentContext;
   }
 
-  const resolvedContext: LoaderContext = {
+  const newContext: LoaderContext = {
     fetch: getFetchFunction(options, context),
     ...context
   };
 
+  // Parse URLs so that subloaders can easily generate correct strings
+  if (newContext.url) {
+    const baseUrl = stripQueryString(newContext.url);
+    newContext.baseUrl = baseUrl;
+    newContext.queryString = extractQueryString(newContext.url);
+    newContext.filename = path.filename(baseUrl);
+    newContext.baseUrl = path.dirname(baseUrl);
+  }
+
   // Recursive loading does not use single loader
-  if (!Array.isArray(resolvedContext.loaders)) {
-    resolvedContext.loaders = null;
+  if (!Array.isArray(newContext.loaders)) {
+    newContext.loaders = null;
   }
 
-  return resolvedContext;
+  return newContext;
 }
 
 // eslint-disable-next-line complexity

package/src/lib/utils/resource-utils.ts

@@ -1,51 +1,73 @@
+// loaders.gl, MIT license
+
 import {isResponse, isBlob} from '../../javascript-utils/is-type';
 import {parseMIMEType, parseMIMETypeFromURL} from './mime-type-utils';
+import {stripQueryString} from './url-utils';
 
-const QUERY_STRING_PATTERN = /\?.*/;
+/**
+ * A loadable resource. Includes:
+ * `Response`, `Blob` (`File` is a subclass), string URLs and data URLs
+ */
+export type Resource = Response | Blob | string;
 
 /**
- * Returns an object with `url` and (MIME) `type` fields
- * If it cannot determine url or type, the corresponding value will be an empty string
+ * Returns the URL associated with this resource.
+ * The returned value may include a query string and need further processing.
+ * If it cannot determine url, the corresponding value will be an empty string
  *
- * @param resource Any type, but only Responses, string URLs and data URLs are processed
+ * @todo string parameters are assumed to be URLs
+ */
+export function getResourceUrl(resource: unknown): string {
+  // If resource is a `Response`, it contains the information directly as a field
+  if (isResponse(resource)) {
+    const response = resource as Response;
+    return response.url;
+  }
+
+  // If the resource is a Blob or a File (subclass of Blob)
+  if (isBlob(resource)) {
+    const blob = resource as Blob;
+    // File objects have a "name" property. Blob objects don't have any
+    // url (name) information
+    return blob.name || '';
+  }
+
+  if (typeof resource === 'string') {
+    return resource;
+  }
+
+  // Unknown
+  return '';
+}
+
+/**
+ * Returns the URL associated with this resource.
+ * The returned value may include a query string and need further processing.
+ * If it cannot determine url, the corresponding value will be an empty string
  *
  * @todo string parameters are assumed to be URLs
  */
-export function getResourceUrlAndType(resource: any): {url: string; type: string} {
+export function getResourceMIMEType(resource: unknown): string {
   // If resource is a response, it contains the information directly
   if (isResponse(resource)) {
-    const url = stripQueryString(resource.url || '');
-    const contentTypeHeader = resource.headers.get('content-type') || '';
-    return {
-      url,
-      type: parseMIMEType(contentTypeHeader) || parseMIMETypeFromURL(url)
-    };
+    const response = resource as Response;
+    const contentTypeHeader = response.headers.get('content-type') || '';
+    const noQueryUrl = stripQueryString(response.url);
+    return parseMIMEType(contentTypeHeader) || parseMIMETypeFromURL(noQueryUrl);
   }
 
   // If the resource is a Blob or a File (subclass of Blob)
   if (isBlob(resource)) {
-    return {
-      // File objects have a "name" property. Blob objects don't have any
-      // url (name) information
-      url: stripQueryString(resource.name || ''),
-      type: resource.type || ''
-    };
+    const blob = resource as Blob;
+    return blob.type || '';
   }
 
   if (typeof resource === 'string') {
-    return {
-      // TODO this could mess up data URL but it doesn't matter as it is just used for inference
-      url: stripQueryString(resource),
-      // If a data url
-      type: parseMIMETypeFromURL(resource)
-    };
+    return parseMIMETypeFromURL(resource);
   }
 
   // Unknown
-  return {
-    url: '',
-    type: ''
-  };
+  return '';
 }
 
 /**
@@ -55,12 +77,14 @@ export function getResourceUrlAndType(resource: any): {url: string; type: string
 
   * @note string parameters are NOT assumed to be URLs
   */
-export function getResourceContentLength(resource: any): number {
+export function getResourceContentLength(resource: unknown): number {
   if (isResponse(resource)) {
-    return resource.headers['content-length'] || -1;
+    const response = resource as Response;
+    return response.headers['content-length'] || -1;
   }
   if (isBlob(resource)) {
-    return resource.size;
+    const blob = resource as Blob;
+    return blob.size;
   }
   if (typeof resource === 'string') {
     // TODO - handle data URL?
@@ -74,7 +98,3 @@ export function getResourceContentLength(resource: any): number {
   }
   return -1;
 }
-
-function stripQueryString(url) {
-  return url.replace(QUERY_STRING_PATTERN, '');
-}

package/src/lib/utils/response-utils.ts

@@ -1,5 +1,5 @@
 import {isResponse} from '../../javascript-utils/is-type';
-import {getResourceContentLength, getResourceUrlAndType} from './resource-utils';
+import {getResourceContentLength, getResourceUrl, getResourceMIMEType} from './resource-utils';
 
 /**
  * Returns a Response object
@@ -22,7 +22,8 @@ export async function makeResponse(resource: any): Promise<Response> {
 
   // `new Response(File)` does not preserve content-type and URL
   // so we add them here
-  const {url, type} = getResourceUrlAndType(resource);
+  const url = getResourceUrl(resource);
+  const type = getResourceMIMEType(resource);
   if (type) {
     headers['content-type'] = type;
   }

package/src/lib/utils/url-utils.ts

@@ -0,0 +1,12 @@
+// loaders.gl, MIT license
+
+const QUERY_STRING_PATTERN = /\?.*/;
+
+export function extractQueryString(url): string {
+  const matches = url.match(QUERY_STRING_PATTERN);
+  return matches && matches[0];
+}
+
+export function stripQueryString(url): string {
+  return url.replace(QUERY_STRING_PATTERN, '');
+}