@nasa-terra/components 0.0.107 → 0.0.109

package/cdn/assets/data-access/download_files.py.txt

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""
+Python script for downloading data from the NASA CMR API
+
+Requirements:
+    > pip install tqdm
+
+To run the script: `python download_data_{{short_name}}_{{version}}_{{date}}.py`
+"""
+
+import os
+import requests
+from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+CMR_BASE_URL = "https://cmr.earthdata.nasa.gov"
+
+SHORT_NAME = "{{short_name}}"
+VERSION = "{{version}}"
+FILTER_TEMPORAL = "{{filter_temporal}}"
+FILTER_BBOX = "{{filter_bbox}}"
+FILTER_SEARCH = "{{filter_search}}"
+FILTER_CLOUD_COVER_MIN = "{{filter_cloud_cover_min}}"
+FILTER_CLOUD_COVER_MAX = "{{filter_cloud_cover_max}}"
+DOWNLOAD_DIR = f"./{SHORT_NAME}_{VERSION}"
+MAX_WORKERS = 5  # Number of parallel downloads
+
+os.makedirs(DOWNLOAD_DIR, exist_ok=True)
+
+
+def query_cmr_granules(
+    short_name: str,
+    version: str,
+    page_size: int = 2000,
+    search_after: str | None = None,
+    **extra_params,
+):
+    """
+    Queries the CMR granules endpoint.
+
+    Args:
+        short_name: The short name of the collection
+        version: The version of the collection
+        page_size: The number of results per page (max is 2000)
+        search_after: The pagination token for subsequent requests (optional, see https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#search-after for more details)
+        **extra_params: Additional query parameters (e.g., temporal, bounding_box, etc.)
+
+    Returns:
+        tuple: (response object, list of granule items)
+    """
+    url = f"{CMR_BASE_URL}/search/granules.umm_json"
+
+    params = {
+        "short_name": short_name,
+        "version": version,
+        "page_size": page_size,
+        **extra_params,  # Merge any additional parameters
+    }
+
+    headers = {"Accept": "application/json"}
+    if search_after:
+        headers["CMR-Search-After"] = search_after
+
+    response = requests.get(url, params=params, headers=headers)
+
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError:
+        print("Failed to fetch granules:", response.text)
+        raise
+
+    data = response.json()
+    items = data.get("items", [])
+
+    return response, items
+
+
+def download_data_from_cmr(
+    short_name: str, version: str, total_granules: int, page_size: int = 2000, **params
+):
+    """Fetches granules for a given collection from the CMR API, then downloads the data from the "GET DATA" URLs"""
+    search_after_value: str | None = None
+    all_download_urls = []
+    granules_without_urls = 0
+
+    # First pass: collect all download URLs
+    print("Collecting download URLs...")
+    with tqdm(total=total_granules, desc="Collecting URLs", unit="granule") as pbar:
+        while True:
+            # Use shared query function
+            response, items = query_cmr_granules(
+                short_name, version, page_size, search_after_value, **params
+            )
+
+            # Collect "GET DATA" URLs
+            for item in items:
+                pbar.update(1)  # Update progress for each granule processed
+                download_urls = []
+                for related_url in item.get("umm", {}).get("RelatedUrls", []):
+                    if related_url.get("Type") == "GET DATA":
+                        download_urls.append(related_url.get("URL"))
+                
+                if download_urls:
+                    all_download_urls.extend(download_urls)
+                else:
+                    granules_without_urls += 1
+
+            # Read the next search-after value from response headers
+            search_after_value = response.headers.get("CMR-Search-After")
+
+            # There is no next search-after value, we've reached the end
+            if not search_after_value:
+                break
+
+    print(f"Found {len(all_download_urls)} files to download")
+    if granules_without_urls > 0:
+        print(f"⚠️ {granules_without_urls} granules have no download URLs")
+
+    # Second pass: download all files in parallel
+    downloaded_count = 0
+    skipped_count = 0
+    failed_count = 0
+    
+    with tqdm(total=len(all_download_urls), desc="Downloading files", unit="file") as pbar:
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            # Submit all download tasks at once
+            future_to_url = {executor.submit(download_file, url): url for url in all_download_urls}
+            
+            # Process completed downloads as they finish
+            for future in as_completed(future_to_url):
+                status, filename = future.result()
+                if status == "success":
+                    downloaded_count += 1
+                    pbar.set_description(f"Downloaded: {filename}")
+                elif status == "skipped":
+                    skipped_count += 1
+                    pbar.set_description(f"Skipped: {filename}")
+                else:  # failed
+                    failed_count += 1
+                    print(f"Failed: {filename}")
+                
+                pbar.update(1)
+
+    print(f"Downloaded: {downloaded_count} files")
+    if skipped_count > 0:
+        print(f"Skipped: {skipped_count} files (already exist)")
+    if failed_count > 0:
+        print(f"Failed: {failed_count} files")
+
+
+def download_file(url: str):
+    """Download a single file and return (success, skipped, failed) status and filename"""
+    local_filename = os.path.join(DOWNLOAD_DIR, os.path.basename(url))
+    if os.path.exists(local_filename):
+        return "skipped", os.path.basename(url)  # skip existing
+
+    try:
+        with requests.get(url, stream=True, timeout=120) as r:
+            r.raise_for_status()
+            with open(local_filename, "wb") as f:
+                for chunk in r.iter_content(8192):
+                    f.write(chunk)
+        return "success", os.path.basename(url)
+    except Exception as e:
+        print(f"⚠️ Error downloading {os.path.basename(url)}: {e}")
+        return "failed", os.path.basename(url)
+
+
+def fetch_total_granules_count_from_cmr(short_name: str, version: str, **params):
+    """Fetches the total number of granules for a given collection and filters from the CMR API"""
+    response, _ = query_cmr_granules(short_name, version, page_size=1, **params)
+    data = response.json()
+    return data.get("hits", 0)
+
+
+def main():
+    """Main function to download data from the CMR API"""
+    filter_params = {}
+
+    if FILTER_TEMPORAL:
+        filter_params["temporal"] = FILTER_TEMPORAL
+
+    if FILTER_BBOX:
+        filter_params["bounding_box"] = FILTER_BBOX
+
+    if FILTER_SEARCH:
+        filter_params["producer_granule_id[]"] = FILTER_SEARCH
+        filter_params["options[producer_granule_id][pattern]"] = 'true'
+
+    if FILTER_CLOUD_COVER_MIN and FILTER_CLOUD_COVER_MAX:
+        filter_params["cloud_cover"] = f"{FILTER_CLOUD_COVER_MIN},{FILTER_CLOUD_COVER_MAX}"
+
+    total_granules = fetch_total_granules_count_from_cmr(
+        SHORT_NAME, VERSION, **filter_params
+    )
+    print(f"Total granules: {total_granules:,}")
+
+    download_data_from_cmr(SHORT_NAME, VERSION, total_granules, **filter_params)
+
+    print("✅ All downloads complete.")
+
+
+if __name__ == "__main__":
+    main()

package/cdn/chunks/{chunk.CLNFE32U.js → chunk.3RVUTYH4.js}

@@ -1,6 +1,6 @@
 import {
   TerraMap
-} from "./chunk.Z76Z3SSD.js";
+} from "./chunk.MTWOB4SN.js";
 import {
   init_buffer,
   init_dirname,

package/cdn/chunks/{chunk.Y3BKT4GE.js → chunk.3Y6CUS3R.js}

@@ -1,6 +1,6 @@
 import {
   TerraPlotToolbar
-} from "./chunk.UGTDMW23.js";
+} from "./chunk.67ZWVFY3.js";
 import {
   init_buffer,
   init_dirname,

package/cdn/chunks/{chunk.UQPG6VRH.js → chunk.4IOEKI2P.js}

@@ -1,6 +1,6 @@
 import {
   TerraBrowseVariables
-} from "./chunk.UATCIRCQ.js";
+} from "./chunk.B6JDR25L.js";
 import {
   init_buffer,
   init_dirname,

package/cdn/chunks/chunk.4Q56BB6I.js

@@ -0,0 +1,77 @@
+import {
+  getVariableEntryId
+} from "./chunk.V6GUDCWA.js";
+import {
+  GiovanniVariableCatalog
+} from "./chunk.7A6NO75T.js";
+import {
+  h
+} from "./chunk.CTTYXXNL.js";
+import {
+  init_buffer,
+  init_dirname,
+  init_process
+} from "./chunk.HBV76IJU.js";
+
+// src/metadata-catalog/tasks.ts
+init_dirname();
+init_buffer();
+init_process();
+var variableCache = /* @__PURE__ */ new Map();
+var pendingRequests = /* @__PURE__ */ new Map();
+function setHostPropertiesFromVariable(host, variable, variableEntryId) {
+  var _a, _b, _c, _d;
+  host.startDate = (_b = host.startDate) != null ? _b : (_a = variable.exampleInitialStartDate) == null ? void 0 : _a.toISOString();
+  host.endDate = (_d = host.endDate) != null ? _d : (_c = variable.exampleInitialEndDate) == null ? void 0 : _c.toISOString();
+  host.catalogVariable = variable;
+  host.variableEntryId = variableEntryId;
+}
+function getFetchVariableTask(host, autoRun = true) {
+  const catalog = new GiovanniVariableCatalog();
+  return new h(host, {
+    task: async (_args) => {
+      const variableEntryId = getVariableEntryId(host);
+      console.debug("Fetch variable ", variableEntryId);
+      if (!variableEntryId) {
+        return;
+      }
+      if (variableCache.has(variableEntryId)) {
+        console.debug("Using cached variable ", variableEntryId);
+        const cachedVariable = variableCache.get(variableEntryId);
+        setHostPropertiesFromVariable(host, cachedVariable, variableEntryId);
+        return;
+      }
+      if (pendingRequests.has(variableEntryId)) {
+        console.debug(
+          "Waiting for pending request for variable ",
+          variableEntryId
+        );
+        const variable = await pendingRequests.get(variableEntryId);
+        if (variable) {
+          console.debug("Using cached variable ", variableEntryId);
+          setHostPropertiesFromVariable(host, variable, variableEntryId);
+        }
+        return;
+      }
+      const requestPromise = catalog.getVariable(variableEntryId);
+      pendingRequests.set(variableEntryId, requestPromise);
+      try {
+        const variable = await requestPromise;
+        console.debug("Found variable ", variable);
+        if (!variable) {
+          return;
+        }
+        variableCache.set(variableEntryId, variable);
+        setHostPropertiesFromVariable(host, variable, variableEntryId);
+      } finally {
+        pendingRequests.delete(variableEntryId);
+      }
+    },
+    args: () => [host.variableEntryId, host.collection, host.variable],
+    autoRun
+  });
+}
+
+export {
+  getFetchVariableTask
+};

package/cdn/chunks/{chunk.TTKHEE2N.js → chunk.4ZSHIBMI.js}

@@ -1,6 +1,6 @@
 import {
   CmrCatalog
-} from "./chunk.SEOPS7OZ.js";
+} from "./chunk.EOF7UX34.js";
 import {
   h
 } from "./chunk.CTTYXXNL.js";

package/cdn/chunks/{chunk.7B2K4OIZ.js → chunk.5FUN6F7Y.js}

@@ -1,6 +1,6 @@
 import {
   TerraTimeSeries
-} from "./chunk.E2Y4JI5T.js";
+} from "./chunk.RZ6JWWRE.js";
 import {
   init_buffer,
   init_dirname,

package/cdn/chunks/{chunk.RSA5EY6Z.js → chunk.5IF36BN5.js}

@@ -1,6 +1,6 @@
 import {
   c
-} from "./chunk.G4NSLCSU.js";
+} from "./chunk.AQTYPOFV.js";
 import {
   x
 } from "./chunk.JQ2FCDGS.js";