eoapi-cdk 8.1.1 → 8.2.1

package/lib/stac-item-loader/runtime/pyproject.toml

@@ -0,0 +1,17 @@
+[project]
+name = "stac-item-loader"
+version = "0.1.0"
+description = "An application for loading STAC items into a pgstac database"
+authors = [
+    { name = "hrodmn", email = "henry@developmentseed.org" }
+]
+requires-python = ">=3.11"
+dependencies = [
+    "boto3",
+    "pypgstac[psycopg]",
+    "stac-pydantic>=3.2.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

package/lib/stac-item-loader/runtime/src/stac_item_loader/handler.py

@@ -0,0 +1,241 @@
+import base64
+import json
+import logging
+import os
+from collections import defaultdict
+from typing import (
+    TYPE_CHECKING,
+    Annotated,
+    Any,
+    DefaultDict,
+    Dict,
+    List,
+    Optional,
+    TypedDict,
+)
+
+import boto3.session
+from pydantic import ValidationError
+from pypgstac.db import PgstacDB
+from pypgstac.load import Loader, Methods
+from stac_pydantic.item import Item
+
+if TYPE_CHECKING:
+    from aws_lambda_typing.context import Context
+else:
+    Context = Annotated[object, "Context object"]
+
+logger = logging.getLogger()
+if logger.hasHandlers():
+    logger.handlers.clear()
+
+log_handler = logging.StreamHandler()  # <--- Renamed handler variable
+
+log_level_name = os.environ.get("LOG_LEVEL", "INFO").upper()
+log_level = logging._nameToLevel.get(log_level_name, logging.INFO)
+logger.setLevel(log_level)
+
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+log_handler.setFormatter(formatter)
+logger.addHandler(log_handler)
+
+botocore_logger = logging.getLogger("botocore")
+botocore_logger.setLevel(logging.WARN)
+
+
+class BatchItemFailure(TypedDict):
+    itemIdentifier: str
+
+
+class PartialBatchFailureResponse(TypedDict):
+    batchItemFailures: List[BatchItemFailure]
+
+
+def get_secret_dict(secret_name: str) -> Dict:
+    """Retrieve secrets from AWS Secrets Manager
+
+    Args:
+        secret_name (str): name of aws secrets manager secret containing database connection secrets
+        profile_name (str, optional): optional name of aws profile for use in debugger only
+
+    Returns:
+        secrets (dict): decrypted secrets in dict
+    """
+
+    # Create a Secrets Manager client
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager")
+
+    get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+
+    if "SecretString" in get_secret_value_response:
+        return json.loads(get_secret_value_response["SecretString"])
+    else:
+        return json.loads(base64.b64decode(get_secret_value_response["SecretBinary"]))
+
+
+def get_pgstac_dsn() -> str:
+    secret_arn = os.getenv("PGSTAC_SECRET_ARN")
+    if not secret_arn:
+        logger.error("Environment variable PGSTAC_SECRET_ARN is not set.")
+        raise EnvironmentError("PGSTAC_SECRET_ARN must be set")
+
+    secret_dict = get_secret_dict(secret_name=secret_arn)
+
+    return f"postgres://{secret_dict['username']}:{secret_dict['password']}@{secret_dict['host']}:{secret_dict['port']}/{secret_dict['dbname']}"
+
+
+def is_s3_event(message_str: str) -> bool:
+    """Check if the event data is an S3 event notification."""
+    return "aws:s3" in message_str
+
+
+def get_stac_item_from_s3(bucket_name: str, object_key: str) -> Dict[str, Any]:
+    """Fetch STAC item JSON from S3."""
+    session = boto3.session.Session()
+    s3_client = session.client("s3")
+
+    try:
+        logger.debug(f"Fetching STAC item from s3://{bucket_name}/{object_key}")
+        response = s3_client.get_object(Bucket=bucket_name, Key=object_key)
+        content = response["Body"].read()
+
+        try:
+            stac_item_json = content.decode("utf-8")
+        except UnicodeDecodeError as e:
+            logger.error(
+                f"Failed to decode S3 object as UTF-8: s3://{bucket_name}/{object_key}"
+            )
+            raise ValueError("S3 object is not valid UTF-8 text") from e
+
+        stac_item_data = json.loads(stac_item_json)
+        logger.debug(
+            f"Successfully parsed STAC item from S3: {stac_item_data.get('id', 'unknown')}"
+        )
+
+        return stac_item_data
+
+    except Exception as e:
+        logger.error(
+            f"Failed to fetch STAC item from s3://{bucket_name}/{object_key}: {e}"
+        )
+        raise
+
+
+def process_s3_event(message_str: str) -> Dict[str, Any]:
+    """Process an S3 event notification and return STAC item data."""
+    try:
+        message_data = json.loads(message_str)
+        records: List[Dict[str, Any]] = message_data.get("Records", [])
+        if not records:
+            raise ValueError("no S3 event records!")
+        elif len(records) > 1:
+            raise ValueError("more than one S3 event record!")
+
+        s3_data = records[0]["s3"]
+        bucket_name = s3_data["bucket"]["name"]
+        object_key = s3_data["object"]["key"]
+
+        # Validate that this looks like a STAC item file
+        if not object_key.endswith((".json", ".geojson")):
+            raise ValueError(
+                f"S3 object key does not appear to be a STAC item: {object_key}"
+            )
+
+        stac_item_data = get_stac_item_from_s3(bucket_name, object_key)
+
+        return stac_item_data
+
+    except KeyError as e:
+        logger.error(f"S3 event missing required field: {e}")
+        raise ValueError(f"Invalid S3 event structure: missing {e}") from e
+    except Exception as e:
+        logger.error(f"Failed to process S3 event: {e}")
+        raise
+
+
+def handler(
+    event: Dict[str, Any], context: Context
+) -> Optional[PartialBatchFailureResponse]:
+    records = event.get("Records", [])
+    aws_request_id = getattr(context, "aws_request_id", "N/A")
+    remaining_time = getattr(context, "get_remaining_time_in_millis", lambda: "N/A")()
+
+    logger.info(f"Received batch with {len(records)} records.")
+    logger.debug(
+        f"Lambda Context: RequestId={aws_request_id}, RemainingTime={remaining_time}ms"
+    )
+    pgstac_dsn = get_pgstac_dsn()
+
+    batch_item_failures: List[BatchItemFailure] = []
+
+    items_by_collection: DefaultDict[str, List[Dict[str, Any]]] = defaultdict(list)
+    message_ids_by_collection: DefaultDict[str, List[str]] = defaultdict(list)
+
+    for record in records:
+        message_id = record.get("messageId")
+        if not message_id:
+            logger.warning("Record missing messageId, cannot report failure for it.")
+            continue
+
+        try:
+            sqs_body_str = record["body"]
+            logger.debug(f"[{message_id}] SQS message body: {sqs_body_str}")
+            sns_notification = json.loads(sqs_body_str)
+
+            message_str = sns_notification["Message"]
+            logger.debug(f"[{message_id}] SNS Message content: {message_str}")
+
+            if is_s3_event(message_str):
+                logger.debug(f"[{message_id}] Processing S3 event notification")
+                message_data = process_s3_event(message_str)
+            else:
+                message_data = json.loads(message_str)
+
+            item = Item(**message_data)
+
+            if not item.collection:
+                raise KeyError(f"item {item.id} is missing a collection id!")
+
+            items_by_collection[item.collection].append(item.model_dump(mode="json"))
+            message_ids_by_collection[item.collection].append(message_id)
+            logger.debug(f"[{message_id}] Successfully processed.")
+
+        except (ValueError, KeyError, ValidationError, json.JSONDecodeError) as e:
+            logger.error(f"[{message_id}] Failed with error: {e}", extra=record)
+            batch_item_failures.append({"itemIdentifier": message_id})
+        except Exception as e:
+            logger.error(f"[{message_id}] Unexpected error: {e}", extra=record)
+            batch_item_failures.append({"itemIdentifier": message_id})
+
+    for collection_id, items in items_by_collection.items():
+        try:
+            with PgstacDB(dsn=pgstac_dsn) as db:
+                loader = Loader(db=db)
+                logger.info(f"[{collection_id}] loading items into database.")
+                loader.load_items(
+                    file=items,  # type: ignore
+                    insert_mode=Methods.upsert,
+                )
+                logger.info(f"[{collection_id}] successfully loaded {len(items)} items.")
+        except Exception as e:
+            logger.error(f"[{collection_id}] failed to load items: {str(e)}")
+
+            batch_item_failures.extend(
+                [
+                    {"itemIdentifier": message_id}
+                    for message_id in message_ids_by_collection[collection_id]
+                ]
+            )
+
+    if batch_item_failures:
+        logger.warning(
+            f"Finished processing batch. {len(batch_item_failures)} failure(s) reported."
+        )
+        logger.info(
+            f"Returning failed item identifiers: {[f['itemIdentifier'] for f in batch_item_failures]}"
+        )
+        return {"batchItemFailures": batch_item_failures}
+    else:
+        logger.info("Finished processing batch. All records successful.")
+        return None

package/lib/stactools-item-generator/index.d.ts

@@ -0,0 +1,258 @@
+import { aws_ec2 as ec2, aws_lambda as lambda, aws_sqs as sqs, aws_sns as sns } from "aws-cdk-lib";
+import { Construct } from "constructs";
+import { CustomLambdaFunctionProps } from "../utils";
+/**
+ * Configuration properties for the StactoolsItemGenerator construct.
+ *
+ * The StactoolsItemGenerator is part of a two-phase serverless STAC ingestion pipeline
+ * that generates STAC items from source data. This construct creates the
+ * infrastructure for the first phase of the pipeline - processing metadata
+ * about assets and transforming them into standardized STAC items.
+ *
+ * @example
+ * const generator = new StactoolsItemGenerator(this, 'ItemGenerator', {
+ *   itemLoadTopicArn: loader.topic.topicArn,
+ *   lambdaTimeoutSeconds: 120,
+ *   maxConcurrency: 100,
+ *   batchSize: 10
+ * });
+ */
+export interface StactoolsItemGeneratorProps {
+    /**
+     * The lambda runtime to use for the item generation function.
+     *
+     * The function is containerized using Docker and can accommodate various
+     * stactools packages. The runtime version should be compatible with the
+     * packages you plan to use for STAC item generation.
+     *
+     * @default lambda.Runtime.PYTHON_3_11
+     */
+    readonly lambdaRuntime?: lambda.Runtime;
+    /**
+     * VPC into which the lambda should be deployed.
+     */
+    readonly vpc?: ec2.IVpc;
+    /**
+     * Subnet into which the lambda should be deployed.
+     */
+    readonly subnetSelection?: ec2.SubnetSelection;
+    /**
+     * The timeout for the item generation lambda in seconds.
+     *
+     * This should accommodate the time needed to:
+     * - Install stactools packages using uvx
+     * - Download and process source data
+     * - Generate STAC metadata
+     * - Publish results to SNS
+     *
+     * The SQS visibility timeout will be set to this value plus 10 seconds.
+     *
+     * @default 120
+     */
+    readonly lambdaTimeoutSeconds?: number;
+    /**
+     * Memory size for the lambda function in MB.
+     *
+     * Higher memory allocation may be needed for processing large geospatial
+     * datasets or when stactools packages have high memory requirements.
+     * More memory also provides proportionally more CPU power.
+     *
+     * @default 1024
+     */
+    readonly memorySize?: number;
+    /**
+     * Maximum number of concurrent executions.
+     *
+     * This controls how many item generation tasks can run simultaneously.
+     * Higher concurrency enables faster processing of large batches but
+     * may strain downstream systems or external data sources.
+     *
+     * @default 100
+     */
+    readonly maxConcurrency?: number;
+    /**
+     * SQS batch size for lambda event source.
+     *
+     * This determines how many generation requests are processed together
+     * in a single lambda invocation. Unlike the loader, generation typically
+     * processes items individually, so smaller batch sizes are common.
+     *
+     * @default 10
+     */
+    readonly batchSize?: number;
+    /**
+     * Additional environment variables for the lambda function.
+     *
+     * These will be merged with default environment variables including
+     * ITEM_LOAD_TOPIC_ARN and LOG_LEVEL. Use this for custom configuration
+     * or to pass credentials for external data sources.
+     */
+    readonly environment?: {
+        [key: string]: string;
+    };
+    /**
+     * ARN of the SNS topic to publish generated items to.
+     *
+     * This is typically the topic from a StacItemLoader construct.
+     * Generated STAC items will be published here for downstream
+     * processing and database insertion.
+     */
+    readonly itemLoadTopicArn: string;
+    /**
+     * Can be used to override the default lambda function properties.
+     *
+     * @default - defined in the construct.
+     */
+    readonly lambdaFunctionOptions?: CustomLambdaFunctionProps;
+}
+/**
+ * AWS CDK Construct for STAC Item Generation Infrastructure
+ *
+ * The StactoolsItemGenerator creates a serverless, event-driven system for generating
+ * STAC (SpatioTemporal Asset Catalog) items from source data. This construct
+ * implements the first phase of a two-stage ingestion pipeline that transforms
+ * raw geospatial data into standardized STAC metadata.
+ *
+ * ## Architecture Overview
+ *
+ * This construct creates the following AWS resources:
+ * - **SNS Topic**: Entry point for triggering item generation workflows
+ * - **SQS Queue**: Buffers generation requests (120-second visibility timeout)
+ * - **Dead Letter Queue**: Captures failed messages after 5 processing attempts
+ * - **Lambda Function**: Containerized function that generates STAC items using stactools
+ *
+ * ## Data Flow
+ *
+ * 1. External systems publish ItemRequest messages to the SNS topic with metadata about assets
+ * 2. The SQS queue buffers these messages and triggers the Lambda function
+ * 3. The Lambda function:
+ *    - Uses `uvx` to install the required stactools package
+ *    - Executes the `create-item` CLI command with provided arguments
+ *    - Publishes generated STAC items to the ItemLoad topic
+ * 4. Failed processing attempts are sent to the dead letter queue
+ *
+ * ## Operational Characteristics
+ *
+ * - **Scalability**: Lambda scales automatically based on queue depth (up to maxConcurrency)
+ * - **Flexibility**: Supports any stactools package through dynamic installation
+ * - **Reliability**: Dead letter queue captures failed generation attempts
+ * - **Isolation**: Each generation task runs in a fresh container environment
+ * - **Observability**: CloudWatch logs retained for one week
+ *
+ * ## Message Schema
+ *
+ * The function expects messages matching the ItemRequest model:
+ *
+ * ```json
+ * {
+ *   "package_name": "stactools-glad-global-forest-change",
+ *   "group_name": "gladglobalforestchange",
+ *   "create_item_args": [
+ *     "https://example.com/data.tif"
+ *   ],
+ *   "collection_id": "glad-global-forest-change-1.11"
+ * }
+ * ```
+ *
+ * ## Usage Example
+ *
+ * ```typescript
+ * // Create item loader first (or get existing topic ARN)
+ * const loader = new StacItemLoader(this, 'ItemLoader', {
+ *   pgstacDb: database
+ * });
+ *
+ * // Create item generator that feeds the loader
+ * const generator = new StactoolsItemGenerator(this, 'ItemGenerator', {
+ *   itemLoadTopicArn: loader.topic.topicArn,
+ *   lambdaTimeoutSeconds: 120,    // Allow time for package installation
+ *   maxConcurrency: 100,          // Control parallel processing
+ *   batchSize: 10                 // Process 10 requests per invocation
+ * });
+ *
+ * // Grant permission to publish to the loader topic
+ * loader.topic.grantPublish(generator.lambdaFunction);
+ * ```
+ *
+ * ## Publishing Generation Requests
+ *
+ * Send messages to the generator topic to trigger item creation:
+ *
+ * ```bash
+ * aws sns publish --topic-arn $ITEM_GEN_TOPIC --message '{
+ *   "package_name": "stactools-glad-global-forest-change",
+ *   "group_name": "gladglobalforestchange",
+ *   "create_item_args": [
+ *     "https://storage.googleapis.com/earthenginepartners-hansen/GFC-2023-v1.11/Hansen_GFC-2023-v1.11_gain_40N_080W.tif"
+ *   ],
+ *   "collection_id": "glad-global-forest-change-1.11"
+ * }'
+ * ```
+ *
+ * ## Batch Processing Example
+ *
+ * For processing many assets, you can loop through URLs:
+ *
+ * ```bash
+ * while IFS= read -r url; do
+ *   aws sns publish --topic-arn "$ITEM_GEN_TOPIC" --message "{
+ *     \"package_name\": \"stactools-glad-glclu2020\",
+ *     \"group_name\": \"gladglclu2020\",
+ *     \"create_item_args\": [\"$url\"]
+ *   }"
+ * done < urls.txt
+ * ```
+ *
+ * ## Monitoring and Troubleshooting
+ *
+ * - Monitor Lambda logs: `/aws/lambda/{FunctionName}`
+ * - Check dead letter queue for failed generation attempts
+ * - Use CloudWatch metrics to track processing rates and errors
+ * - Failed items can be replayed from the dead letter queue
+ *
+ * ## Supported Stactools Packages
+ *
+ * Any package available on PyPI that follows the stactools plugin pattern
+ * can be used. Examples include:
+ * - `stactools-glad-global-forest-change`
+ * - `stactools-glad-glclu2020`
+ * - `stactools-landsat`
+ * - `stactools-sentinel2`
+ *
+ * @see {@link https://github.com/stactools-packages} for available stactools packages
+ * @see {@link https://stactools.readthedocs.io/} for stactools documentation
+ */
+export declare class StactoolsItemGenerator extends Construct {
+    /**
+     * The SQS queue that buffers item generation requests.
+     *
+     * This queue receives messages from the SNS topic containing ItemRequest
+     * payloads. It's configured with a visibility timeout that matches the
+     * Lambda timeout plus buffer time to prevent duplicate processing.
+     */
+    readonly queue: sqs.Queue;
+    /**
+     * Dead letter queue for failed item generation attempts.
+     *
+     * Messages that fail processing after 5 attempts are sent here for
+     * inspection and potential replay. This helps with debugging stactools
+     * package issues, network failures, or malformed requests.
+     */
+    readonly deadLetterQueue: sqs.Queue;
+    /**
+     * The SNS topic that receives item generation requests.
+     *
+     * External systems publish ItemRequest messages to this topic to trigger
+     * STAC item generation. The topic fans out to the SQS queue for processing.
+     */
+    readonly topic: sns.Topic;
+    /**
+     * The containerized Lambda function that generates STAC items.
+     *
+     * This Docker-based function dynamically installs stactools packages
+     * using uvx, processes source data, and publishes generated STAC items
+     * to the configured ItemLoad SNS topic.
+     */
+    readonly lambdaFunction: lambda.DockerImageFunction;
+    constructor(scope: Construct, id: string, props: StactoolsItemGeneratorProps);
+}