eoapi-cdk 5.0.0

package/lib/ingestor-api/runtime/src/schemas.py

@@ -0,0 +1,148 @@
+import base64
+import binascii
+import enum
+import json
+from datetime import datetime
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from urllib.parse import urlparse
+
+from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
+from pydantic import (
+    BaseModel,
+    Json,
+    PositiveInt,
+    dataclasses,
+    error_wrappers,
+    validator,
+)
+from stac_pydantic import Collection, Item, shared
+
+from . import validators
+
+if TYPE_CHECKING:
+    from . import services
+
+
+class AccessibleAsset(shared.Asset):
+    @validator("href")
+    def is_accessible(cls, href):
+        url = urlparse(href)
+
+        if url.scheme in ["https", "http"]:
+            validators.url_is_accessible(href=href)
+        elif url.scheme in ["s3"]:
+            validators.s3_object_is_accessible(
+                bucket=url.hostname, key=url.path.lstrip("/")
+            )
+        else:
+            ValueError(f"Unsupported scheme: {url.scheme}")
+
+        return href
+
+
+class AccessibleItem(Item):
+    assets: Dict[str, AccessibleAsset]
+
+    @validator("collection")
+    def exists(cls, collection):
+        validators.collection_exists(collection_id=collection)
+        return collection
+
+
+class StacCollection(Collection):
+    id: str
+    item_assets: Dict
+
+
+class Status(str, enum.Enum):
+    started = "started"
+    queued = "queued"
+    failed = "failed"
+    succeeded = "succeeded"
+    cancelled = "cancelled"
+
+
+class Ingestion(BaseModel):
+    id: str
+    status: Status
+    message: Optional[str]
+    created_by: str
+    created_at: datetime = None
+    updated_at: datetime = None
+
+    item: Union[Item, Json[Item]]
+
+    @validator("created_at", pre=True, always=True, allow_reuse=True)
+    @validator("updated_at", pre=True, always=True, allow_reuse=True)
+    def set_ts_now(cls, v):
+        return v or datetime.now()
+
+    def enqueue(self, db: "services.Database"):
+        self.status = Status.queued
+        return self.save(db)
+
+    def cancel(self, db: "services.Database"):
+        self.status = Status.cancelled
+        return self.save(db)
+
+    def save(self, db: "services.Database"):
+        self.updated_at = datetime.now()
+        db.write(self)
+        return self
+
+    def dynamodb_dict(self):
+        """DynamoDB-friendly serialization"""
+        # convert to dictionary
+        output = self.dict(exclude={"item"})
+
+        # add STAC item as string
+        output["item"] = self.item.json()
+
+        # make JSON-friendly (will be able to do with Pydantic V2, https://github.com/pydantic/pydantic/issues/1409#issuecomment-1423995424)
+        return jsonable_encoder(output)
+
+
+@dataclasses.dataclass
+class ListIngestionRequest:
+    status: Status = Status.queued
+    limit: PositiveInt = None
+    next: Optional[str] = None
+
+    def __post_init_post_parse__(self) -> None:
+        # https://github.com/tiangolo/fastapi/issues/1474#issuecomment-1049987786
+        if self.next is None:
+            return
+
+        try:
+            self.next = json.loads(base64.b64decode(self.next))
+        except (UnicodeDecodeError, binascii.Error):
+            raise RequestValidationError(
+                [
+                    error_wrappers.ErrorWrapper(
+                        ValueError(
+                            "Unable to decode next token. Should be base64 encoded JSON"
+                        ),
+                        "query.next",
+                    )
+                ]
+            )
+
+
+class ListIngestionResponse(BaseModel):
+    items: List[Ingestion]
+    next: Optional[str]
+
+    @validator("next", pre=True)
+    def b64_encode_next(cls, next):
+        """
+        Base64 encode next parameter for easier transportability
+        """
+        if isinstance(next, dict):
+            return base64.b64encode(json.dumps(next).encode())
+        return next
+
+
+class UpdateIngestionRequest(BaseModel):
+    status: Status = None
+    message: str = None

package/lib/ingestor-api/runtime/src/services.py

@@ -0,0 +1,44 @@
+from typing import TYPE_CHECKING, List
+
+from boto3.dynamodb import conditions
+from pydantic import parse_obj_as
+
+from . import schemas
+
+if TYPE_CHECKING:
+    from mypy_boto3_dynamodb.service_resource import Table
+
+
+class Database:
+    def __init__(self, table: "Table"):
+        self.table = table
+
+    def write(self, ingestion: schemas.Ingestion):
+        self.table.put_item(Item=ingestion.dynamodb_dict())
+
+    def fetch_one(self, username: str, ingestion_id: str):
+        response = self.table.get_item(
+            Key={"created_by": username, "id": ingestion_id},
+        )
+        try:
+            return schemas.Ingestion.parse_obj(response["Item"])
+        except KeyError:
+            raise NotInDb("Record not found")
+
+    def fetch_many(
+        self, status: str, next: dict = None, limit: int = None
+    ) -> schemas.ListIngestionResponse:
+        response = self.table.query(
+            IndexName="status",
+            KeyConditionExpression=conditions.Key("status").eq(status),
+            **{"Limit": limit} if limit else {},
+            **{"ExclusiveStartKey": next} if next else {},
+        )
+        return {
+            "items": parse_obj_as(List[schemas.Ingestion], response["Items"]),
+            "next": response.get("LastEvaluatedKey"),
+        }
+
+
+class NotInDb(Exception):
+    ...

package/lib/ingestor-api/runtime/src/utils.py

@@ -0,0 +1,52 @@
+from typing import Sequence
+
+import boto3
+import pydantic
+from pypgstac.db import PgstacDB
+from pypgstac.load import Methods
+from fastapi.encoders import jsonable_encoder
+
+from .loader import Loader
+from .schemas import Ingestion
+
+
+class DbCreds(pydantic.BaseModel):
+    username: str
+    password: str
+    host: str
+    port: int
+    dbname: str
+    engine: str
+
+    @property
+    def dsn_string(self) -> str:
+        return f"{self.engine}://{self.username}:{self.password}@{self.host}:{self.port}/{self.dbname}"  # noqa
+
+
+def get_db_credentials(secret_arn: str) -> DbCreds:
+    """
+    Load pgSTAC database credentials from AWS Secrets Manager.
+    """
+    print("Fetching DB credentials...")
+    session = boto3.session.Session(region_name=secret_arn.split(":")[3])
+    client = session.client(service_name="secretsmanager")
+    response = client.get_secret_value(SecretId=secret_arn)
+    return DbCreds.parse_raw(response["SecretString"])
+
+
+def load_items(creds: DbCreds, ingestions: Sequence[Ingestion]):
+    """
+    Bulk insert STAC records into pgSTAC.
+    """
+    with PgstacDB(dsn=creds.dsn_string, debug=True) as db:
+        loader = Loader(db=db)
+
+        # serialize to JSON-friendly dicts (won't be necessary in Pydantic v2, https://github.com/pydantic/pydantic/issues/1409#issuecomment-1423995424)
+        items = jsonable_encoder(i.item for i in ingestions)
+        loading_result = loader.load_items(
+            file=items,
+            # use insert_ignore to avoid overwritting existing items or upsert to replace
+            insert_mode=Methods.upsert,
+        )
+
+        return loading_result

package/lib/ingestor-api/runtime/src/validators.py

@@ -0,0 +1,72 @@
+import functools
+
+import boto3
+import requests
+
+
+@functools.cache
+def get_s3_credentials():
+    from .config import settings
+
+    print("Fetching S3 Credentials...")
+
+    response = boto3.client("sts").assume_role(
+        RoleArn=settings.data_access_role,
+        RoleSessionName="stac-ingestor-data-validation",
+    )
+    return {
+        "aws_access_key_id": response["Credentials"]["AccessKeyId"],
+        "aws_secret_access_key": response["Credentials"]["SecretAccessKey"],
+        "aws_session_token": response["Credentials"]["SessionToken"],
+    }
+
+
+def s3_object_is_accessible(bucket: str, key: str):
+    """
+    Ensure we can send HEAD requests to S3 objects.
+    """
+    from .config import settings
+
+    client = boto3.client("s3", **get_s3_credentials())
+    try:
+        client.head_object(
+            Bucket=bucket,
+            Key=key,
+            **{"RequestPayer": "requester"} if settings.requester_pays else {},
+        )
+    except client.exceptions.ClientError as e:
+        raise ValueError(
+            f"Asset not accessible: {e.__dict__['response']['Error']['Message']}"
+        )
+
+
+def url_is_accessible(href: str):
+    """
+    Ensure URLs are accessible via HEAD requests.
+    """
+    try:
+        requests.head(href).raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        raise ValueError(
+            f"Asset not accessible: {e.response.status_code} {e.response.reason}"
+        )
+
+
+@functools.cache
+def collection_exists(collection_id: str) -> bool:
+    """
+    Ensure collection exists in STAC
+    """
+    from .config import settings
+
+    url = "/".join(
+        f'{url.strip("/")}' for url in [settings.stac_url, "collections", collection_id]
+    )
+
+    if (response := requests.get(url)).ok:
+        return True
+
+    raise ValueError(
+        f"Invalid collection '{collection_id}', received "
+        f"{response.status_code} response code from STAC API"
+    )

package/lib/ingestor-api/runtime/tests/conftest.py

@@ -0,0 +1,271 @@
+import os
+
+import boto3
+import pytest
+from fastapi.testclient import TestClient
+from moto import mock_dynamodb, mock_ssm
+from stac_pydantic import Item
+
+
+@pytest.fixture
+def test_environ():
+    # Mocked AWS Credentials for moto (best practice recommendation from moto)
+    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
+    os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
+    os.environ["AWS_SECURITY_TOKEN"] = "testing"
+    os.environ["AWS_SESSION_TOKEN"] = "testing"
+
+    # Config mocks
+    os.environ["DYNAMODB_TABLE"] = "test_table"
+    os.environ["JWKS_URL"] = "https://test-jwks.url"
+    os.environ["STAC_URL"] = "https://test-stac.url"
+    os.environ["DATA_ACCESS_ROLE"] = "arn:aws:iam::123456789012:role/test-role"
+    os.environ["DB_SECRET_ARN"] = "testing"
+
+
+@pytest.fixture
+def mock_ssm_parameter_store():
+    with mock_ssm():
+        yield boto3.client("ssm")
+
+
+@pytest.fixture
+def app(test_environ, mock_ssm_parameter_store):
+    from src.main import app
+
+    return app
+
+
+@pytest.fixture
+def api_client(app):
+    return TestClient(app)
+
+
+@pytest.fixture
+def mock_table(app, test_environ):
+    from src import dependencies
+    from src.config import settings
+
+    with mock_dynamodb():
+        client = boto3.resource("dynamodb")
+        mock_table = client.create_table(
+            TableName=settings.dynamodb_table,
+            AttributeDefinitions=[
+                {"AttributeName": "created_by", "AttributeType": "S"},
+                {"AttributeName": "id", "AttributeType": "S"},
+                {"AttributeName": "status", "AttributeType": "S"},
+                {"AttributeName": "created_at", "AttributeType": "S"},
+            ],
+            KeySchema=[
+                {"AttributeName": "created_by", "KeyType": "HASH"},
+                {"AttributeName": "id", "KeyType": "RANGE"},
+            ],
+            BillingMode="PAY_PER_REQUEST",
+            GlobalSecondaryIndexes=[
+                {
+                    "IndexName": "status",
+                    "KeySchema": [
+                        {"AttributeName": "status", "KeyType": "HASH"},
+                        {"AttributeName": "created_at", "KeyType": "RANGE"},
+                    ],
+                    "Projection": {"ProjectionType": "ALL"},
+                }
+            ],
+        )
+        app.dependency_overrides[dependencies.get_table] = lambda: mock_table
+        yield mock_table
+        app.dependency_overrides.pop(dependencies.get_table)
+
+
+@pytest.fixture
+def example_stac_item():
+    return {
+        "stac_version": "1.0.0",
+        "stac_extensions": [],
+        "type": "Feature",
+        "id": "20201211_223832_CS2",
+        "bbox": [
+            172.91173669923782,
+            1.3438851951615003,
+            172.95469614953714,
+            1.3690476620161975,
+        ],
+        "geometry": {
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    [172.91173669923782, 1.3438851951615003],
+                    [172.95469614953714, 1.3438851951615003],
+                    [172.95469614953714, 1.3690476620161975],
+                    [172.91173669923782, 1.3690476620161975],
+                    [172.91173669923782, 1.3438851951615003],
+                ]
+            ],
+        },
+        "properties": {
+            "datetime": "2020-12-11T22:38:32.125000Z",
+            "eo:cloud_cover": 1,
+        },
+        "collection": "simple-collection",
+        "links": [
+            {
+                "rel": "collection",
+                "href": "./collection.json",
+                "type": "application/json",
+                "title": "Simple Example Collection",
+            },
+            {
+                "rel": "root",
+                "href": "./collection.json",
+                "type": "application/json",
+                "title": "Simple Example Collection",
+            },
+            {
+                "rel": "parent",
+                "href": "./collection.json",
+                "type": "application/json",
+                "title": "Simple Example Collection",
+            },
+        ],
+        "assets": {
+            "visual": {
+                "href": "https://TEST_API.com/open-cogs/stac-examples/20201211_223832_CS2.tif",  # noqa
+                "type": "image/tiff; application=geotiff; profile=cloud-optimized",
+                "title": "3-Band Visual",
+                "roles": ["visual"],
+            },
+            "thumbnail": {
+                "href": "https://TEST_API.com/open-cogs/stac-examples/20201211_223832_CS2.jpg",  # noqa
+                "title": "Thumbnail",
+                "type": "image/jpeg",
+                "roles": ["thumbnail"],
+            },
+        },
+    }
+
+
+@pytest.fixture
+def example_stac_collection():
+    return {
+        "id": "simple-collection",
+        "type": "Collection",
+        "stac_extensions": [
+            "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
+            "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
+            "https://stac-extensions.github.io/view/v1.0.0/schema.json",
+        ],
+        "item_assets": {
+            "data": {
+                "type": "image/tiff; application=geotiff; profile=cloud-optimized",
+                "roles": ["data"],
+            }
+        },
+        "stac_version": "1.0.0",
+        "description": "A simple collection demonstrating core catalog fields with links to a couple of items",
+        "title": "Simple Example Collection",
+        "providers": [
+            {
+                "name": "Remote Data, Inc",
+                "description": "Producers of awesome spatiotemporal assets",
+                "roles": ["producer", "processor"],
+                "url": "http://remotedata.io",
+            }
+        ],
+        "extent": {
+            "spatial": {
+                "bbox": [
+                    [
+                        172.91173669923782,
+                        1.3438851951615003,
+                        172.95469614953714,
+                        1.3690476620161975,
+                    ]
+                ]
+            },
+            "temporal": {
+                "interval": [["2020-12-11T22:38:32.125Z", "2020-12-14T18:02:31.437Z"]]
+            },
+        },
+        "license": "CC-BY-4.0",
+        "summaries": {
+            "platform": ["cool_sat1", "cool_sat2"],
+            "constellation": ["ion"],
+            "instruments": ["cool_sensor_v1", "cool_sensor_v2"],
+            "gsd": {"minimum": 0.512, "maximum": 0.66},
+            "eo:cloud_cover": {"minimum": 1.2, "maximum": 1.2},
+            "proj:epsg": {"minimum": 32659, "maximum": 32659},
+            "view:sun_elevation": {"minimum": 54.9, "maximum": 54.9},
+            "view:off_nadir": {"minimum": 3.8, "maximum": 3.8},
+            "view:sun_azimuth": {"minimum": 135.7, "maximum": 135.7},
+        },
+        "links": [
+            {
+                "rel": "root",
+                "href": "./collection.json",
+                "type": "application/json",
+                "title": "Simple Example Collection",
+            },
+            {
+                "rel": "item",
+                "href": "./simple-item.json",
+                "type": "application/geo+json",
+                "title": "Simple Item",
+            },
+            {
+                "rel": "item",
+                "href": "./core-item.json",
+                "type": "application/geo+json",
+                "title": "Core Item",
+            },
+            {
+                "rel": "item",
+                "href": "./extended-item.json",
+                "type": "application/geo+json",
+                "title": "Extended Item",
+            },
+            {
+                "rel": "self",
+                "href": "https://raw.githubusercontent.com/radiantearth/stac-spec/v1.0.0/examples/collection.json",
+                "type": "application/json",
+            },
+        ],
+    }
+
+
+@pytest.fixture
+def client(app):
+    """
+    Return an API Client
+    """
+    app.dependency_overrides = {}
+    return TestClient(app)
+
+
+@pytest.fixture
+def client_authenticated(app):
+    """
+    Returns an API client which skips the authentication
+    """
+    from src.dependencies import get_username
+
+    app.dependency_overrides[get_username] = lambda: "test_user"
+    return TestClient(app)
+
+
+@pytest.fixture
+def stac_collection(example_stac_collection):
+    from src import schemas
+
+    return schemas.StacCollection(**example_stac_collection)
+
+
+@pytest.fixture
+def example_ingestion(example_stac_item):
+    from src import schemas
+
+    return schemas.Ingestion(
+        id=example_stac_item["id"],
+        created_by="test-user",
+        status=schemas.Status.queued,
+        item=Item.parse_obj(example_stac_item),
+    )

package/lib/ingestor-api/runtime/tests/test_collection.py

@@ -0,0 +1,35 @@
+from unittest.mock import Mock, patch
+import pytest
+from pypgstac.load import Methods
+from src.utils import DbCreds
+import src.collection as collection
+import os
+
+
+@pytest.fixture()
+def loader():
+    with patch("src.collection.Loader", autospec=True) as m:
+        yield m
+
+
+@pytest.fixture()
+def pgstacdb():
+    with patch("src.collection.PgstacDB", autospec=True) as m:
+        m.return_value.__enter__.return_value = Mock()
+        yield m
+
+
+def test_load_collections(stac_collection, loader, pgstacdb):
+    with patch(
+        "src.collection.get_db_credentials",
+        return_value=DbCreds(
+            username="", password="", host="", port=1, dbname="", engine=""
+        ),
+    ):
+        os.environ["DB_SECRET_ARN"] = ""
+        collection.ingest(stac_collection)
+
+    loader.return_value.load_collections.assert_called_once_with(
+        file=[stac_collection.to_dict()],
+        insert_mode=Methods.upsert,
+    )

package/lib/ingestor-api/runtime/tests/test_collection_endpoint.py

@@ -0,0 +1,41 @@
+from unittest.mock import patch
+
+publish_collections_endpoint = "/collections"
+delete_collection_endpoint = "/collections/{collection_id}"
+
+
+@patch("src.collection.ingest")
+def test_auth_publish_collection(
+    ingest, stac_collection, example_stac_collection, client_authenticated
+):
+    token = "token"
+    response = client_authenticated.post(
+        publish_collections_endpoint,
+        headers={"Authorization": f"bearer {token}"},
+        json=example_stac_collection,
+    )
+    ingest.assert_called_once_with(stac_collection)
+    assert response.status_code == 201
+
+
+def test_unauth_publish_collection(client, example_stac_collection):
+    response = client.post(publish_collections_endpoint, json=example_stac_collection)
+    assert response.status_code == 403
+
+
+@patch("src.collection.delete")
+def test_auth_delete_collection(delete, example_stac_collection, client_authenticated):
+    token = "token"
+    response = client_authenticated.delete(
+        delete_collection_endpoint.format(collection_id=example_stac_collection["id"]),
+        headers={"Authorization": f"bearer {token}"},
+    )
+    delete.assert_called_once_with(collection_id=example_stac_collection["id"])
+    assert response.status_code == 200
+
+
+def test_unauth_delete_collection(client, example_stac_collection):
+    response = client.delete(
+        delete_collection_endpoint.format(collection_id=example_stac_collection["id"]),
+    )
+    assert response.status_code == 403