@opengroundplan/core 0.1.1

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/catalog/_iceberg.py

@@ -0,0 +1,214 @@
+"""Apache Iceberg adapter — one of only two modules allowed to name a storage location.
+
+pyiceberg is a catalog plus a read/write client. Compaction, snapshot expiry, and orphan
+cleanup are Iceberg *engine procedures* (Spark, Trino, Flink), so those jobs report `delegated`
+with the exact call to run rather than pretending to have done the work. That is the honest
+version of "maintenance nobody budgeted for".
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from pyiceberg.catalog import load_catalog
+from pyiceberg.partitioning import PartitionField, PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.transforms import IdentityTransform
+from pyiceberg.types import (
+    BooleanType,
+    DateType,
+    DoubleType,
+    LongType,
+    NestedField,
+    StringType,
+    TimestamptzType,
+)
+
+from ..ports import MaintenanceOutcome, Table
+from ..schema import AsOf, TableDefinition
+from ..settings import LakehouseSettings
+from ..tables import definition_of
+
+# Logical type -> Iceberg type. Extend here, never at a call site.
+_TYPES = {
+    "string": StringType(),
+    "long": LongType(),
+    "double": DoubleType(),
+    "boolean": BooleanType(),
+    "date": DateType(),
+    "timestamptz": TimestamptzType(),
+}
+
+# Iceberg reserves partition field ids from 1000 upward.
+_PARTITION_FIELD_ID_BASE = 1000
+
+
+def _hours_ago(hours: int) -> datetime:
+    return datetime.now(UTC) - timedelta(hours=hours)
+
+
+def _as_millis(moment: datetime) -> int:
+    return int(moment.timestamp() * 1000)
+
+
+def _as_sql(moment: datetime) -> str:
+    """Engine procedures take a TIMESTAMP literal, so the detail must be paste-able SQL."""
+    return moment.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def _schema_of(definition: TableDefinition) -> Schema:
+    return Schema(
+        *(
+            NestedField(
+                field_id=index + 1,
+                name=column.name,
+                field_type=_TYPES[column.type],
+                required=column.required,
+                doc=column.doc or None,
+            )
+            for index, column in enumerate(definition.columns)
+        )
+    )
+
+
+def _partition_spec(definition: TableDefinition, schema: Schema) -> PartitionSpec:
+    """Identity transforms only: definitions already restrict partitions to date columns."""
+    return PartitionSpec(
+        *(
+            PartitionField(
+                source_id=schema.find_field(name).field_id,
+                field_id=_PARTITION_FIELD_ID_BASE + index,
+                transform=IdentityTransform(),
+                name=name,
+            )
+            for index, name in enumerate(definition.partition_by)
+        )
+    )
+
+
+class IcebergTable(Table):
+    def __init__(
+        self, table: Any, definition: TableDefinition, settings: LakehouseSettings
+    ) -> None:
+        self._table = table
+        self.definition = definition
+        self._settings = settings
+
+    @property
+    def _fqn(self) -> str:
+        return f"{self._settings.catalog_name}.{self._settings.namespace}.{self.definition.name}"
+
+    def scan(self, *, columns: tuple[str, ...] = (), as_of: AsOf | None = None) -> Any:
+        snapshot_id = self._snapshot_for(as_of)
+        scan = self._table.scan(selected_fields=columns or ("*",), snapshot_id=snapshot_id)
+        return scan.to_arrow()
+
+    def append(self, rows: Any) -> None:
+        self._table.append(rows)
+
+    def evolve_to(self, definition: TableDefinition) -> tuple[str, ...]:
+        """Additive only. Dropping or retyping a column breaks readers still on the old snapshot."""
+        existing = set(self._table.schema().column_names)
+        added = tuple(c for c in definition.columns if c.name not in existing)
+        if added:
+            with self._table.update_schema() as update:
+                for column in added:
+                    update.add_column(column.name, _TYPES[column.type], column.doc or None)
+        self.definition = definition
+        return tuple(column.name for column in added)
+
+    def history(self) -> tuple[dict[str, Any], ...]:
+        return tuple(
+            {"snapshot_id": snapshot.snapshot_id, "timestamp_ms": snapshot.timestamp_ms}
+            for snapshot in self._table.snapshots()
+        )
+
+    def _snapshot_for(self, as_of: AsOf | None) -> int | None:
+        if as_of is None:
+            return None
+        if as_of.version is not None:
+            raise ValueError("Iceberg addresses history by snapshot_id or timestamp, not version")
+        if as_of.snapshot_id is not None:
+            return as_of.snapshot_id
+        # Newest snapshot at or before the instant asked about; resolved from the snapshot log so
+        # no client-version-specific timestamp helper is required.
+        at_or_before = [s for s in self.history() if s["timestamp_ms"] <= as_of.timestamp_ms]
+        if not at_or_before:
+            raise ValueError(f"{self.definition.name}: no snapshot at {as_of.timestamp_ms}")
+        return max(at_or_before, key=lambda s: s["timestamp_ms"])["snapshot_id"]
+
+    def compact(self) -> MaintenanceOutcome:
+        target = self._settings.target_file_size_bytes
+        minimum = self._settings.min_input_files_to_compact
+        return MaintenanceOutcome(
+            job="compact",
+            table=self.definition.name,
+            status="delegated",
+            detail=(
+                f"CALL system.rewrite_data_files(table => '{self._fqn}', options => "
+                f"map('target-file-size-bytes','{target}','min-input-files','{minimum}'))"
+            ),
+        )
+
+    def expire_snapshots(self) -> MaintenanceOutcome:
+        cutoff = _hours_ago(self._settings.snapshot_retention_hours)
+        keep = self._settings.min_snapshots_to_keep
+        expiring = sum(1 for s in self.history() if s["timestamp_ms"] < _as_millis(cutoff))
+        return MaintenanceOutcome(
+            job="expire_snapshots",
+            table=self.definition.name,
+            status="delegated",
+            detail=(
+                f"{expiring} snapshot(s) past retention; "
+                f"CALL system.expire_snapshots(table => '{self._fqn}', "
+                f"older_than => TIMESTAMP '{_as_sql(cutoff)}', retain_last => {keep})"
+            ),
+        )
+
+    def remove_orphan_files(self) -> MaintenanceOutcome:
+        cutoff = _hours_ago(self._settings.orphan_file_grace_hours)
+        return MaintenanceOutcome(
+            job="remove_orphan_files",
+            table=self.definition.name,
+            status="delegated",
+            detail=(
+                f"CALL system.remove_orphan_files(table => '{self._fqn}', "
+                f"older_than => TIMESTAMP '{_as_sql(cutoff)}')"
+            ),
+        )
+
+
+class IcebergCatalog:
+    def __init__(self, settings: LakehouseSettings) -> None:
+        self._settings = settings
+        self._catalog = load_catalog(
+            settings.catalog_name,
+            uri=settings.catalog_uri,
+            warehouse=settings.warehouse_uri,
+        )
+        self._catalog.create_namespace_if_not_exists(settings.namespace)
+
+    def _identifier(self, name: str) -> str:
+        return f"{self._settings.namespace}.{name}"
+
+    def load(self, name: str) -> Table:
+        return IcebergTable(
+            self._catalog.load_table(self._identifier(name)), definition_of(name), self._settings
+        )
+
+    def create_if_absent(self, definition: TableDefinition) -> Table:
+        schema = _schema_of(definition)
+        table = self._catalog.create_table_if_not_exists(
+            self._identifier(definition.name),
+            schema=schema,
+            partition_spec=_partition_spec(definition, schema),
+        )
+        return IcebergTable(table, definition, self._settings)
+
+    def list_tables(self) -> tuple[str, ...]:
+        return tuple(entry[-1] for entry in self._catalog.list_tables(self._settings.namespace))
+
+
+def build_catalog(settings: LakehouseSettings) -> IcebergCatalog:
+    return IcebergCatalog(settings)

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/maintenance/__init__.py

@@ -0,0 +1,9 @@
+"""Compaction, snapshot expiry, and orphan cleanup.
+
+Unglamorous, non-optional, and the first line cut from a lakehouse budget — so it ships as a
+runnable job rather than a paragraph in a runbook nobody opens.
+"""
+
+from .jobs import JOBS, run_maintenance, run_table
+
+__all__ = ["JOBS", "run_maintenance", "run_table"]

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/maintenance/__main__.py

@@ -0,0 +1,31 @@
+"""CLI entry point: `python -m <package>.lakehouse.maintenance [job ...]`. Schedule it."""
+
+from __future__ import annotations
+
+import sys
+
+from .jobs import JOBS, run_maintenance
+
+
+def main(argv: list[str] | None = None) -> int:
+    requested = tuple(argv if argv is not None else sys.argv[1:]) or JOBS
+    unknown = [job for job in requested if job not in JOBS]
+    if unknown:
+        print(f"unknown job(s): {', '.join(unknown)}; known: {', '.join(JOBS)}", file=sys.stderr)
+        return 2
+
+    delegated = 0
+    for outcome in run_maintenance(requested):
+        print(f"{outcome.status:<9} {outcome.table}.{outcome.job}: {outcome.detail}")
+        delegated += outcome.status == "delegated"
+
+    # Delegated work is real work nobody has run yet. Surface it loudly; do not fail the
+    # schedule over it, or the timer gets disabled and the small files win.
+    if delegated:
+        doc = "docs/architecture/lakehouse.python.md"
+        print(f"{delegated} job(s) need a compute engine — see {doc}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/maintenance/jobs.py

@@ -0,0 +1,38 @@
+"""The maintenance schedule, as code.
+
+Small-file growth is proportional to write frequency, so this belongs on a timer from day one.
+Every threshold comes from `LakehouseSettings`; nothing here decides a number for itself.
+"""
+
+from __future__ import annotations
+
+from ..catalog import open_catalog
+from ..ports import MaintenanceOutcome, Table
+from ..schema import TableDefinition
+from ..tables import TABLES, definition_of
+
+JOBS = ("evolve", "compact", "expire_snapshots", "remove_orphan_files")
+
+
+def _evolve(table: Table, definition: TableDefinition) -> MaintenanceOutcome:
+    added = table.evolve_to(definition)
+    return MaintenanceOutcome(
+        job="evolve",
+        table=definition.name,
+        status="applied" if added else "skipped",
+        detail=f"added {', '.join(added)}" if added else "schema already current",
+    )
+
+
+def run_table(name: str, jobs: tuple[str, ...] = JOBS) -> list[MaintenanceOutcome]:
+    definition = definition_of(name)
+    # create_if_absent first: a fresh warehouse bootstraps itself instead of failing the schedule.
+    table = open_catalog().create_if_absent(definition)
+    return [
+        _evolve(table, definition) if job == "evolve" else getattr(table, job)()
+        for job in jobs
+    ]
+
+
+def run_maintenance(jobs: tuple[str, ...] = JOBS) -> list[MaintenanceOutcome]:
+    return [outcome for name in sorted(TABLES) for outcome in run_table(name, jobs)]

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/ports.py

@@ -0,0 +1,58 @@
+"""What every table format must provide.
+
+Query and maintenance code depends on these Protocols, never on a client library — that is the
+seam that keeps `pyiceberg` or `deltalake` out of the rest of the repo.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal, Protocol
+
+from .schema import AsOf, TableDefinition
+
+# `delegated` is not a failure: some maintenance is a compute-engine procedure, not a client
+# call. Reporting it keeps the cost visible instead of silently unbudgeted.
+MaintenanceStatus = Literal["applied", "delegated", "skipped"]
+
+
+@dataclass(frozen=True)
+class MaintenanceOutcome:
+    job: str
+    table: str
+    status: MaintenanceStatus
+    detail: str
+
+
+class Table(Protocol):
+    """A table handle. Every operation is named in terms of the table, never of its files."""
+
+    definition: TableDefinition
+
+    def scan(self, *, columns: tuple[str, ...] = (), as_of: AsOf | None = None) -> Any: ...
+
+    def append(self, rows: Any) -> None: ...
+
+    def evolve_to(self, definition: TableDefinition) -> tuple[str, ...]:
+        """Apply additive schema changes; returns the columns added."""
+        ...
+
+    def history(self) -> tuple[dict[str, Any], ...]:
+        """Snapshots or commits, newest last — the raw material for time travel."""
+        ...
+
+    def compact(self) -> MaintenanceOutcome: ...
+
+    def expire_snapshots(self) -> MaintenanceOutcome: ...
+
+    def remove_orphan_files(self) -> MaintenanceOutcome: ...
+
+
+class TableCatalog(Protocol):
+    """The name-to-storage authority. Nothing else in the repo is allowed to be one."""
+
+    def load(self, name: str) -> Table: ...
+
+    def create_if_absent(self, definition: TableDefinition) -> Table: ...
+
+    def list_tables(self) -> tuple[str, ...]: ...

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/queries/__init__.py

@@ -0,0 +1,13 @@
+"""Read paths.
+
+Nothing in this package may name a bucket, a prefix, or a file — only a table. That single
+rule is what `tests/architecture/test_lakehouse_boundaries.py` fails the build over.
+"""
+
+from .reads import {{ firstTable | snake }}_as_of, {{ firstTable | snake }}_columns, recent_{{ firstTable | snake }}
+
+__all__ = [
+    "{{ firstTable | snake }}_as_of",
+    "{{ firstTable | snake }}_columns",
+    "recent_{{ firstTable | snake }}",
+]

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/queries/reads.py

@@ -0,0 +1,30 @@
+"""Example reads: current state, and the same read against an older version of the table.
+
+Both go through the catalog. Neither can see storage — that is not a convention here, it is
+what the boundary test checks.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..catalog import open_catalog
+from ..schema import AsOf
+from ..tables import definition_of
+
+_TABLE = "{{ firstTable | snake }}"
+
+
+def recent_{{ firstTable | snake }}(columns: tuple[str, ...] = ()) -> Any:
+    """Current table state. The catalog resolves the name; this function never sees a location."""
+    return open_catalog().load(_TABLE).scan(columns=columns)
+
+
+def {{ firstTable | snake }}_as_of(timestamp_ms: int, columns: tuple[str, ...] = ()) -> Any:
+    """Time travel — the audit answer to "what did this table say last Tuesday?"."""
+    return open_catalog().load(_TABLE).scan(columns=columns, as_of=AsOf(timestamp_ms=timestamp_ms))
+
+
+def {{ firstTable | snake }}_columns() -> tuple[str, ...]:
+    """Column names from the definition, so callers never hardcode a projection list."""
+    return tuple(column.name for column in definition_of(_TABLE).columns)

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/schema.py

@@ -0,0 +1,66 @@
+"""Table descriptions as plain data.
+
+Stating what a table *is* must not require a client library — that is what lets the definition
+be reviewed, diffed, and evolved without an engine in the loop.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+# Logical type names. Adapters map these to the format's own type system; call sites never do.
+LOGICAL_TYPES = ("string", "long", "double", "boolean", "date", "timestamptz")
+
+
+@dataclass(frozen=True)
+class Column:
+    name: str
+    type: str
+    required: bool = False
+    doc: str = ""
+
+    def __post_init__(self) -> None:
+        if self.type not in LOGICAL_TYPES:
+            raise ValueError(f"{self.name}: unknown type {self.type!r}; add it to LOGICAL_TYPES")
+
+
+@dataclass(frozen=True)
+class TableDefinition:
+    """A table is named, not located. The catalog owns the mapping from this name to storage."""
+
+    name: str
+    columns: tuple[Column, ...]
+    # Partition on a low-cardinality derived column (a date), never on a raw timestamp — one
+    # partition per microsecond is how a lakehouse dies of small files.
+    partition_by: tuple[str, ...] = ()
+    description: str = ""
+
+    def __post_init__(self) -> None:
+        known = {column.name for column in self.columns}
+        missing = [name for name in self.partition_by if name not in known]
+        if missing:
+            raise ValueError(f"{self.name}: partitions on undefined columns {missing}")
+
+    def column(self, name: str) -> Column:
+        for column in self.columns:
+            if column.name == name:
+                return column
+        raise KeyError(f"{self.name} has no column {name!r}")
+
+
+@dataclass(frozen=True)
+class AsOf:
+    """Time travel coordinates — exactly one is set.
+
+    `snapshot_id` is Iceberg's, `version` is Delta's; `timestamp_ms` works on both and is the
+    one to reach for when the question came from an auditor rather than a log line.
+    """
+
+    snapshot_id: int | None = None
+    version: int | None = None
+    timestamp_ms: int | None = None
+
+    def __post_init__(self) -> None:
+        given = [v for v in (self.snapshot_id, self.version, self.timestamp_ms) if v is not None]
+        if len(given) != 1:
+            raise ValueError("AsOf takes exactly one of snapshot_id, version, timestamp_ms")

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/settings.py

@@ -0,0 +1,61 @@
+"""The one typed default table for the lakehouse. Nothing else may read the environment."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, fields
+from functools import lru_cache
+
+ENV_PREFIX = "{{ solutionName | upperSnake }}_LAKEHOUSE_"
+
+
+@dataclass(frozen=True)
+class LakehouseSettings:
+    """Every lakehouse tunable, defaulted once. Only the catalog reads the location fields."""
+
+    # Catalog identity. Tables are addressed by name through this, never by location.
+    catalog_name: str = "{{ solutionName | snake }}"
+    catalog_uri: str = "sqlite:///{{ solutionName | kebab }}-catalog.db"
+    # Grouping inside the catalog. Split it per domain once one namespace stops being enough.
+    namespace: str = "lakehouse"
+
+    # The single value in the repo that says where bytes live. Point it at s3://, abfss://, or
+    # gs:// in a deployed environment; the scheme-less default keeps a local run working, and
+    # `file://./x` does not — delta-rs reads the "." as a host.
+    warehouse_uri: str = "./warehouse"
+
+    # Maintenance budget. Streaming and micro-batch writes produce small files faster than
+    # anyone plans for, and every one of them costs a metadata read at query time.
+    target_file_size_bytes: int = 512 * 1024 * 1024
+    min_input_files_to_compact: int = 5
+
+    # How far back time travel reaches. Longer is not free — expired snapshots pin data files.
+    snapshot_retention_hours: int = 168
+    min_snapshots_to_keep: int = 3
+
+    # Grace period before an unreferenced file may be deleted. Must exceed the longest running
+    # write or reader; Delta refuses anything under 168h for exactly that reason.
+    orphan_file_grace_hours: int = 168
+
+    # Deleting a file a live reader still needs is unrecoverable, so destructive maintenance
+    # reports before it acts until someone turns this off deliberately.
+    dry_run_maintenance: bool = True
+
+
+def _to_bool(raw: str) -> bool:
+    return raw.strip().lower() in ("1", "true", "yes", "on")
+
+
+# Keyed by the annotation *string*: `from __future__ import annotations` makes field.type a str.
+_CASTS = {"str": str, "int": int, "bool": _to_bool}
+
+
+@lru_cache
+def get_settings() -> LakehouseSettings:
+    """Cached so the environment is read once and every caller shares one instance."""
+    overrides = {}
+    for field in fields(LakehouseSettings):
+        raw = os.environ.get(ENV_PREFIX + field.name.upper())
+        if raw is not None:
+            overrides[field.name] = _CASTS[field.type](raw)
+    return LakehouseSettings(**overrides)

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/tables/__firstTable.snake__.py

@@ -0,0 +1,22 @@
+"""One table, described once.
+
+Change the schema by editing this definition and running the `evolve` maintenance job.
+Additive only: readers on an older snapshot still expect the columns they were written with.
+"""
+
+from ..schema import Column, TableDefinition
+
+{{ firstTable | upperSnake }} = TableDefinition(
+    name="{{ firstTable | snake }}",
+    description="{{ firstTable | title }} recorded by {{ solutionName | title }}.",
+    columns=(
+        Column("{{ firstTable | snake }}_id", "string", required=True, doc="Idempotency key for writers."),
+        # Partition column: date, not timestamp. Identity-partitioning a timestamp yields one
+        # partition per microsecond, which is the classic way to bury a lakehouse in small files.
+        Column("event_date", "date", required=True, doc="UTC date of occurred_at; the partition."),
+        Column("occurred_at", "timestamptz", required=True, doc="When it happened, not landed."),
+        Column("subject", "string", doc="Entity the fact is about."),
+        Column("payload", "string", doc="JSON body. Widen the schema instead of overloading this."),
+    ),
+    partition_by=("event_date",),
+)

package/templates/architectures/lakehouse/files/src/__solutionName.snake__/lakehouse/tables/__init__.py

@@ -0,0 +1,28 @@
+"""Every table this repo owns, by name.
+
+This registry is the whole public namespace of the lakehouse. If a table is not here, no query
+can reach it — which is the point: there is no second way in via a path.
+"""
+
+from __future__ import annotations
+
+from ..schema import TableDefinition
+from .{{ firstTable | snake }} import {{ firstTable | upperSnake }}
+
+TABLES: dict[str, TableDefinition] = {
+    definition.name: definition for definition in ({{ firstTable | upperSnake }},)
+}
+
+
+def definition_of(name: str) -> TableDefinition:
+    try:
+        return TABLES[name]
+    except KeyError:
+        known = ", ".join(sorted(TABLES)) or "none"
+        raise KeyError(
+            f"{name!r} is not a registered table (known: {known}); "
+            f"add a module under lakehouse/tables/ and list it here"
+        ) from None
+
+
+__all__ = ["TABLES", "definition_of"]