forgecraft-mcp 1.2.0 → 1.3.2

package/templates/cli/review.yaml

@@ -1,53 +1,53 @@
-tag: CLI
-section: review
-blocks:
-  - id: cli-architecture-review
-    tier: recommended
-    dimension: architecture
-    title: "CLI Architecture Review"
-    description: |
-      Evaluate CLI-specific patterns: argument parsing, output formatting, and exit codes.
-    checklist:
-      - id: cli-argument-parsing
-        description: "Arguments parsed with a proper library (yargs, commander, clap). No manual argv slicing."
-        severity: important
-      - id: cli-exit-codes
-        description: "Exit codes follow convention: 0 = success, 1 = general error, 2 = usage error. Documented."
-        severity: important
-      - id: cli-help-output
-        description: "--help flag yields clear usage with examples. Every subcommand has a description."
-        severity: important
-      - id: cli-stdin-stdout
-        description: "Supports stdin/stdout piping for composability. Machine-readable output (JSON) via --json flag."
-        severity: nice-to-have
-
-  - id: cli-code-quality-review
-    tier: recommended
-    dimension: code-quality
-    title: "CLI Code Quality Review"
-    description: |
-      Evaluate CLI UX and error handling.
-    checklist:
-      - id: cli-error-messages
-        description: "Errors show what went wrong, why, and how to fix it. Include the failing input value."
-        severity: critical
-      - id: cli-progress-feedback
-        description: "Long operations show progress (spinner, progress bar). Silent mode available via --quiet."
-        severity: important
-      - id: cli-destructive-confirm
-        description: "Destructive operations require confirmation unless --force or --yes flag is passed."
-        severity: important
-
-  - id: cli-test-review
-    tier: recommended
-    dimension: tests
-    title: "CLI Test Review"
-    description: |
-      Evaluate CLI-specific testing patterns.
-    checklist:
-      - id: cli-integration-tests
-        description: "End-to-end tests invoke the CLI binary and assert on stdout, stderr, and exit code."
-        severity: critical
-      - id: cli-snapshot-output
-        description: "Help text and formatted output tested via snapshots to catch unintended changes."
-        severity: nice-to-have
+tag: CLI
+section: review
+blocks:
+  - id: cli-architecture-review
+    tier: recommended
+    dimension: architecture
+    title: "CLI Architecture Review"
+    description: |
+      Evaluate CLI-specific patterns: argument parsing, output formatting, and exit codes.
+    checklist:
+      - id: cli-argument-parsing
+        description: "Arguments parsed with a proper library (yargs, commander, clap). No manual argv slicing."
+        severity: important
+      - id: cli-exit-codes
+        description: "Exit codes follow convention: 0 = success, 1 = general error, 2 = usage error. Documented."
+        severity: important
+      - id: cli-help-output
+        description: "--help flag yields clear usage with examples. Every subcommand has a description."
+        severity: important
+      - id: cli-stdin-stdout
+        description: "Supports stdin/stdout piping for composability. Machine-readable output (JSON) via --json flag."
+        severity: nice-to-have
+
+  - id: cli-code-quality-review
+    tier: recommended
+    dimension: code-quality
+    title: "CLI Code Quality Review"
+    description: |
+      Evaluate CLI UX and error handling.
+    checklist:
+      - id: cli-error-messages
+        description: "Errors show what went wrong, why, and how to fix it. Include the failing input value."
+        severity: critical
+      - id: cli-progress-feedback
+        description: "Long operations show progress (spinner, progress bar). Silent mode available via --quiet."
+        severity: important
+      - id: cli-destructive-confirm
+        description: "Destructive operations require confirmation unless --force or --yes flag is passed."
+        severity: important
+
+  - id: cli-test-review
+    tier: recommended
+    dimension: tests
+    title: "CLI Test Review"
+    description: |
+      Evaluate CLI-specific testing patterns.
+    checklist:
+      - id: cli-integration-tests
+        description: "End-to-end tests invoke the CLI binary and assert on stdout, stderr, and exit code."
+        severity: critical
+      - id: cli-snapshot-output
+        description: "Help text and formatted output tested via snapshots to catch unintended changes."
+        severity: nice-to-have

package/templates/cli/structure.yaml

@@ -1,16 +1,16 @@
-tag: CLI
-section: structure
-language: typescript
-entries:
-  - path: src/index.ts
-    type: file
-    description: "Entry point with shebang (#!/usr/bin/env node)"
-  - path: src/cli
-    type: directory
-    description: "CLI command definitions and argument parsing"
-  - path: src/shared/config
-    type: directory
-    description: "CLI configuration and env handling"
-  - path: src/shared/errors
-    type: directory
-    description: "Error hierarchy with user-friendly messages"
+tag: CLI
+section: structure
+language: typescript
+entries:
+  - path: src/index.ts
+    type: file
+    description: "Entry point with shebang (#!/usr/bin/env node)"
+  - path: src/cli
+    type: directory
+    description: "CLI command definitions and argument parsing"
+  - path: src/shared/config
+    type: directory
+    description: "CLI configuration and env handling"
+  - path: src/shared/errors
+    type: directory
+    description: "Error hierarchy with user-friendly messages"

package/templates/data-lineage/instructions.yaml

@@ -1,28 +1,28 @@
-tag: DATA-LINEAGE
-section: instructions
-blocks:
-  - id: field-coverage
-    tier: recommended
-    title: "100% Field Coverage Enforcement"
-    content: |
-      ## 100% Field Coverage Enforcement
-
-      - Every data field must have a documented origin (source system, table, column) and destination (target system, table, column).
-      - Maintain a field-level lineage registry: no field enters production without a lineage entry. Enforce this in CI.
-      - Track field transformations explicitly: document every rename, type cast, aggregation, join, and filter applied to each field.
-      - Implement automated coverage checks: compare the lineage registry against actual schema definitions. Flag any untracked fields as errors.
-      - Version lineage metadata alongside code: lineage definitions live in the repository, not in external wikis or spreadsheets.
-      - Generate lineage reports per pipeline run: which fields were read, transformed, and written, with row counts and data quality metrics.
-
-  - id: lineage-tracking-decorators
-    tier: recommended
-    title: "Lineage Tracking Decorators & Annotations"
-    content: |
-      ## Lineage Tracking Decorators & Annotations
-
-      - Use decorators or annotations on transformation functions to declare input fields, output fields, and transformation type.
-      - Standardize lineage metadata format: source, transformation, destination, timestamp, pipeline_id, run_id.
-      - Emit lineage events at runtime: every transformation step publishes a lineage event to a centralized lineage store.
-      - Support both code-level lineage (decorators on functions) and config-level lineage (YAML/JSON transformation specs).
-      - Integrate lineage with data quality: when a quality check fails, trace back through lineage to identify the source of bad data.
-      - Visualize lineage graphs: generate dependency diagrams showing field-level flow from source to consumption.
+tag: DATA-LINEAGE
+section: instructions
+blocks:
+  - id: field-coverage
+    tier: recommended
+    title: "100% Field Coverage Enforcement"
+    content: |
+      ## 100% Field Coverage Enforcement
+
+      - Every data field must have a documented origin (source system, table, column) and destination (target system, table, column).
+      - Maintain a field-level lineage registry: no field enters production without a lineage entry. Enforce this in CI.
+      - Track field transformations explicitly: document every rename, type cast, aggregation, join, and filter applied to each field.
+      - Implement automated coverage checks: compare the lineage registry against actual schema definitions. Flag any untracked fields as errors.
+      - Version lineage metadata alongside code: lineage definitions live in the repository, not in external wikis or spreadsheets.
+      - Generate lineage reports per pipeline run: which fields were read, transformed, and written, with row counts and data quality metrics.
+
+  - id: lineage-tracking-decorators
+    tier: recommended
+    title: "Lineage Tracking Decorators & Annotations"
+    content: |
+      ## Lineage Tracking Decorators & Annotations
+
+      - Use decorators or annotations on transformation functions to declare input fields, output fields, and transformation type.
+      - Standardize lineage metadata format: source, transformation, destination, timestamp, pipeline_id, run_id.
+      - Emit lineage events at runtime: every transformation step publishes a lineage event to a centralized lineage store.
+      - Support both code-level lineage (decorators on functions) and config-level lineage (YAML/JSON transformation specs).
+      - Integrate lineage with data quality: when a quality check fails, trace back through lineage to identify the source of bad data.
+      - Visualize lineage graphs: generate dependency diagrams showing field-level flow from source to consumption.

package/templates/data-lineage/mcp-servers.yaml

@@ -1,22 +1,22 @@
-tag: DATA-LINEAGE
-section: mcp-servers
-servers:
-  - name: postgres
-    description: "PostgreSQL database inspection — query lineage metadata and schema definitions"
-    command: npx
-    args: ["-y", "@modelcontextprotocol/server-postgres"]
-    tags: [DATA-LINEAGE, DATA-PIPELINE]
-    category: database
-    tier: recommended
-    env:
-      POSTGRES_CONNECTION_STRING: ""
-    url: "https://github.com/modelcontextprotocol/servers/tree/main/src/postgres"
-
-  - name: filesystem
-    description: "Filesystem access for lineage definition files and pipeline configurations"
-    command: npx
-    args: ["-y", "@modelcontextprotocol/server-filesystem"]
-    tags: [DATA-LINEAGE, UNIVERSAL]
-    category: filesystem
-    tier: optional
-    url: "https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem"
+tag: DATA-LINEAGE
+section: mcp-servers
+servers:
+  - name: postgres
+    description: "PostgreSQL database inspection — query lineage metadata and schema definitions"
+    command: npx
+    args: ["-y", "@modelcontextprotocol/server-postgres"]
+    tags: [DATA-LINEAGE, DATA-PIPELINE]
+    category: database
+    tier: recommended
+    env:
+      POSTGRES_CONNECTION_STRING: ""
+    url: "https://github.com/modelcontextprotocol/servers/tree/main/src/postgres"
+
+  - name: filesystem
+    description: "Filesystem access for lineage definition files and pipeline configurations"
+    command: npx
+    args: ["-y", "@modelcontextprotocol/server-filesystem"]
+    tags: [DATA-LINEAGE, UNIVERSAL]
+    category: filesystem
+    tier: optional
+    url: "https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem"

package/templates/data-pipeline/instructions.yaml

@@ -1,84 +1,84 @@
-tag: DATA-PIPELINE
-section: instructions
-blocks:
-  - id: etl-patterns
-    tier: recommended
-    title: "ETL & Pipeline Design"
-    content: |
-      ## ETL & Pipeline Orchestration
-
-      - Design every pipeline stage to be idempotent. Re-running a stage with the same input must produce the same output without side effects or duplicates.
-      - Use a DAG-based orchestrator (e.g., Airflow, Dagster, Prefect) to define task dependencies, schedule runs, and handle retries with exponential backoff.
-      - Partition data by time (daily/hourly) or logical key so that re-processing a failed partition does not require reprocessing the entire dataset.
-      - Implement the Extract-Load-Transform (ELT) pattern when the target warehouse supports compute; push transformations into SQL/dbt models for transparency and version control.
-      - Tag every pipeline run with a unique run ID. Propagate this ID through all stages and into output metadata for end-to-end lineage tracking.
-      - Keep extraction, transformation, and loading logic in separate, independently testable modules. Avoid monolithic scripts that mix concerns.
-      - Define SLAs for each pipeline (e.g., "daily sales pipeline completes by 06:00 UTC"). Alert on SLA breach, not just on failure.
-
-  - id: data-validation
-    tier: recommended
-    title: "Data Validation & Quality"
-    content: |
-      ## Data Validation & Quality Gates
-
-      - Validate data at every boundary: after extraction (schema conformance), after transformation (business rules), and before loading (referential integrity).
-      - Use schema contracts (e.g., JSON Schema, Avro, Protobuf, or Great Expectations suites) to enforce column types, nullability, and allowed value ranges.
-      - Implement row-count and distribution checks between stages. A sudden drop or spike in row count (> 20% deviation from baseline) should trigger an alert and pause downstream processing.
-      - Quarantine invalid records into a dead-letter table rather than dropping them silently. Include the original record, the validation error, and the run ID.
-      - Maintain a data quality dashboard that tracks freshness, completeness, uniqueness, and accuracy metrics per dataset over time.
-      - Write unit tests for transformation logic using fixed input fixtures. Write integration tests that run a mini-pipeline against a test database.
-
-  - id: reliability-patterns
-    tier: recommended
-    title: "Reliability & Error Handling"
-    content: |
-      ## Reliability, Retry & Recovery
-
-      - Configure retries with exponential backoff and jitter for transient failures (network timeouts, rate limits, temporary unavailability). Cap retries at 3-5 attempts.
-      - Use checkpointing or write-ahead logs for long-running pipelines so that a restart resumes from the last successful checkpoint, not from the beginning.
-      - Implement circuit breakers on external API calls to prevent cascading failures when a source system is degraded.
-      - Log structured events (JSON) for every stage: start, success, failure, retry, and skip. Include record counts, duration, and error details.
-      - Design for exactly-once semantics where possible using upserts (INSERT ... ON CONFLICT UPDATE) or deduplication keys in the target store.
-      - Maintain a pipeline runbook that documents failure modes, recovery steps, and escalation contacts. Review and update it quarterly.
-
-  - id: data-pipeline-testing
-    tier: recommended
-    title: "Data Pipeline Testing Requirements"
-    content: |
-      ## Data Pipeline Testing Requirements
-
-      ### Data Quality Assertions at Every Stage
-      Assert at each pipeline boundary — after extraction, after transformation, before loading:
-      - **Row count** — within expected range or ± % of prior run baseline.
-      - **Null rate** — per column, against stated threshold (e.g., `customer_id` must never be null).
-      - **Schema compliance** — column names, types, nullable flags match the declared schema.
-      - **Referential integrity** — foreign key values exist in the referenced table.
-      - **Distribution checks** — value ranges and cardinality within expected bounds. Alert on sudden shifts (> 20% deviation from trailing 7-day average).
-      Use Great Expectations, dbt tests, or equivalent. Assertions are first-class artifacts — committed, versioned, and enforced in CI.
-
-      ### Idempotency Tests
-      Re-running the pipeline with the same input must produce identical output with no side effects. Test explicitly:
-      - Run pipeline N once; capture output row count and checksum.
-      - Run pipeline N again with the same source data.
-      - Assert: output row count and checksum are identical; no duplicates in target; no reprocessed records in dead-letter queue.
-      Idempotency is a contract, not an assumption. If a stage is not idempotent by design, that decision requires an ADR.
-
-      ### Backfill Correctness Tests
-      Historical data processed against current transformation logic must produce correct results:
-      - Define a fixed historical fixture (date range, known input, known output).
-      - Run current pipeline logic against the fixture.
-      - Assert output matches expected historical output.
-      Any change to transformation logic that causes a backfill regression is a breaking change requiring an ADR.
-
-      ### Dead-Letter Queue Draining Tests
-      Malformed records must be handled — not silently dropped and not blocking the pipeline:
-      - Inject a record that violates the schema; assert it routes to the dead-letter table/queue.
-      - Inject a record that passes schema but fails a business rule; assert correct routing.
-      - Assert the DLQ record contains: original payload, validation error, run ID, timestamp.
-      - Test DLQ drain procedure: records can be corrected, resubmitted, and processed successfully.
-
-      ### Volume / Scale Tests at Staging
-      Unit-scale fixtures are insufficient for pipeline correctness. Run at representative data volume:
-      - At staging: use a realistic data volume (minimum 10% of production record count, or a known large-batch scenario).
-      - Assert: completion within SLA window; no memory exhaustion; partition output correct.
-      - Document the scale test fixture size and the SLA threshold in the spec.
+tag: DATA-PIPELINE
+section: instructions
+blocks:
+  - id: etl-patterns
+    tier: recommended
+    title: "ETL & Pipeline Design"
+    content: |
+      ## ETL & Pipeline Orchestration
+
+      - Design every pipeline stage to be idempotent. Re-running a stage with the same input must produce the same output without side effects or duplicates.
+      - Use a DAG-based orchestrator (e.g., Airflow, Dagster, Prefect) to define task dependencies, schedule runs, and handle retries with exponential backoff.
+      - Partition data by time (daily/hourly) or logical key so that re-processing a failed partition does not require reprocessing the entire dataset.
+      - Implement the Extract-Load-Transform (ELT) pattern when the target warehouse supports compute; push transformations into SQL/dbt models for transparency and version control.
+      - Tag every pipeline run with a unique run ID. Propagate this ID through all stages and into output metadata for end-to-end lineage tracking.
+      - Keep extraction, transformation, and loading logic in separate, independently testable modules. Avoid monolithic scripts that mix concerns.
+      - Define SLAs for each pipeline (e.g., "daily sales pipeline completes by 06:00 UTC"). Alert on SLA breach, not just on failure.
+
+  - id: data-validation
+    tier: recommended
+    title: "Data Validation & Quality"
+    content: |
+      ## Data Validation & Quality Gates
+
+      - Validate data at every boundary: after extraction (schema conformance), after transformation (business rules), and before loading (referential integrity).
+      - Use schema contracts (e.g., JSON Schema, Avro, Protobuf, or Great Expectations suites) to enforce column types, nullability, and allowed value ranges.
+      - Implement row-count and distribution checks between stages. A sudden drop or spike in row count (> 20% deviation from baseline) should trigger an alert and pause downstream processing.
+      - Quarantine invalid records into a dead-letter table rather than dropping them silently. Include the original record, the validation error, and the run ID.
+      - Maintain a data quality dashboard that tracks freshness, completeness, uniqueness, and accuracy metrics per dataset over time.
+      - Write unit tests for transformation logic using fixed input fixtures. Write integration tests that run a mini-pipeline against a test database.
+
+  - id: reliability-patterns
+    tier: recommended
+    title: "Reliability & Error Handling"
+    content: |
+      ## Reliability, Retry & Recovery
+
+      - Configure retries with exponential backoff and jitter for transient failures (network timeouts, rate limits, temporary unavailability). Cap retries at 3-5 attempts.
+      - Use checkpointing or write-ahead logs for long-running pipelines so that a restart resumes from the last successful checkpoint, not from the beginning.
+      - Implement circuit breakers on external API calls to prevent cascading failures when a source system is degraded.
+      - Log structured events (JSON) for every stage: start, success, failure, retry, and skip. Include record counts, duration, and error details.
+      - Design for exactly-once semantics where possible using upserts (INSERT ... ON CONFLICT UPDATE) or deduplication keys in the target store.
+      - Maintain a pipeline runbook that documents failure modes, recovery steps, and escalation contacts. Review and update it quarterly.
+
+  - id: data-pipeline-testing
+    tier: recommended
+    title: "Data Pipeline Testing Requirements"
+    content: |
+      ## Data Pipeline Testing Requirements
+
+      ### Data Quality Assertions at Every Stage
+      Assert at each pipeline boundary — after extraction, after transformation, before loading:
+      - **Row count** — within expected range or ± % of prior run baseline.
+      - **Null rate** — per column, against stated threshold (e.g., `customer_id` must never be null).
+      - **Schema compliance** — column names, types, nullable flags match the declared schema.
+      - **Referential integrity** — foreign key values exist in the referenced table.
+      - **Distribution checks** — value ranges and cardinality within expected bounds. Alert on sudden shifts (> 20% deviation from trailing 7-day average).
+      Use Great Expectations, dbt tests, or equivalent. Assertions are first-class artifacts — committed, versioned, and enforced in CI.
+
+      ### Idempotency Tests
+      Re-running the pipeline with the same input must produce identical output with no side effects. Test explicitly:
+      - Run pipeline N once; capture output row count and checksum.
+      - Run pipeline N again with the same source data.
+      - Assert: output row count and checksum are identical; no duplicates in target; no reprocessed records in dead-letter queue.
+      Idempotency is a contract, not an assumption. If a stage is not idempotent by design, that decision requires an ADR.
+
+      ### Backfill Correctness Tests
+      Historical data processed against current transformation logic must produce correct results:
+      - Define a fixed historical fixture (date range, known input, known output).
+      - Run current pipeline logic against the fixture.
+      - Assert output matches expected historical output.
+      Any change to transformation logic that causes a backfill regression is a breaking change requiring an ADR.
+
+      ### Dead-Letter Queue Draining Tests
+      Malformed records must be handled — not silently dropped and not blocking the pipeline:
+      - Inject a record that violates the schema; assert it routes to the dead-letter table/queue.
+      - Inject a record that passes schema but fails a business rule; assert correct routing.
+      - Assert the DLQ record contains: original payload, validation error, run ID, timestamp.
+      - Test DLQ drain procedure: records can be corrected, resubmitted, and processed successfully.
+
+      ### Volume / Scale Tests at Staging
+      Unit-scale fixtures are insufficient for pipeline correctness. Run at representative data volume:
+      - At staging: use a realistic data volume (minimum 10% of production record count, or a known large-batch scenario).
+      - Assert: completion within SLA window; no memory exhaustion; partition output correct.
+      - Document the scale test fixture size and the SLA threshold in the spec.

package/templates/data-pipeline/mcp-servers.yaml

@@ -1,13 +1,13 @@
-tag: DATA-PIPELINE
-section: mcp-servers
-servers:
-  - name: postgres
-    description: "PostgreSQL database inspection, queries, and schema management"
-    command: npx
-    args: ["-y", "@modelcontextprotocol/server-postgres"]
-    tags: [DATA-PIPELINE, API]
-    category: database
-    tier: recommended
-    env:
-      POSTGRES_CONNECTION_STRING: ""
-    url: "https://github.com/modelcontextprotocol/servers/tree/main/src/postgres"
+tag: DATA-PIPELINE
+section: mcp-servers
+servers:
+  - name: postgres
+    description: "PostgreSQL database inspection, queries, and schema management"
+    command: npx
+    args: ["-y", "@modelcontextprotocol/server-postgres"]
+    tags: [DATA-PIPELINE, API]
+    category: database
+    tier: recommended
+    env:
+      POSTGRES_CONNECTION_STRING: ""
+    url: "https://github.com/modelcontextprotocol/servers/tree/main/src/postgres"

package/templates/data-pipeline/nfr.yaml

@@ -1,39 +1,39 @@
-tag: DATA-PIPELINE
-section: nfr
-blocks:
-  - id: pipeline-reliability
-    tier: recommended
-    title: "Pipeline Reliability"
-    content: |
-      ## NFR: Pipeline Reliability
-
-      ### SLAs
-      - Data freshness SLA: {{freshness_sla | default: 1 hour}} from source to destination.
-      - Pipeline success rate: ≥ {{pipeline_success_rate | default: 99%}} per day.
-      - Alert on SLA breach within 15 minutes.
-
-      ### Idempotency
-      - Every pipeline step is idempotent — safe to rerun without duplicating data.
-      - Backfill supported: reprocess historical date ranges without manual intervention.
-      - Partial failures resume from last checkpoint, not from scratch.
-
-      ### Data Quality
-      - Schema validation at ingestion. Reject or quarantine malformed records.
-      - Row count, null rate, uniqueness, and distribution checks after each stage.
-      - Data quality metrics tracked over time. Alert on anomalies.
-
-  - id: pipeline-performance
-    tier: recommended
-    title: "Pipeline Performance"
-    content: |
-      ## NFR: Pipeline Performance
-
-      ### Throughput
-      - Target throughput: {{pipeline_throughput | default: 100K records/minute}}.
-      - Batch vs. streaming decision documented with rationale.
-      - Partitioning strategy aligned with query patterns.
-
-      ### Resource Management
-      - Compute resources auto-scale with data volume. No over-provisioning for peak.
-      - Cost per GB processed tracked. Budget alerts at 80% threshold.
-      - Temporary/staging data cleaned up automatically after pipeline completion.
+tag: DATA-PIPELINE
+section: nfr
+blocks:
+  - id: pipeline-reliability
+    tier: recommended
+    title: "Pipeline Reliability"
+    content: |
+      ## NFR: Pipeline Reliability
+
+      ### SLAs
+      - Data freshness SLA: {{freshness_sla | default: 1 hour}} from source to destination.
+      - Pipeline success rate: ≥ {{pipeline_success_rate | default: 99%}} per day.
+      - Alert on SLA breach within 15 minutes.
+
+      ### Idempotency
+      - Every pipeline step is idempotent — safe to rerun without duplicating data.
+      - Backfill supported: reprocess historical date ranges without manual intervention.
+      - Partial failures resume from last checkpoint, not from scratch.
+
+      ### Data Quality
+      - Schema validation at ingestion. Reject or quarantine malformed records.
+      - Row count, null rate, uniqueness, and distribution checks after each stage.
+      - Data quality metrics tracked over time. Alert on anomalies.
+
+  - id: pipeline-performance
+    tier: recommended
+    title: "Pipeline Performance"
+    content: |
+      ## NFR: Pipeline Performance
+
+      ### Throughput
+      - Target throughput: {{pipeline_throughput | default: 100K records/minute}}.
+      - Batch vs. streaming decision documented with rationale.
+      - Partitioning strategy aligned with query patterns.
+
+      ### Resource Management
+      - Compute resources auto-scale with data volume. No over-provisioning for peak.
+      - Cost per GB processed tracked. Budget alerts at 80% threshold.
+      - Temporary/staging data cleaned up automatically after pipeline completion.

package/templates/data-pipeline/structure.yaml

@@ -1,23 +1,23 @@
-tag: DATA-PIPELINE
-section: structure
-entries:
-  - path: dags/
-    description: "Pipeline orchestration definitions (Airflow DAGs, Prefect flows, Dagster jobs)"
-  - path: src/extractors/
-    description: "Data source connectors: API, database, file extractors"
-  - path: src/transformers/
-    description: "Data transformation logic: cleaning, enrichment, aggregation"
-  - path: src/loaders/
-    description: "Data sinks: warehouse loaders, file writers, API publishers"
-  - path: src/validators/
-    description: "Data quality checks: schema validation, business rules, anomaly detection"
-  - path: src/config/
-    description: "Pipeline configuration: sources, schedules, thresholds (YAML)"
-  - path: tests/
-    description: "Unit tests for transformers, validators, and business logic"
-  - path: tests/fixtures/
-    description: "Sample data files for deterministic testing"
-  - path: sql/
-    description: "SQL transformations and DDL for warehouse tables"
-  - path: scripts/
-    description: "Operational scripts: backfill, reprocess, data repair"
+tag: DATA-PIPELINE
+section: structure
+entries:
+  - path: dags/
+    description: "Pipeline orchestration definitions (Airflow DAGs, Prefect flows, Dagster jobs)"
+  - path: src/extractors/
+    description: "Data source connectors: API, database, file extractors"
+  - path: src/transformers/
+    description: "Data transformation logic: cleaning, enrichment, aggregation"
+  - path: src/loaders/
+    description: "Data sinks: warehouse loaders, file writers, API publishers"
+  - path: src/validators/
+    description: "Data quality checks: schema validation, business rules, anomaly detection"
+  - path: src/config/
+    description: "Pipeline configuration: sources, schedules, thresholds (YAML)"
+  - path: tests/
+    description: "Unit tests for transformers, validators, and business logic"
+  - path: tests/fixtures/
+    description: "Sample data files for deterministic testing"
+  - path: sql/
+    description: "SQL transformations and DDL for warehouse tables"
+  - path: scripts/
+    description: "Operational scripts: backfill, reprocess, data repair"