@grainulation/silo 1.0.0

package/packs/ci-cd.json

@@ -0,0 +1,175 @@
+{
+  "name": "CI/CD and Deployment",
+  "description": "Pipeline design patterns, deployment strategies (blue-green, canary, rolling), feature flags, rollback procedures, and artifact management for continuous delivery.",
+  "version": "1.0.0",
+  "claims": [
+    {
+      "id": "cicd-001",
+      "type": "constraint",
+      "topic": "pipeline time budget",
+      "content": "CI pipelines should complete in under 10 minutes for the critical path (lint, test, build). Pipelines over 15 minutes measurably reduce PR throughput and encourage developers to batch changes, increasing merge conflict risk. Parallelize test suites and cache dependencies aggressively.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "production",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "pipeline", "performance", "developer-experience"]
+    },
+    {
+      "id": "cicd-002",
+      "type": "factual",
+      "topic": "blue-green deployment mechanics",
+      "content": "Blue-green deployment maintains two identical production environments. Traffic routes to blue (current) while green receives the new version. After green passes health checks, traffic switches via load balancer. Rollback is instant (switch back to blue). Cost: 2x infrastructure during deployment window. Database migrations must be backwards-compatible since both versions share the database.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "deployment", "blue-green", "rollback"]
+    },
+    {
+      "id": "cicd-003",
+      "type": "recommendation",
+      "topic": "canary deployment percentage",
+      "content": "Canary deployments should start at 1-5% of traffic, bake for 10-15 minutes while monitoring error rates and latency, then expand to 25%, 50%, 100% with automated rollback if error rate exceeds 1% above baseline or p99 latency increases by more than 50%. Full rollout should take 30-60 minutes minimum.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "production",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "deployment", "canary", "progressive-rollout"]
+    },
+    {
+      "id": "cicd-004",
+      "type": "risk",
+      "topic": "database migration coupling",
+      "content": "Deploying application code and database migrations simultaneously causes failures if the migration is slow or fails mid-deploy. Decouple migrations: deploy migration first (must be backwards-compatible with current code), verify, then deploy application code. Use expand-and-contract pattern for schema changes.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "production",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "database", "migrations", "deployment"]
+    },
+    {
+      "id": "cicd-005",
+      "type": "recommendation",
+      "topic": "feature flag lifecycle",
+      "content": "Feature flags should have an owner, a creation date, and a planned removal date. Flags older than 90 days without removal should trigger a tech debt alert. Stale feature flags accumulate as dead code paths and make testing combinatorially complex. Use a flag management service (LaunchDarkly, Unleash, or custom) rather than config files.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "feature-flags", "tech-debt", "lifecycle"]
+    },
+    {
+      "id": "cicd-006",
+      "type": "constraint",
+      "topic": "immutable build artifacts",
+      "content": "Build artifacts must be immutable: the exact same artifact (Docker image, binary, package) that passes CI is the one deployed to staging and production. Never rebuild for different environments. Use environment variables or config injection for environment-specific settings. Tag artifacts with git SHA for traceability.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "artifacts", "immutability", "docker"]
+    },
+    {
+      "id": "cicd-007",
+      "type": "risk",
+      "topic": "rollback without data rollback",
+      "content": "Code rollbacks do not roll back data. If version N+1 wrote data in a new format, rolling back to version N may leave incompatible data in the database. Design writes to be forward-compatible: version N should tolerate data written by N+1. This constraint makes additive-only schema changes critical.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "production",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "rollback", "database", "data-compatibility"]
+    },
+    {
+      "id": "cicd-008",
+      "type": "factual",
+      "topic": "DORA metrics benchmarks",
+      "content": "DORA elite team benchmarks (2023): deployment frequency (on-demand, multiple per day), lead time for changes (less than 1 hour), mean time to restore (less than 1 hour), change failure rate (0-15%). High performers deploy 973x more frequently than low performers. Measure these four metrics to assess CI/CD health.",
+      "source": { "origin": "industry", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "dora", "metrics", "engineering-effectiveness"]
+    },
+    {
+      "id": "cicd-009",
+      "type": "recommendation",
+      "topic": "trunk-based development",
+      "content": "Trunk-based development (short-lived branches merged to main within 1-2 days) correlates with higher DORA metrics than Gitflow. Long-lived feature branches (over 3 days) have exponentially higher merge conflict probability. Use feature flags instead of feature branches for incomplete work.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "branching", "trunk-based", "gitflow"]
+    },
+    {
+      "id": "cicd-010",
+      "type": "recommendation",
+      "topic": "test pyramid enforcement",
+      "content": "Enforce the test pyramid in CI: 70% unit tests (fast, isolated), 20% integration tests (database, APIs), 10% end-to-end tests (browser, full stack). Inverted pyramids (heavy E2E, few unit tests) result in slow, flaky pipelines. Quarantine flaky tests immediately rather than retrying them.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "testing", "test-pyramid", "flaky-tests"]
+    },
+    {
+      "id": "cicd-011",
+      "type": "risk",
+      "topic": "secret exposure in CI logs",
+      "content": "CI systems frequently leak secrets in build logs through debug output, error messages, or environment variable dumps. Use secret masking features in your CI platform. Never pass secrets as command-line arguments (visible in process listings). Audit CI logs quarterly for accidental secret exposure.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "production",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "security", "secrets", "logs"]
+    },
+    {
+      "id": "cicd-012",
+      "type": "estimate",
+      "topic": "deploy frequency impact on MTTR",
+      "content": "Teams deploying daily have a mean time to restore (MTTR) of 1-4 hours. Teams deploying weekly average 1-3 days MTTR. Teams deploying monthly average 1-2 weeks MTTR. Smaller, more frequent deploys are easier to diagnose because the blast radius of each change is smaller.",
+      "source": { "origin": "industry", "artifact": null, "connector": null },
+      "evidence": "web",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["ci-cd", "deploy-frequency", "mttr", "reliability"]
+    }
+  ]
+}

package/packs/compliance.json

@@ -0,0 +1,203 @@
+{
+  "name": "Compliance Constraints",
+  "description": "Regulatory constraint sets for HIPAA, SOC 2, and GDPR research sprints. Use as a starting point when evaluating compliance-sensitive architecture or vendor decisions.",
+  "version": "1.0.0",
+  "claims": [
+    {
+      "id": "hipaa-001",
+      "type": "constraint",
+      "topic": "HIPAA encryption at rest",
+      "content": "Protected Health Information (PHI) must be encrypted at rest using AES-256 or equivalent, per HIPAA Security Rule 45 CFR 164.312(a)(2)(iv).",
+      "source": { "origin": "regulation", "artifact": "45 CFR 164.312", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["hipaa", "encryption", "phi"]
+    },
+    {
+      "id": "hipaa-002",
+      "type": "constraint",
+      "topic": "HIPAA encryption in transit",
+      "content": "PHI in transit must be encrypted via TLS 1.2 or higher. Unencrypted transmission channels (HTTP, FTP) are prohibited for PHI.",
+      "source": { "origin": "regulation", "artifact": "45 CFR 164.312(e)(1)", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["hipaa", "encryption", "transit"]
+    },
+    {
+      "id": "hipaa-003",
+      "type": "constraint",
+      "topic": "HIPAA business associate agreements",
+      "content": "A Business Associate Agreement (BAA) must be signed with every third-party vendor that processes, stores, or transmits PHI, including cloud providers.",
+      "source": { "origin": "regulation", "artifact": "45 CFR 164.502(e)", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["hipaa", "baa", "vendor"]
+    },
+    {
+      "id": "hipaa-004",
+      "type": "constraint",
+      "topic": "HIPAA minimum necessary access",
+      "content": "Access to PHI must follow minimum necessary standard — users see only the data required for their role. Role-based access control (RBAC) is the typical implementation.",
+      "source": { "origin": "regulation", "artifact": "45 CFR 164.502(b)", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["hipaa", "access-control", "rbac"]
+    },
+    {
+      "id": "hipaa-005",
+      "type": "constraint",
+      "topic": "HIPAA audit log retention",
+      "content": "Audit logs of all PHI access events must be retained for a minimum of 6 years. Logs must include user identity, timestamp, and action performed.",
+      "source": { "origin": "regulation", "artifact": "45 CFR 164.312(b)", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["hipaa", "audit", "logging"]
+    },
+    {
+      "id": "soc2-001",
+      "type": "constraint",
+      "topic": "SOC 2 Type II observation period",
+      "content": "SOC 2 Type II requires demonstrating that security controls operated effectively over a minimum observation period of 6 months (typically 12 months).",
+      "source": { "origin": "standard", "artifact": "AICPA Trust Services Criteria", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["soc2", "audit", "controls"]
+    },
+    {
+      "id": "soc2-002",
+      "type": "constraint",
+      "topic": "SOC 2 change management",
+      "content": "Change management controls must track all production changes with approval records, rollback plans, and post-deployment verification.",
+      "source": { "origin": "standard", "artifact": "AICPA CC8.1", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["soc2", "change-management", "controls"]
+    },
+    {
+      "id": "soc2-003",
+      "type": "constraint",
+      "topic": "SOC 2 incident response",
+      "content": "Incident response plan must define severity levels, escalation paths, communication templates, and post-incident review process. Annual tabletop exercises required.",
+      "source": { "origin": "standard", "artifact": "AICPA CC7.3-CC7.5", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["soc2", "incident-response", "controls"]
+    },
+    {
+      "id": "soc2-004",
+      "type": "constraint",
+      "topic": "SOC 2 access reviews",
+      "content": "Logical access reviews must be performed quarterly. Terminated employee access must be revoked within 24 hours of separation.",
+      "source": { "origin": "standard", "artifact": "AICPA CC6.1-CC6.3", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["soc2", "access-control", "offboarding"]
+    },
+    {
+      "id": "gdpr-001",
+      "type": "constraint",
+      "topic": "GDPR cross-border data transfer",
+      "content": "Personal data of EU residents cannot be transferred outside the EEA without an adequacy decision, Standard Contractual Clauses (SCCs), or Binding Corporate Rules.",
+      "source": { "origin": "regulation", "artifact": "GDPR Articles 44-49", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["gdpr", "data-transfer", "international"]
+    },
+    {
+      "id": "gdpr-002",
+      "type": "constraint",
+      "topic": "GDPR right to erasure",
+      "content": "Data subjects have the right to erasure ('right to be forgotten'). Systems must support complete deletion of a user's personal data within 30 days of request.",
+      "source": { "origin": "regulation", "artifact": "GDPR Article 17", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["gdpr", "erasure", "data-subject-rights"]
+    },
+    {
+      "id": "gdpr-003",
+      "type": "constraint",
+      "topic": "GDPR breach notification",
+      "content": "Data breaches affecting personal data must be reported to the supervisory authority within 72 hours of discovery. Affected individuals must be notified without undue delay if risk is high.",
+      "source": { "origin": "regulation", "artifact": "GDPR Articles 33-34", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["gdpr", "breach-notification", "incident-response"]
+    },
+    {
+      "id": "gdpr-004",
+      "type": "constraint",
+      "topic": "GDPR lawful basis for processing",
+      "content": "Processing personal data requires a lawful basis (consent, contract, legal obligation, vital interest, public task, or legitimate interest). Consent must be freely given, specific, and withdrawable.",
+      "source": { "origin": "regulation", "artifact": "GDPR Article 6", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["gdpr", "lawful-basis", "consent"]
+    },
+    {
+      "id": "gdpr-005",
+      "type": "constraint",
+      "topic": "GDPR data protection impact assessment",
+      "content": "A Data Protection Impact Assessment (DPIA) is mandatory before processing that is likely to result in high risk to individuals, including large-scale profiling or systematic monitoring.",
+      "source": { "origin": "regulation", "artifact": "GDPR Article 35", "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["gdpr", "dpia", "risk-assessment"]
+    }
+  ]
+}

package/packs/data-engineering.json

@@ -0,0 +1,175 @@
+{
+  "name": "Data Engineering",
+  "description": "ETL vs ELT patterns, data quality enforcement, schema evolution strategies, batch vs streaming tradeoffs, data lake anti-patterns, and change data capture.",
+  "version": "1.0.0",
+  "claims": [
+    {
+      "id": "data-001",
+      "type": "factual",
+      "topic": "ETL vs ELT tradeoffs",
+      "content": "ETL (Extract-Transform-Load) transforms data before loading into the target, reducing storage costs but making the pipeline brittle to schema changes. ELT (Extract-Load-Transform) loads raw data first and transforms in the warehouse, leveraging cheap columnar storage and enabling reprocessing without re-extraction. ELT is the dominant pattern for cloud data warehouses (Snowflake, BigQuery, Redshift).",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "etl", "elt", "warehouse"]
+    },
+    {
+      "id": "data-002",
+      "type": "constraint",
+      "topic": "schema evolution compatibility",
+      "content": "Schema changes must be backwards-compatible for consumers: adding optional fields (safe), removing fields (breaking), renaming fields (breaking), changing types (breaking). Use schema registries (Confluent, AWS Glue) to enforce compatibility rules. Avro supports full forward/backward/full compatibility modes. Breaking changes require a new topic or table version.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "schema", "evolution", "avro"]
+    },
+    {
+      "id": "data-003",
+      "type": "recommendation",
+      "topic": "data quality checks in pipelines",
+      "content": "Every data pipeline should include automated quality checks: row count expectations (within 10% of previous run), null rate thresholds per column, uniqueness constraints on key columns, freshness checks (data arrived within expected window), and referential integrity across tables. Use tools like Great Expectations, dbt tests, or Soda to define checks as code.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "production",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "data-quality", "testing", "dbt"]
+    },
+    {
+      "id": "data-004",
+      "type": "risk",
+      "topic": "data lake swamp anti-pattern",
+      "content": "Data lakes degenerate into data swamps when: (a) data is dumped without schema documentation, (b) there is no catalog or discovery mechanism, (c) no access controls or data classification, (d) no retention policies, (e) no data quality monitoring. Within 12-18 months, undisciplined data lakes become untrusted and abandoned. Apply medallion architecture (bronze/silver/gold) to impose structure.",
+      "source": { "origin": "industry", "artifact": null, "connector": null },
+      "evidence": "web",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "data-lake", "anti-pattern", "governance"]
+    },
+    {
+      "id": "data-005",
+      "type": "factual",
+      "topic": "batch vs streaming latency tradeoffs",
+      "content": "Batch processing (hourly/daily) is simpler, cheaper, and sufficient when business requirements tolerate T+1 or T+hour latency. Streaming (Kafka, Kinesis, Flink) delivers sub-second latency but costs 3-10x more in infrastructure and 2-3x more in engineering complexity (exactly-once semantics, out-of-order events, state management). Default to batch unless latency requirements demand streaming.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "production",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "batch", "streaming", "architecture"]
+    },
+    {
+      "id": "data-006",
+      "type": "recommendation",
+      "topic": "change data capture pattern",
+      "content": "Change Data Capture (CDC) reads database transaction logs (binlog, WAL) to stream row-level changes to downstream systems. Use Debezium for open-source CDC from PostgreSQL, MySQL, MongoDB, and SQL Server. CDC avoids polling overhead, captures deletes (which polling misses), and preserves event ordering. Initial snapshot + streaming log is the standard bootstrap pattern.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "cdc", "debezium", "replication"]
+    },
+    {
+      "id": "data-007",
+      "type": "risk",
+      "topic": "late-arriving data in event streams",
+      "content": "Event streams contain late-arriving data due to network delays, mobile offline sync, or batch uploads. Windowed aggregations must handle late data with watermarks (maximum allowed lateness). Flink default watermark is 0 (no late data tolerance). Set watermarks based on observed p99 lateness in your data. Late events beyond the watermark are either dropped or routed to a dead-letter topic for manual reconciliation.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "production",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "streaming", "late-data", "watermarks"]
+    },
+    {
+      "id": "data-008",
+      "type": "recommendation",
+      "topic": "idempotent pipeline design",
+      "content": "Data pipelines must be idempotent: running the same pipeline twice with the same input produces the same output without duplicates. Implement with: write to a staging table, then MERGE/upsert to the target (not INSERT). Use partition overwrite for append-only tables. Idempotency enables safe retries after partial failures, which occur in approximately 5-10% of batch runs.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "production",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "idempotency", "pipeline", "reliability"]
+    },
+    {
+      "id": "data-009",
+      "type": "estimate",
+      "topic": "Parquet vs CSV storage savings",
+      "content": "Columnar formats (Parquet, ORC) reduce storage by 75-90% compared to CSV/JSON for analytical workloads, and query performance improves 10-100x due to column pruning and predicate pushdown. A 100 GB CSV dataset typically compresses to 5-15 GB in Parquet with Snappy compression. Always use Parquet or ORC for data lake storage, never raw CSV.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "tested",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "parquet", "storage", "performance"]
+    },
+    {
+      "id": "data-010",
+      "type": "constraint",
+      "topic": "PII handling in data pipelines",
+      "content": "PII must be classified, tagged, and handled according to data governance policy at ingestion time, not after the fact. Apply column-level encryption or tokenization for sensitive fields (SSN, email, phone). Implement row-level access controls in the warehouse. Maintain a data catalog that tracks PII lineage from source to all downstream tables. GDPR right-to-erasure requires the ability to delete a user across all derived datasets.",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "documented",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "pii", "gdpr", "governance"]
+    },
+    {
+      "id": "data-011",
+      "type": "factual",
+      "topic": "exactly-once semantics cost",
+      "content": "Exactly-once processing in streaming systems (Kafka transactions, Flink checkpointing) adds 10-30% throughput overhead compared to at-least-once. At-least-once with idempotent consumers (using unique event IDs and upsert writes) achieves the same end result with lower complexity. True exactly-once is only required when side effects cannot be made idempotent (sending emails, charging payments).",
+      "source": { "origin": "best-practice", "artifact": null, "connector": null },
+      "evidence": "tested",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "streaming", "exactly-once", "kafka"]
+    },
+    {
+      "id": "data-012",
+      "type": "recommendation",
+      "topic": "dbt for transformation layer",
+      "content": "Use dbt (data build tool) for the transformation layer in ELT pipelines. dbt provides: version-controlled SQL transformations, automatic DAG resolution, built-in testing (not_null, unique, accepted_values, relationships), documentation generation, and incremental materialization for large tables. It has become the industry standard for analytics engineering with adoption at over 30,000 companies.",
+      "source": { "origin": "industry", "artifact": null, "connector": null },
+      "evidence": "web",
+      "status": "active",
+      "phase_added": "define",
+      "timestamp": "2025-01-01T00:00:00.000Z",
+      "conflicts_with": [],
+      "resolved_by": null,
+      "tags": ["data-engineering", "dbt", "transformation", "analytics-engineering"]
+    }
+  ]
+}