@umacloud/knowledge 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/00-governance/governance-capabilities.md +557 -0
- package/00-governance/knowledge-map.md +39 -0
- package/00-governance/maintenance-policy.md +76 -0
- package/00-governance/review-checklist.md +81 -0
- package/README.md +13 -0
- package/ai/01-standards/agent-development-complete.md +691 -0
- package/ai/01-standards/llm-application-complete.md +488 -0
- package/ai/01-standards/mlops-complete.md +798 -0
- package/ai/01-standards/prompt-engineering-complete.md +646 -0
- package/ai/01-standards/rag-architecture-complete.md +649 -0
- package/ai/02-playbooks/llm-evaluation-playbook.md +847 -0
- package/ai/03-checklists/ai-project-checklist.md +215 -0
- package/ai/04-antipatterns/ai-antipatterns.md +661 -0
- package/ai/05-cases/case-rag-production.md +147 -0
- package/ai/06-glossary/ai-glossary.md +162 -0
- package/ai/agent-evaluation-benchmark.md +53 -0
- package/ai/ai-agent-memory-context-management.md +41 -0
- package/ai/ai-cost-capacity-optimization-playbook.md +42 -0
- package/ai/ai-data-security-and-compliance-playbook.md +37 -0
- package/ai/ai-domain-index-and-checklist.md +40 -0
- package/ai/ai-governance-maturity-model.md +50 -0
- package/ai/ai-model-selection-and-routing-strategy.md +47 -0
- package/ai/ai-observability-and-oncall-runbook.md +52 -0
- package/ai/ai-rag-engineering-playbook.md +42 -0
- package/ai/ai-red-team-and-safety-evaluation.md +42 -0
- package/ai/ai-release-readiness-and-rollback-gate.md +42 -0
- package/ai/llm-agent-engineering-deep-dive.md +57 -0
- package/ai/prompt-and-tool-guardrails.md +52 -0
- package/api/01-standards/enterprise-api-standards.md +198 -0
- package/api/01-standards/rest-api-design-guide.md +63 -0
- package/api/02-playbooks/api-pagination-playbook.md +93 -0
- package/api/02-playbooks/graphql-production-playbook.md +176 -0
- package/api/03-checklists/api-review-checklist.md +55 -0
- package/api/04-antipatterns/api-antipatterns.md +112 -0
- package/architecture/01-standards/api-gateway-patterns.md +496 -0
- package/architecture/01-standards/cloud-native-patterns.md +644 -0
- package/architecture/01-standards/distributed-systems-patterns.md +591 -0
- package/architecture/01-standards/event-driven-architecture.md +595 -0
- package/architecture/01-standards/microservices-patterns-complete.md +968 -0
- package/architecture/01-standards/microservices-patterns.md +495 -0
- package/architecture/01-standards/system-design-interview.md +664 -0
- package/architecture/02-playbooks/microservices-patterns-playbook.md +137 -0
- package/architecture/02-playbooks/migration-playbook.md +780 -0
- package/architecture/02-playbooks/system-design-playbook.md +779 -0
- package/architecture/03-checklists/architecture-decision-checklist.md +297 -0
- package/architecture/04-antipatterns/architecture-antipatterns.md +417 -0
- package/architecture/05-cases/case-netflix-microservices.md +413 -0
- package/architecture/06-glossary/architecture-glossary.md +164 -0
- package/architecture/adr-template-and-examples.md +38 -0
- package/architecture/api-gateway-deep-dive.md +1291 -0
- package/architecture/configuration-management.md +1162 -0
- package/architecture/distributed-transactions.md +1220 -0
- package/architecture/microservices-complete.md +735 -0
- package/architecture/resilience-and-disaster-patterns.md +37 -0
- package/architecture/service-governance.md +1198 -0
- package/architecture/system-architecture-deep-dive.md +37 -0
- package/backend/01-standards/analytics-and-growth.md +65 -0
- package/backend/01-standards/api-and-error-conventions.md +120 -0
- package/backend/01-standards/application-layering-and-packaging.md +160 -0
- package/backend/01-standards/auth-implementation.md +104 -0
- package/backend/01-standards/backend-framework-idioms.md +74 -0
- package/backend/01-standards/background-jobs-and-async.md +66 -0
- package/backend/01-standards/caching-strategies-complete.md +390 -0
- package/backend/01-standards/config-and-observability.md +77 -0
- package/backend/01-standards/data-modeling-and-persistence.md +94 -0
- package/backend/01-standards/django-complete.md +1765 -0
- package/backend/01-standards/email-and-notifications.md +64 -0
- package/backend/01-standards/fastapi-complete.md +925 -0
- package/backend/01-standards/file-upload-and-storage.md +66 -0
- package/backend/01-standards/graphql-api-complete.md +416 -0
- package/backend/01-standards/llm-application-standard.md +78 -0
- package/backend/01-standards/message-queue-patterns.md +379 -0
- package/backend/01-standards/microservices-and-distributed.md +78 -0
- package/backend/01-standards/nestjs-complete.md +2167 -0
- package/backend/01-standards/payment-integration.md +80 -0
- package/backend/01-standards/rate-limiting-complete.md +451 -0
- package/backend/01-standards/realtime-and-websocket.md +65 -0
- package/backend/01-standards/search-and-filtering.md +64 -0
- package/backend/01-standards/spring-boot-complete.md +445 -0
- package/backend/02-playbooks/api-design-playbook.md +718 -0
- package/backend/02-playbooks/email-send-playbook.md +130 -0
- package/backend/02-playbooks/file-upload-s3-playbook.md +153 -0
- package/backend/02-playbooks/typescript-enterprise-playbook.md +133 -0
- package/backend/02-playbooks/websocket-realtime-playbook.md +154 -0
- package/backend/03-checklists/api-launch-checklist.md +189 -0
- package/backend/04-antipatterns/backend-antipatterns.md +1051 -0
- package/blockchain/01-standards/blockchain-basics.md +557 -0
- package/blockchain/01-standards/smart-contract-development.md +1315 -0
- package/cicd/01-standards/deployment-and-delivery-standard.md +96 -0
- package/cicd/01-standards/github-actions-complete.md +473 -0
- package/cicd/01-standards/release-and-store-submission.md +75 -0
- package/cicd/02-playbooks/cicd-pipeline-playbook.md +144 -0
- package/cicd/02-playbooks/release-management-playbook.md +605 -0
- package/cicd/03-checklists/pipeline-security-checklist.md +168 -0
- package/cicd/04-antipatterns/cicd-antipatterns.md +589 -0
- package/cicd/05-cases/case-deployment-automation.md +221 -0
- package/cicd/05-cases/case-gitops-transformation.md +212 -0
- package/cicd/06-glossary/cicd-glossary.md +114 -0
- package/cicd/cicd-blueprint-deep-dive.md +38 -0
- package/cicd/release-readiness-gate.md +37 -0
- package/cloud-native/01-standards/container-security.md +741 -0
- package/cloud-native/01-standards/kubernetes-complete.md +812 -0
- package/cloud-native/02-playbooks/api-gateway-playbook.md +155 -0
- package/cloud-native/02-playbooks/gitops-with-argocd.md +760 -0
- package/cloud-native/02-playbooks/k8s-troubleshooting-playbook.md +1942 -0
- package/cloud-native/02-playbooks/message-queue-playbook.md +129 -0
- package/cloud-native/02-playbooks/multicloud-governance.md +726 -0
- package/cloud-native/02-playbooks/serverless-patterns.md +788 -0
- package/cloud-native/02-playbooks/service-mesh-playbook.md +612 -0
- package/cloud-native/02-playbooks/terraform-iac-playbook.md +143 -0
- package/cloud-native/03-checklists/container-security-checklist.md +431 -0
- package/cloud-native/03-checklists/k8s-production-readiness-checklist.md +460 -0
- package/cloud-native/04-antipatterns/container-antipatterns.md +660 -0
- package/cloud-native/04-antipatterns/k8s-antipatterns.md +743 -0
- package/cloud-native/05-cases/case-k8s-migration.md +478 -0
- package/cloud-native/05-cases/case-k8s-scaling.md +642 -0
- package/cloud-native/05-cases/case-k8s-security-incident.md +397 -0
- package/cloud-native/06-glossary/cloud-native-glossary.md +337 -0
- package/cross-platform/01-standards/cross-platform-frameworks.md +83 -0
- package/cross-platform/01-standards/platform-selection-and-architecture.md +77 -0
- package/data/01-standards/elasticsearch-complete.md +2098 -0
- package/data/01-standards/postgresql-complete.md +1613 -0
- package/data/01-standards/redis-complete.md +1527 -0
- package/data/02-playbooks/database-optimization-playbook.md +403 -0
- package/data/02-playbooks/elasticsearch-production-playbook.md +132 -0
- package/data/03-checklists/database-launch-checklist.md +187 -0
- package/data/04-antipatterns/database-antipatterns.md +873 -0
- package/data/05-cases/case-database-migration.md +310 -0
- package/data/06-glossary/database-glossary.md +440 -0
- package/data/data-governance-and-modeling-deep-dive.md +39 -0
- package/data-engineering/01-standards/airflow-complete.md +523 -0
- package/data-engineering/01-standards/kafka-complete.md +1521 -0
- package/data-engineering/02-playbooks/spark-etl-playbook.md +496 -0
- package/data-engineering/03-checklists/pipeline-launch-checklist.md +194 -0
- package/data-engineering/04-antipatterns/data-pipeline-antipatterns.md +684 -0
- package/data-engineering/05-cases/case-real-time-pipeline.md +355 -0
- package/data-engineering/06-glossary/data-engineering-glossary.md +429 -0
- package/database/01-standards/database-schema-standards.md +147 -0
- package/database/02-playbooks/postgresql-optimization-quick.md +52 -0
- package/database/02-playbooks/postgresql-performance-optimization.md +58 -0
- package/database/02-playbooks/postgresql-production-playbook.md +146 -0
- package/database/02-playbooks/redis-caching-playbook.md +117 -0
- package/database/03-checklists/database-review-checklist.md +50 -0
- package/database/04-antipatterns/database-antipatterns.md +112 -0
- package/design/01-standards/ui-design-system-complete.md +423 -0
- package/design/02-playbooks/design-handoff-playbook.md +254 -0
- package/design/02-playbooks/design-review-playbook.md +388 -0
- package/design/03-checklists/design-review-checklist.md +246 -0
- package/design/04-antipatterns/design-antipatterns.md +378 -0
- package/design/05-cases/case-design-system-adoption.md +328 -0
- package/design/06-glossary/design-glossary.md +329 -0
- package/design/ui-full-lifecycle-cross-platform-playbook.md +571 -0
- package/design/ux-system-deep-dive.md +38 -0
- package/design-systems/00-craft-rules.md +71 -0
- package/design-systems/aesthetic-families.md +43 -0
- package/design-systems/anti-ai-slop.md +162 -0
- package/design-systems/bold-geometric.md +120 -0
- package/design-systems/brutalist-bold.md +103 -0
- package/design-systems/editorial-clean.md +109 -0
- package/design-systems/glass-aurora.md +108 -0
- package/design-systems/modern-minimal.md +145 -0
- package/design-systems/premium-luxury.md +106 -0
- package/design-systems/product-type-design-map.md +48 -0
- package/design-systems/soft-warm.md +123 -0
- package/design-systems/tech-utility.md +113 -0
- package/desktop/01-standards/desktop-app-standard.md +72 -0
- package/desktop/01-standards/desktop-design.md +71 -0
- package/development/00-governance/document-template.md +41 -0
- package/development/01-standards/api-versioning-strategies.md +432 -0
- package/development/01-standards/authentication-patterns-complete.md +479 -0
- package/development/01-standards/css-architecture-complete.md +550 -0
- package/development/01-standards/database-migration-strategies.md +484 -0
- package/development/01-standards/elasticsearch-complete.md +347 -0
- package/development/01-standards/git-complete.md +371 -0
- package/development/01-standards/golang-complete.md +1565 -0
- package/development/01-standards/graphql-complete.md +298 -0
- package/development/01-standards/javascript-bundlers-complete.md +469 -0
- package/development/01-standards/javascript-typescript-complete.md +528 -0
- package/development/01-standards/jest-complete.md +275 -0
- package/development/01-standards/linux-complete.md +234 -0
- package/development/01-standards/logging-observability-complete.md +526 -0
- package/development/01-standards/microservices-communication.md +502 -0
- package/development/01-standards/mongodb-complete.md +406 -0
- package/development/01-standards/oauth2-complete.md +285 -0
- package/development/01-standards/performance-optimization-complete.md +289 -0
- package/development/01-standards/playwright-complete.md +247 -0
- package/development/01-standards/postgresql-complete.md +456 -0
- package/development/01-standards/pytest-complete.md +340 -0
- package/development/01-standards/python-async-programming.md +902 -0
- package/development/01-standards/python-complete.md +956 -0
- package/development/01-standards/python-decorators-complete.md +799 -0
- package/development/01-standards/python-design-patterns.md +2854 -0
- package/development/01-standards/python-packaging-distribution.md +420 -0
- package/development/01-standards/python-testing-strategies.md +607 -0
- package/development/01-standards/python-web-frameworks-comparison.md +471 -0
- package/development/01-standards/redis-complete.md +317 -0
- package/development/01-standards/rest-api-complete.md +316 -0
- package/development/01-standards/rust-complete.md +578 -0
- package/development/01-standards/typescript-advanced-types.md +1513 -0
- package/development/01-standards/web-security-complete.md +292 -0
- package/development/02-playbooks/api-design-playbook.md +810 -0
- package/development/02-playbooks/database-migration-playbook.md +580 -0
- package/development/02-playbooks/debugging-playbook.md +692 -0
- package/development/02-playbooks/feature-delivery-playbook.md +430 -0
- package/development/02-playbooks/incident-hotfix-playbook.md +387 -0
- package/development/02-playbooks/performance-optimization-playbook.md +531 -0
- package/development/02-playbooks/performance-tuning-playbook.md +652 -0
- package/development/02-playbooks/refactor-playbook.md +403 -0
- package/development/02-playbooks/release-playbook.md +469 -0
- package/development/03-checklists/architecture-review-checklist.md +168 -0
- package/development/03-checklists/data-migration-checklist.md +157 -0
- package/development/03-checklists/oncall-handover-checklist.md +173 -0
- package/development/03-checklists/pr-checklist.md +158 -0
- package/development/03-checklists/production-readiness-checklist.md +190 -0
- package/development/03-checklists/release-readiness-checklist.md +154 -0
- package/development/03-checklists/security-review-checklist.md +182 -0
- package/development/04-antipatterns/api-antipatterns.md +657 -0
- package/development/04-antipatterns/architecture-antipatterns.md +686 -0
- package/development/04-antipatterns/backend-antipatterns.md +648 -0
- package/development/04-antipatterns/cicd-antipatterns.md +540 -0
- package/development/04-antipatterns/code-smell-antipatterns.md +571 -0
- package/development/04-antipatterns/data-antipatterns.md +658 -0
- package/development/04-antipatterns/database-antipatterns.md +578 -0
- package/development/04-antipatterns/frontend-antipatterns.md +635 -0
- package/development/04-antipatterns/reliability-antipatterns.md +700 -0
- package/development/04-antipatterns/security-antipatterns.md +747 -0
- package/development/05-cases/case-api-version-migration.md +428 -0
- package/development/05-cases/case-authorization-hardening.md +383 -0
- package/development/05-cases/case-bluegreen-rollback.md +466 -0
- package/development/05-cases/case-cache-snowball-protection.md +485 -0
- package/development/05-cases/case-ci-cd-pipeline.md +544 -0
- package/development/05-cases/case-database-scaling.md +500 -0
- package/development/05-cases/case-db-hotspot-optimization.md +487 -0
- package/development/05-cases/case-incident-mttr-reduction.md +563 -0
- package/development/05-cases/case-microservice-migration.md +375 -0
- package/development/05-cases/case-performance-optimization.md +406 -0
- package/development/05-cases/case-security-incident-response.md +345 -0
- package/development/06-glossary/full-stack-glossary.md +166 -0
- package/development/09-maturity/quarterly-audit-template.md +35 -0
- package/development/11-ui-excellence/ui-aesthetic-system.md +41 -0
- package/development/11-ui-excellence/ui-engineering-excellence.md +435 -0
- package/development/12-scenarios/development-scenarios-guide.md +565 -0
- package/development/13-implementation-assets/implementation-toolkit.md +282 -0
- package/development/13-implementation-assets/knowledge-gates-execution.md +43 -0
- package/development/14-full-lifecycle/software-lifecycle-gates.md +511 -0
- package/development/15-lifecycle-templates/project-templates-collection.md +791 -0
- package/development/api-contract-and-versioning-guide.md +36 -0
- package/development/api-governance-complete.md +43 -0
- package/development/backend-engineering-complete.md +43 -0
- package/development/code-review-quality-complete.md +43 -0
- package/development/concurrency-reliability-complete.md +43 -0
- package/development/database-engineering-complete.md +43 -0
- package/development/engineering-effectiveness-complete.md +43 -0
- package/development/engineering-standards-deep-dive.md +38 -0
- package/development/frontend-engineering-complete.md +43 -0
- package/development/performance-capacity-complete.md +43 -0
- package/development/refactor-migration-complete.md +42 -0
- package/development/refactoring-and-techdebt-playbook.md +37 -0
- package/development/security-in-development-complete.md +43 -0
- package/devops/01-standards/cicd-pipeline-complete.md +262 -0
- package/devops/01-standards/docker-complete.md +1490 -0
- package/devops/01-standards/github-actions-complete.md +337 -0
- package/devops/01-standards/kubernetes-complete.md +638 -0
- package/devops/01-standards/terraform-complete.md +2117 -0
- package/devops/02-playbooks/docker-compose-playbook.md +233 -0
- package/devops/02-playbooks/docker-k8s-production-playbook.md +186 -0
- package/devops/02-playbooks/docker-production-playbook.md +952 -0
- package/edge-iot/01-standards/edge-iot-complete.md +473 -0
- package/experts/architect/api-design.md +178 -0
- package/experts/architect/methodology.md +124 -0
- package/experts/architect/security.md +75 -0
- package/experts/backend-lead/methodology.md +216 -0
- package/experts/devops/methodology.md +160 -0
- package/experts/frontend-lead/methodology.md +178 -0
- package/experts/product-manager/industry/ecommerce.md +43 -0
- package/experts/product-manager/industry/saas.md +40 -0
- package/experts/product-manager/methodology.md +97 -0
- package/experts/qa-lead/methodology.md +123 -0
- package/experts/qa-lead/test-strategy.md +128 -0
- package/experts/uiux-designer/methodology.md +125 -0
- package/frontend/01-standards/accessibility-complete.md +532 -0
- package/frontend/01-standards/accessibility-standard.md +74 -0
- package/frontend/01-standards/admin-dashboard-and-crud.md +72 -0
- package/frontend/01-standards/design-tokens-complete.md +444 -0
- package/frontend/01-standards/forms-and-validation.md +77 -0
- package/frontend/01-standards/frontend-architecture-and-layering.md +119 -0
- package/frontend/01-standards/i18n-and-localization.md +65 -0
- package/frontend/01-standards/nextjs-complete.md +451 -0
- package/frontend/01-standards/react-complete.md +713 -0
- package/frontend/01-standards/react-hooks-complete-guide.md +1100 -0
- package/frontend/01-standards/react-hooks-complete.md +1171 -0
- package/frontend/01-standards/seo-and-web-vitals.md +77 -0
- package/frontend/01-standards/state-management-complete.md +444 -0
- package/frontend/01-standards/vue-complete.md +499 -0
- package/frontend/01-standards/vue3-complete.md +2002 -0
- package/frontend/01-standards/web-framework-best-practices.md +64 -0
- package/frontend/01-standards/web-performance-complete.md +495 -0
- package/frontend/02-playbooks/accessibility-a11y-playbook.md +161 -0
- package/frontend/02-playbooks/frontend-performance-playbook.md +707 -0
- package/frontend/02-playbooks/i18n-internationalization-playbook.md +120 -0
- package/frontend/02-playbooks/performance-optimization-playbook.md +163 -0
- package/frontend/02-playbooks/react-nextjs-production-playbook.md +167 -0
- package/frontend/02-playbooks/react-state-management-playbook.md +173 -0
- package/frontend/03-checklists/component-quality-checklist.md +166 -0
- package/frontend/03-checklists/frontend-launch-checklist.md +299 -0
- package/frontend/04-antipatterns/frontend-antipatterns.md +886 -0
- package/frontend/05-cases/case-performance-optimization.md +274 -0
- package/harmony/01-standards/harmonyos-arkts-standard.md +75 -0
- package/harmony/01-standards/harmonyos-design.md +65 -0
- package/high-quality-engineering-playbook.md +54 -0
- package/incident/01-standards/incident-response-complete.md +303 -0
- package/incident/02-playbooks/chaos-engineering-playbook.md +883 -0
- package/incident/02-playbooks/postmortem-playbook.md +398 -0
- package/incident/03-checklists/incident-readiness-checklist.md +181 -0
- package/incident/04-antipatterns/incident-antipatterns.md +490 -0
- package/incident/05-cases/case-cascade-failure.md +176 -0
- package/incident/06-glossary/incident-glossary.md +114 -0
- package/incident/postmortem-and-response-deep-dive.md +39 -0
- package/industries/ecommerce/ecommerce-complete.md +631 -0
- package/industries/education/education-complete.md +555 -0
- package/industries/fintech/fintech-complete.md +501 -0
- package/industries/gaming/gaming-complete.md +587 -0
- package/industries/healthcare/healthcare-complete.md +452 -0
- package/low-code/01-standards/low-code-complete.md +944 -0
- package/miniprogram/01-standards/ai-common-mistakes.md +61 -0
- package/miniprogram/01-standards/miniprogram-custom-navbar-capsule.md +77 -0
- package/miniprogram/01-standards/miniprogram-design.md +61 -0
- package/miniprogram/01-standards/miniprogram-standard.md +81 -0
- package/mobile/01-standards/android-material-design.md +70 -0
- package/mobile/01-standards/flutter-complete.md +384 -0
- package/mobile/01-standards/ios-design-hig.md +78 -0
- package/mobile/01-standards/mobile-app-standard.md +85 -0
- package/mobile/01-standards/react-native-complete.md +352 -0
- package/mobile/02-playbooks/mobile-cross-platform-playbook.md +175 -0
- package/mobile/02-playbooks/mobile-performance.md +473 -0
- package/mobile/03-checklists/mobile-release-checklist.md +234 -0
- package/mobile/04-antipatterns/mobile-antipatterns.md +798 -0
- package/mobile/05-cases/case-app-performance.md +500 -0
- package/mobile/05-cases/case-app-startup-optimization.md +218 -0
- package/mobile/06-glossary/mobile-glossary.md +484 -0
- package/observability/01-standards/observability-standards.md +103 -0
- package/observability/02-playbooks/prometheus-grafana-playbook.md +135 -0
- package/observability/02-playbooks/structured-logging-playbook.md +73 -0
- package/observability/03-checklists/observability-checklist.md +54 -0
- package/observability/04-antipatterns/observability-antipatterns.md +106 -0
- package/operations/01-standards/prometheus-monitoring-complete.md +1578 -0
- package/operations/02-playbooks/capacity-planning-playbook.md +620 -0
- package/operations/03-checklists/production-launch-checklist.md +365 -0
- package/operations/04-antipatterns/operations-antipatterns.md +664 -0
- package/operations/05-cases/case-sre-practices.md +581 -0
- package/operations/06-glossary/operations-glossary.md +120 -0
- package/operations/aiops-anomaly-detection.md +758 -0
- package/operations/capacity-planning.md +1061 -0
- package/operations/chaos-engineering.md +659 -0
- package/operations/incident-command-system.md +38 -0
- package/operations/observability-complete.md +442 -0
- package/operations/slo-sli-playbook.md +517 -0
- package/operations/sre-operations-deep-dive.md +39 -0
- package/package.json +8 -0
- package/performance/01-standards/performance-and-scalability.md +80 -0
- package/performance/01-standards/performance-standards.md +156 -0
- package/performance/02-playbooks/query-optimization-playbook.md +103 -0
- package/performance/03-checklists/performance-checklist.md +56 -0
- package/performance/04-antipatterns/performance-antipatterns.md +146 -0
- package/product/01-standards/product-management-complete.md +285 -0
- package/product/02-playbooks/feature-launch-playbook.md +207 -0
- package/product/02-playbooks/user-research-playbook.md +532 -0
- package/product/03-checklists/feature-launch-checklist.md +275 -0
- package/product/04-antipatterns/product-antipatterns.md +355 -0
- package/product/05-cases/case-mvp-to-scale.md +384 -0
- package/product/06-glossary/product-glossary.md +462 -0
- package/product/feature-prioritization-framework.md +40 -0
- package/product/kpi-and-metric-tree.md +37 -0
- package/product/product-discovery-and-prd-deep-dive.md +41 -0
- package/quantum/01-standards/quantum-complete.md +1186 -0
- package/security/01-standards/api-security-complete.md +511 -0
- package/security/01-standards/container-runtime-security.md +574 -0
- package/security/01-standards/data-protection-gdpr.md +543 -0
- package/security/01-standards/owasp-top10-complete.md +1890 -0
- package/security/01-standards/secure-coding-baseline.md +90 -0
- package/security/01-standards/supply-chain-security.md +441 -0
- package/security/01-standards/web-security-checklist.md +108 -0
- package/security/01-standards/zero-trust-architecture.md +521 -0
- package/security/02-playbooks/auth-sso-playbook.md +166 -0
- package/security/02-playbooks/incident-response-security-playbook.md +588 -0
- package/security/02-playbooks/owasp-api-security-playbook.md +129 -0
- package/security/02-playbooks/payment-integration-playbook.md +119 -0
- package/security/02-playbooks/penetration-testing-playbook.md +517 -0
- package/security/03-checklists/security-audit-checklist.md +356 -0
- package/security/04-antipatterns/security-coding-antipatterns.md +580 -0
- package/security/05-cases/case-log4shell-incident.md +537 -0
- package/security/05-cases/case-major-breaches.md +468 -0
- package/security/06-glossary/security-glossary.md +212 -0
- package/security/compliance-automation.md +993 -0
- package/security/container-security.md +680 -0
- package/security/devsecops-complete.md +426 -0
- package/security/sast-dast-sca.md +775 -0
- package/security/secrets-management.md +594 -0
- package/security/security-architecture-deep-dive.md +37 -0
- package/security/threat-modeling-stride-playbook.md +40 -0
- package/seed-templates/auth-system.md +59 -0
- package/seed-templates/blog-content.md +94 -0
- package/seed-templates/dashboard.md +89 -0
- package/seed-templates/docs-site.md +73 -0
- package/seed-templates/e-commerce.md +50 -0
- package/seed-templates/saas-landing.md +92 -0
- package/seed-templates/settings-page.md +51 -0
- package/testing/01-standards/test-strategy-and-layering.md +83 -0
- package/testing/01-standards/testing-strategy-complete.md +422 -0
- package/testing/01-standards/unit-testing-best-practices.md +118 -0
- package/testing/02-playbooks/e2e-testing-playbook.md +988 -0
- package/testing/02-playbooks/testing-strategy-playbook.md +126 -0
- package/testing/03-checklists/test-strategy-checklist.md +208 -0
- package/testing/04-antipatterns/testing-antipatterns.md +718 -0
- package/testing/05-cases/case-testing-transformation.md +300 -0
- package/testing/06-glossary/testing-glossary.md +110 -0
- package/testing/risk-based-test-matrix.md +36 -0
- package/testing/testing-strategy-deep-dive.md +37 -0
|
@@ -0,0 +1,429 @@
|
|
|
1
|
+
---
|
|
2
|
+
id: data-engineering-glossary
|
|
3
|
+
title: Data Engineering Glossary
|
|
4
|
+
domain: data-engineering
|
|
5
|
+
category: 06-glossary
|
|
6
|
+
difficulty: intermediate
|
|
7
|
+
tags: [agent, checklist, data, data-engineering, engineering, glossary, 术语对比速查表, 概述]
|
|
8
|
+
quality_score: 70
|
|
9
|
+
last_updated: 2026-06-15
|
|
10
|
+
---
|
|
11
|
+
# Data Engineering Glossary
|
|
12
|
+
|
|
13
|
+
## 概述
|
|
14
|
+
|
|
15
|
+
数据工程术语表涵盖数据采集、传输、存储、处理和服务全链路的核心概念与工具。本术语表按字母顺序排列,每个词条包含定义、使用场景和关联术语,适用于数据工程师、后端工程师和技术决策者。
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## A
|
|
20
|
+
|
|
21
|
+
### Airflow (Apache Airflow)
|
|
22
|
+
|
|
23
|
+
**定义**:Apache 基金会的开源工作流编排平台,使用 Python 定义 DAG(有向无环图)来描述数据管道的依赖关系和执行顺序。
|
|
24
|
+
|
|
25
|
+
**使用场景**:ETL/ELT 任务调度、定时数据同步、多步骤数据处理流水线编排。
|
|
26
|
+
|
|
27
|
+
**核心概念**:DAG(工作流定义)、Operator(任务执行单元)、Scheduler(调度器)、Executor(执行引擎)、XCom(任务间通信)。
|
|
28
|
+
|
|
29
|
+
**关联术语**:DAG、ETL、Prefect、Dagster
|
|
30
|
+
|
|
31
|
+
### Avro
|
|
32
|
+
|
|
33
|
+
**定义**:Apache 开源的行式数据序列化格式,Schema 与数据一起存储,支持 Schema 演进(Schema Evolution),广泛用于 Kafka 消息序列化。
|
|
34
|
+
|
|
35
|
+
**使用场景**:Kafka 消息编解码、数据交换格式、Schema Registry 配合使用。
|
|
36
|
+
|
|
37
|
+
**对比**:Avro(行存、适合写入和消息传递) vs Parquet(列存、适合分析查询)。
|
|
38
|
+
|
|
39
|
+
**关联术语**:Schema Registry、Kafka、Parquet、Protobuf
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## B
|
|
44
|
+
|
|
45
|
+
### Batch Processing
|
|
46
|
+
|
|
47
|
+
**定义**:批处理,指对一段时间内累积的数据进行集中处理的计算模式。与流处理(Stream Processing)相对。
|
|
48
|
+
|
|
49
|
+
**使用场景**:每日报表生成、历史数据回填、大规模数据转换。
|
|
50
|
+
|
|
51
|
+
**典型工具**:Spark、Hive、MapReduce。
|
|
52
|
+
|
|
53
|
+
**关联术语**:Stream Processing、Lambda Architecture、Spark
|
|
54
|
+
|
|
55
|
+
### Bronze/Silver/Gold (Medallion Architecture)
|
|
56
|
+
|
|
57
|
+
**定义**:数据湖中的分层架构模式(也称奖牌架构)。Bronze 层存原始数据,Silver 层存清洗后的数据,Gold 层存业务聚合数据。
|
|
58
|
+
|
|
59
|
+
**使用场景**:数据湖建设、Databricks Lakehouse 架构。
|
|
60
|
+
|
|
61
|
+
**关联术语**:Data Lake、Data Lakehouse、Delta Lake
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## C
|
|
66
|
+
|
|
67
|
+
### CDC (Change Data Capture)
|
|
68
|
+
|
|
69
|
+
**定义**:变更数据捕获,一种识别和捕获数据库中数据变更(INSERT/UPDATE/DELETE)的技术,将变更实时或近实时地传播到下游系统。
|
|
70
|
+
|
|
71
|
+
**实现方式**:
|
|
72
|
+
- **基于日志**:读取数据库 binlog/WAL(推荐,对源库影响小)
|
|
73
|
+
- **基于查询**:定期轮询变更(简单但延迟高、对源库有压力)
|
|
74
|
+
- **基于触发器**:数据库触发器捕获变更(侵入性强,不推荐)
|
|
75
|
+
|
|
76
|
+
**典型工具**:Debezium、Maxwell、Canal(阿里开源)、AWS DMS。
|
|
77
|
+
|
|
78
|
+
**关联术语**:Debezium、Binlog、WAL、Stream Processing
|
|
79
|
+
|
|
80
|
+
### Column Store
|
|
81
|
+
|
|
82
|
+
**定义**:列式存储,按列而非按行组织数据的存储格式。同一列的数据物理上连续存储,适合分析查询(通常只读取少数列)和高压缩比。
|
|
83
|
+
|
|
84
|
+
**典型系统**:ClickHouse、Apache Druid、Redshift、BigQuery。
|
|
85
|
+
|
|
86
|
+
**关联术语**:Parquet、ORC、OLAP
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## D
|
|
91
|
+
|
|
92
|
+
### DAG (Directed Acyclic Graph)
|
|
93
|
+
|
|
94
|
+
**定义**:有向无环图,在数据工程中用于描述任务之间的依赖关系和执行顺序。任务 A 完成后才能执行任务 B 即 A -> B 的有向边。
|
|
95
|
+
|
|
96
|
+
**使用场景**:Airflow 工作流定义、Spark 执行计划、dbt 模型依赖。
|
|
97
|
+
|
|
98
|
+
**关联术语**:Airflow、dbt、Spark
|
|
99
|
+
|
|
100
|
+
### Data Catalog
|
|
101
|
+
|
|
102
|
+
**定义**:数据目录,集中管理组织内所有数据资产的元数据的系统。提供数据发现、数据血缘、数据质量和访问控制功能。
|
|
103
|
+
|
|
104
|
+
**典型工具**:Apache Atlas、DataHub(LinkedIn 开源)、Amundsen(Lyft 开源)、Google Data Catalog。
|
|
105
|
+
|
|
106
|
+
**关联术语**:Metadata、Data Lineage、Data Governance
|
|
107
|
+
|
|
108
|
+
### Data Lake
|
|
109
|
+
|
|
110
|
+
**定义**:数据湖,以原始格式存储海量结构化、半结构化和非结构化数据的集中式存储系统。强调"先存储,后处理"(Schema-on-Read)。
|
|
111
|
+
|
|
112
|
+
**典型实现**:AWS S3 + Athena、Azure Data Lake Storage、HDFS。
|
|
113
|
+
|
|
114
|
+
**优点**:存储成本低、支持多种数据格式、灵活性高。
|
|
115
|
+
|
|
116
|
+
**风险**:缺乏治理可能退化为"数据沼泽"(Data Swamp)。
|
|
117
|
+
|
|
118
|
+
**关联术语**:Data Warehouse、Data Lakehouse、Bronze/Silver/Gold
|
|
119
|
+
|
|
120
|
+
### Data Lakehouse
|
|
121
|
+
|
|
122
|
+
**定义**:数据湖仓一体,结合数据湖的灵活性和数据仓库的管理能力的新型架构。在数据湖之上增加 ACID 事务、Schema 管理和性能优化。
|
|
123
|
+
|
|
124
|
+
**典型实现**:Databricks(Delta Lake)、Apache Iceberg、Apache Hudi。
|
|
125
|
+
|
|
126
|
+
**关联术语**:Data Lake、Data Warehouse、Delta Lake、Iceberg
|
|
127
|
+
|
|
128
|
+
### Data Lineage
|
|
129
|
+
|
|
130
|
+
**定义**:数据血缘,追踪数据从源头到最终消费者的完整流转路径。包括数据经过了哪些转换、存储在哪些系统中、被哪些报表使用。
|
|
131
|
+
|
|
132
|
+
**使用场景**:影响分析(修改上游会影响哪些下游)、合规审计、数据质量根因分析。
|
|
133
|
+
|
|
134
|
+
**关联术语**:Data Catalog、Metadata、dbt
|
|
135
|
+
|
|
136
|
+
### Data Mesh
|
|
137
|
+
|
|
138
|
+
**定义**:一种去中心化的数据架构范式,将数据所有权分散到各业务领域团队,每个团队负责自己的数据产品(Data Product),中台提供自助式基础设施。
|
|
139
|
+
|
|
140
|
+
**四大原则**:领域所有权、数据即产品、自助式平台、联邦治理。
|
|
141
|
+
|
|
142
|
+
**关联术语**:Data Product、Domain-Driven Design
|
|
143
|
+
|
|
144
|
+
### Data Warehouse
|
|
145
|
+
|
|
146
|
+
**定义**:数据仓库,面向分析场景的集中式数据存储系统。数据在写入时经过清洗和建模(Schema-on-Write),结构化程度高。
|
|
147
|
+
|
|
148
|
+
**典型系统**:Snowflake、BigQuery、Redshift、ClickHouse。
|
|
149
|
+
|
|
150
|
+
**对比**:Data Warehouse(结构化、分析优化)vs Data Lake(原始格式、灵活)。
|
|
151
|
+
|
|
152
|
+
**关联术语**:Data Lake、OLAP、Star Schema、dbt
|
|
153
|
+
|
|
154
|
+
### dbt (Data Build Tool)
|
|
155
|
+
|
|
156
|
+
**定义**:数据转换工具,允许数据分析师和工程师使用 SQL 定义数据模型并管理转换逻辑。dbt 处理"T"(Transform),不处理"E"和"L"。
|
|
157
|
+
|
|
158
|
+
**核心能力**:SQL 模型化、依赖管理(自动 DAG)、测试框架、文档生成、增量处理。
|
|
159
|
+
|
|
160
|
+
**版本**:dbt Core(开源 CLI)、dbt Cloud(托管平台)。
|
|
161
|
+
|
|
162
|
+
**关联术语**:ELT、Data Warehouse、DAG
|
|
163
|
+
|
|
164
|
+
### Debezium
|
|
165
|
+
|
|
166
|
+
**定义**:Red Hat 开源的分布式 CDC 平台,基于 Kafka Connect 实现。支持 MySQL、PostgreSQL、MongoDB、Oracle 等数据库的变更捕获。
|
|
167
|
+
|
|
168
|
+
**工作原理**:读取数据库的事务日志(binlog/WAL),将变更事件发布到 Kafka Topic。
|
|
169
|
+
|
|
170
|
+
**关联术语**:CDC、Kafka Connect、Binlog
|
|
171
|
+
|
|
172
|
+
### Delta Lake
|
|
173
|
+
|
|
174
|
+
**定义**:Databricks 开源的存储层,在数据湖(如 S3/HDFS)之上提供 ACID 事务、Schema 演进、时间旅行(Time Travel)和统一的批流处理。
|
|
175
|
+
|
|
176
|
+
**关联术语**:Data Lakehouse、Iceberg、Hudi、Parquet
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## E
|
|
181
|
+
|
|
182
|
+
### ELT (Extract, Load, Transform)
|
|
183
|
+
|
|
184
|
+
**定义**:先抽取和加载原始数据到目标系统(通常是数据仓库或数据湖),然后在目标系统中进行转换。与 ETL 的区别在于转换发生的位置。
|
|
185
|
+
|
|
186
|
+
**优势**:利用目标系统的计算能力进行转换,适合云数据仓库场景。
|
|
187
|
+
|
|
188
|
+
**典型工具链**:Fivetran/Airbyte(EL) + dbt(T) + Snowflake/BigQuery(目标系统)。
|
|
189
|
+
|
|
190
|
+
**关联术语**:ETL、dbt、Data Warehouse
|
|
191
|
+
|
|
192
|
+
### ETL (Extract, Transform, Load)
|
|
193
|
+
|
|
194
|
+
**定义**:数据集成的经典三步流程。从源系统抽取(Extract)数据,在中间层进行清洗转换(Transform),然后加载(Load)到目标系统。
|
|
195
|
+
|
|
196
|
+
**使用场景**:传统数据仓库建设、跨系统数据同步。
|
|
197
|
+
|
|
198
|
+
**典型工具**:Informatica、Talend、Apache NiFi、AWS Glue。
|
|
199
|
+
|
|
200
|
+
**对比**:ETL(转换在中间层)vs ELT(转换在目标系统)。
|
|
201
|
+
|
|
202
|
+
**关联术语**:ELT、Data Warehouse、Airflow
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
## F
|
|
207
|
+
|
|
208
|
+
### Flink (Apache Flink)
|
|
209
|
+
|
|
210
|
+
**定义**:分布式流处理框架,支持有状态的流计算和批处理。以"流优先"设计,将批处理视为有界流的特例。
|
|
211
|
+
|
|
212
|
+
**核心能力**:精确一次(Exactly-Once)语义、事件时间处理、窗口计算、状态管理、Savepoint/Checkpoint。
|
|
213
|
+
|
|
214
|
+
**使用场景**:实时数据分析、CEP(复杂事件处理)、实时 ETL、实时风控。
|
|
215
|
+
|
|
216
|
+
**对比**:Flink(真正的流处理,低延迟)vs Spark Streaming(微批处理,吞吐量高)。
|
|
217
|
+
|
|
218
|
+
**关联术语**:Stream Processing、Kafka、Exactly-Once、Spark
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## I
|
|
223
|
+
|
|
224
|
+
### Iceberg (Apache Iceberg)
|
|
225
|
+
|
|
226
|
+
**定义**:Netflix 开源的表格式(Table Format),为数据湖提供类似数据仓库的管理能力。支持 ACID 事务、Schema 演进、分区演进和时间旅行。
|
|
227
|
+
|
|
228
|
+
**对比**:Iceberg vs Delta Lake vs Hudi - 三者功能相似,Iceberg 的引擎无关性最好(支持 Spark/Flink/Trino/Presto)。
|
|
229
|
+
|
|
230
|
+
**关联术语**:Data Lakehouse、Delta Lake、Hudi、Parquet
|
|
231
|
+
|
|
232
|
+
### Idempotency
|
|
233
|
+
|
|
234
|
+
**定义**:幂等性,指同一操作执行多次与执行一次的效果相同。在数据管道中至关重要,因为重试和重放是常见场景。
|
|
235
|
+
|
|
236
|
+
**实现方式**:使用唯一标识去重、UPSERT 操作、幂等写入(如 Kafka 幂等生产者)。
|
|
237
|
+
|
|
238
|
+
**关联术语**:Exactly-Once、At-Least-Once
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## K
|
|
243
|
+
|
|
244
|
+
### Kafka (Apache Kafka)
|
|
245
|
+
|
|
246
|
+
**定义**:分布式事件流平台,用于构建实时数据管道和流式应用。消息以 Topic 为单位组织,分区(Partition)实现并行,副本(Replica)保证高可用。
|
|
247
|
+
|
|
248
|
+
**核心概念**:
|
|
249
|
+
- **Producer**:消息生产者
|
|
250
|
+
- **Consumer**:消息消费者
|
|
251
|
+
- **Consumer Group**:消费者组,实现消息的并行消费
|
|
252
|
+
- **Topic**:消息主题,逻辑分类
|
|
253
|
+
- **Partition**:分区,物理并行单元
|
|
254
|
+
- **Offset**:消息偏移量,消费位置标记
|
|
255
|
+
|
|
256
|
+
**使用场景**:系统间异步通信、事件驱动架构、日志聚合、CDC 传输、流处理数据源。
|
|
257
|
+
|
|
258
|
+
**关联术语**:Schema Registry、Kafka Connect、Flink、CDC
|
|
259
|
+
|
|
260
|
+
### Kafka Connect
|
|
261
|
+
|
|
262
|
+
**定义**:Kafka 的数据集成框架,提供标准化的 Connector 接口,用于在 Kafka 和外部系统之间批量移动数据。
|
|
263
|
+
|
|
264
|
+
**类型**:Source Connector(外部 -> Kafka)、Sink Connector(Kafka -> 外部)。
|
|
265
|
+
|
|
266
|
+
**典型 Connector**:JDBC Source/Sink、Debezium(CDC Source)、Elasticsearch Sink、S3 Sink。
|
|
267
|
+
|
|
268
|
+
**关联术语**:Kafka、Debezium、CDC
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
## O
|
|
273
|
+
|
|
274
|
+
### OLAP (Online Analytical Processing)
|
|
275
|
+
|
|
276
|
+
**定义**:联机分析处理,面向复杂查询和数据分析的数据库处理方式。特点是读多写少、查询涉及大量数据聚合。
|
|
277
|
+
|
|
278
|
+
**典型系统**:ClickHouse、Apache Druid、StarRocks、BigQuery。
|
|
279
|
+
|
|
280
|
+
**对比**:OLAP(分析型,列存,复杂聚合)vs OLTP(事务型,行存,高并发读写)。
|
|
281
|
+
|
|
282
|
+
**关联术语**:OLTP、Column Store、Data Warehouse
|
|
283
|
+
|
|
284
|
+
### OLTP (Online Transaction Processing)
|
|
285
|
+
|
|
286
|
+
**定义**:联机事务处理,面向日常业务操作的数据库处理方式。特点是高并发、低延迟、单行读写为主。
|
|
287
|
+
|
|
288
|
+
**典型系统**:MySQL、PostgreSQL、Oracle、MongoDB。
|
|
289
|
+
|
|
290
|
+
**关联术语**:OLAP、ACID
|
|
291
|
+
|
|
292
|
+
### Orchestration
|
|
293
|
+
|
|
294
|
+
**定义**:编排,在数据工程中指协调和管理多个数据任务的执行顺序、依赖关系和失败处理。
|
|
295
|
+
|
|
296
|
+
**典型工具**:Airflow、Prefect、Dagster、Argo Workflows。
|
|
297
|
+
|
|
298
|
+
**关联术语**:Airflow、DAG、Pipeline
|
|
299
|
+
|
|
300
|
+
---
|
|
301
|
+
|
|
302
|
+
## P
|
|
303
|
+
|
|
304
|
+
### Parquet
|
|
305
|
+
|
|
306
|
+
**定义**:Apache 开源的列式存储文件格式,广泛用于大数据分析场景。支持高效的列裁剪(Column Pruning)和行组过滤(Predicate Pushdown),压缩比高。
|
|
307
|
+
|
|
308
|
+
**使用场景**:数据湖存储、Spark/Hive 分析、数据归档。
|
|
309
|
+
|
|
310
|
+
**对比**:Parquet(列存、分析优化)vs Avro(行存、序列化优化)vs ORC(列存、Hive 生态)。
|
|
311
|
+
|
|
312
|
+
**关联术语**:Avro、ORC、Column Store、Data Lake
|
|
313
|
+
|
|
314
|
+
### Pipeline
|
|
315
|
+
|
|
316
|
+
**定义**:数据管道,指数据从源头经过一系列处理步骤到达目标的完整流程。可以是批处理管道或实时流管道。
|
|
317
|
+
|
|
318
|
+
**关联术语**:ETL、ELT、DAG、Orchestration
|
|
319
|
+
|
|
320
|
+
---
|
|
321
|
+
|
|
322
|
+
## S
|
|
323
|
+
|
|
324
|
+
### Schema Registry
|
|
325
|
+
|
|
326
|
+
**定义**:Schema 注册中心,集中管理和验证数据 Schema(模式定义)的服务。确保生产者和消费者使用兼容的 Schema,防止数据格式不一致导致的下游故障。
|
|
327
|
+
|
|
328
|
+
**典型实现**:Confluent Schema Registry(支持 Avro/Protobuf/JSON Schema)。
|
|
329
|
+
|
|
330
|
+
**核心功能**:
|
|
331
|
+
- Schema 版本管理
|
|
332
|
+
- 兼容性检查(Backward/Forward/Full)
|
|
333
|
+
- Schema 演进支持
|
|
334
|
+
|
|
335
|
+
**关联术语**:Avro、Kafka、Schema Evolution
|
|
336
|
+
|
|
337
|
+
### SCD (Slowly Changing Dimension)
|
|
338
|
+
|
|
339
|
+
**定义**:缓慢变化维度,数据仓库中处理维度表数据随时间变化的策略。
|
|
340
|
+
|
|
341
|
+
**常用类型**:
|
|
342
|
+
- **Type 1**:直接覆盖旧值(不保留历史)
|
|
343
|
+
- **Type 2**:新增行保留历史(增加有效日期列)
|
|
344
|
+
- **Type 3**:增加列记录新旧值(仅保留一次变更)
|
|
345
|
+
|
|
346
|
+
**关联术语**:Data Warehouse、Star Schema、Dimension Table
|
|
347
|
+
|
|
348
|
+
### Spark (Apache Spark)
|
|
349
|
+
|
|
350
|
+
**定义**:统一的大规模数据处理引擎,支持批处理、流处理(Structured Streaming)、机器学习(MLlib)和图计算(GraphX)。
|
|
351
|
+
|
|
352
|
+
**核心概念**:RDD、DataFrame、SparkSQL、Catalyst Optimizer。
|
|
353
|
+
|
|
354
|
+
**使用场景**:大规模 ETL、数据分析、机器学习特征工程。
|
|
355
|
+
|
|
356
|
+
**关联术语**:Flink、Hadoop、Parquet、Delta Lake
|
|
357
|
+
|
|
358
|
+
### Star Schema
|
|
359
|
+
|
|
360
|
+
**定义**:星型模型,数据仓库中最常见的维度建模方式。中心是事实表(Fact Table),周围是维度表(Dimension Table),形如星形。
|
|
361
|
+
|
|
362
|
+
**优点**:查询简单直观、聚合性能好。
|
|
363
|
+
|
|
364
|
+
**对比**:Star Schema(简单、冗余)vs Snowflake Schema(规范化、复杂)。
|
|
365
|
+
|
|
366
|
+
**关联术语**:Data Warehouse、Fact Table、Dimension Table、SCD
|
|
367
|
+
|
|
368
|
+
### Stream Processing
|
|
369
|
+
|
|
370
|
+
**定义**:流处理,对持续到达的数据逐条或小批量进行实时处理的计算模式。与批处理相对。
|
|
371
|
+
|
|
372
|
+
**典型工具**:Flink、Kafka Streams、Spark Structured Streaming。
|
|
373
|
+
|
|
374
|
+
**核心挑战**:事件时间 vs 处理时间、乱序处理、状态管理、精确一次语义。
|
|
375
|
+
|
|
376
|
+
**关联术语**:Batch Processing、Flink、Kafka、Event Time
|
|
377
|
+
|
|
378
|
+
---
|
|
379
|
+
|
|
380
|
+
## W
|
|
381
|
+
|
|
382
|
+
### WAL (Write-Ahead Log)
|
|
383
|
+
|
|
384
|
+
**定义**:预写日志,数据库在修改数据前先将变更写入日志的机制。用于崩溃恢复和数据复制。PostgreSQL 的 WAL 是 CDC 的重要数据源。
|
|
385
|
+
|
|
386
|
+
**关联术语**:CDC、Binlog、Debezium
|
|
387
|
+
|
|
388
|
+
### Window Function
|
|
389
|
+
|
|
390
|
+
**定义**:窗口函数,在流处理中对一段时间范围或一定数量的事件进行聚合计算。
|
|
391
|
+
|
|
392
|
+
**常见类型**:
|
|
393
|
+
- **Tumbling Window**:滚动窗口,固定大小不重叠
|
|
394
|
+
- **Sliding Window**:滑动窗口,固定大小可重叠
|
|
395
|
+
- **Session Window**:会话窗口,基于活动间隔动态划分
|
|
396
|
+
|
|
397
|
+
**关联术语**:Flink、Stream Processing、Event Time
|
|
398
|
+
|
|
399
|
+
---
|
|
400
|
+
|
|
401
|
+
## 术语对比速查表
|
|
402
|
+
|
|
403
|
+
| 维度 | 选项 A | 选项 B | 选择依据 |
|
|
404
|
+
|------|--------|--------|---------|
|
|
405
|
+
| 处理模式 | Batch | Stream | 延迟要求(分钟级 vs 秒级) |
|
|
406
|
+
| 集成模式 | ETL | ELT | 目标系统计算能力 |
|
|
407
|
+
| 存储格式 | Parquet | Avro | 分析查询 vs 消息传递 |
|
|
408
|
+
| 流引擎 | Flink | Spark Streaming | 真流 vs 微批 |
|
|
409
|
+
| 表格式 | Iceberg | Delta Lake | 引擎无关性 vs Databricks 生态 |
|
|
410
|
+
| 存储架构 | Data Lake | Data Warehouse | 灵活性 vs 管理能力 |
|
|
411
|
+
| 编排工具 | Airflow | Prefect | 成熟度 vs 开发体验 |
|
|
412
|
+
| CDC 工具 | Debezium | Canal | 通用 vs MySQL 专用 |
|
|
413
|
+
|
|
414
|
+
---
|
|
415
|
+
|
|
416
|
+
## Agent Checklist
|
|
417
|
+
|
|
418
|
+
以下为 AI Agent 在数据工程项目中使用本术语表的要点:
|
|
419
|
+
|
|
420
|
+
- [ ] 数据管道设计中正确区分 ETL 和 ELT 模式
|
|
421
|
+
- [ ] 根据延迟要求选择批处理(Spark)或流处理(Flink/Kafka Streams)
|
|
422
|
+
- [ ] CDC 场景优先选择基于日志的方案(Debezium)而非轮询
|
|
423
|
+
- [ ] 分析场景使用列式存储格式(Parquet)而非行式格式
|
|
424
|
+
- [ ] Kafka 消息使用 Schema Registry 管理 Schema 兼容性
|
|
425
|
+
- [ ] 数据湖遵循 Medallion Architecture(Bronze/Silver/Gold)分层
|
|
426
|
+
- [ ] 数据仓库使用维度建模(Star Schema)并处理 SCD
|
|
427
|
+
- [ ] 数据管道确保幂等性,支持安全重试和重放
|
|
428
|
+
- [ ] 使用 dbt 管理 SQL 转换逻辑、测试和文档
|
|
429
|
+
- [ ] 建立数据目录和血缘追踪,支持影响分析和合规审计
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
---
|
|
2
|
+
id: database-schema-standards
|
|
3
|
+
title: 数据库 Schema 设计标准
|
|
4
|
+
domain: database
|
|
5
|
+
category: 01-standards
|
|
6
|
+
difficulty: intermediate
|
|
7
|
+
tags: [database, postgresql, schema, migration, index, foreign-key, normalization, sql, postgres, mysql]
|
|
8
|
+
quality_score: 91
|
|
9
|
+
maintainer: platform-team@umadev.com
|
|
10
|
+
last_updated: 2026-06-14
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# 数据库 Schema 设计标准
|
|
14
|
+
|
|
15
|
+
## 命名约定
|
|
16
|
+
|
|
17
|
+
### 表名
|
|
18
|
+
```sql
|
|
19
|
+
-- ✅ 复数蛇形命名
|
|
20
|
+
CREATE TABLE users (...);
|
|
21
|
+
CREATE TABLE order_items (...);
|
|
22
|
+
CREATE TABLE product_categories (...);
|
|
23
|
+
|
|
24
|
+
-- ❌ 单数 / 驼峰 / 前缀
|
|
25
|
+
CREATE TABLE user (...);
|
|
26
|
+
CREATE TABLE OrderItems (...);
|
|
27
|
+
CREATE TABLE tbl_users (...);
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
### 列名
|
|
31
|
+
```sql
|
|
32
|
+
-- ✅ 蛇形,语义清晰
|
|
33
|
+
created_at, updated_at, deleted_at
|
|
34
|
+
owner_id, project_id, assignee_id
|
|
35
|
+
is_active, is_deleted
|
|
36
|
+
email_verified_at
|
|
37
|
+
|
|
38
|
+
-- ❌ 缩写 / 匈牙利命名
|
|
39
|
+
dt_cre, strName, bolActive
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### 索引名
|
|
43
|
+
```sql
|
|
44
|
+
-- 格式:idx_{table}_{columns}
|
|
45
|
+
CREATE INDEX idx_orders_user_id ON orders(user_id);
|
|
46
|
+
CREATE INDEX idx_orders_status_created ON orders(status, created_at);
|
|
47
|
+
|
|
48
|
+
-- 唯一索引:uq_{table}_{columns}
|
|
49
|
+
CREATE UNIQUE INDEX uq_users_email ON users(email);
|
|
50
|
+
|
|
51
|
+
-- 外键约束:fk_{from_table}_{to_table}
|
|
52
|
+
ALTER TABLE orders ADD CONSTRAINT fk_orders_users FOREIGN KEY (user_id) REFERENCES users(id);
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## 每表必备列
|
|
56
|
+
|
|
57
|
+
```sql
|
|
58
|
+
CREATE TABLE products (
|
|
59
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
60
|
+
-- 业务列...
|
|
61
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
62
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
63
|
+
);
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
| 列 | 类型 | 说明 |
|
|
67
|
+
|---|---|---|
|
|
68
|
+
| `id` | UUID | 主键,gen_random_uuid() |
|
|
69
|
+
| `created_at` | TIMESTAMPTZ | 创建时间 |
|
|
70
|
+
| `updated_at` | TIMESTAMPTZ | 更新时间(触发器自动维护) |
|
|
71
|
+
| `deleted_at` | TIMESTAMPTZ | 软删除(可选) |
|
|
72
|
+
|
|
73
|
+
## 外键设计
|
|
74
|
+
|
|
75
|
+
```sql
|
|
76
|
+
-- ON DELETE 策略
|
|
77
|
+
ALTER TABLE order_items ADD CONSTRAINT fk_items_orders
|
|
78
|
+
FOREIGN KEY (order_id) REFERENCES orders(id)
|
|
79
|
+
ON DELETE CASCADE; -- 订单删除时,明细也删除
|
|
80
|
+
|
|
81
|
+
ALTER TABLE tasks ADD CONSTRAINT fk_tasks_projects
|
|
82
|
+
FOREIGN KEY (project_id) REFERENCES projects(id)
|
|
83
|
+
ON DELETE SET NULL; -- 项目删除时,任务保留但 project_id 置空
|
|
84
|
+
|
|
85
|
+
ALTER TABLE comments ADD CONSTRAINT fk_comments_users
|
|
86
|
+
FOREIGN KEY (author_id) REFERENCES users(id)
|
|
87
|
+
ON DELETE RESTRICT; -- 不允许删除有评论的用户
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
| 策略 | 场景 |
|
|
91
|
+
|------|------|
|
|
92
|
+
| CASCADE | 子资源无意义(订单→明细) |
|
|
93
|
+
| SET NULL | 子资源独立存在(项目→任务) |
|
|
94
|
+
| RESTRICT | 禁止删除有依赖的记录 |
|
|
95
|
+
|
|
96
|
+
## 索引原则
|
|
97
|
+
|
|
98
|
+
```sql
|
|
99
|
+
-- 每个外键必须有索引
|
|
100
|
+
CREATE INDEX idx_orders_user_id ON orders(user_id);
|
|
101
|
+
|
|
102
|
+
-- 常用查询条件的列建索引
|
|
103
|
+
CREATE INDEX idx_orders_status ON orders(status);
|
|
104
|
+
|
|
105
|
+
-- 复合索引遵循最左前缀
|
|
106
|
+
-- 查询 WHERE user_id = ? AND status = ? 用这个索引
|
|
107
|
+
CREATE INDEX idx_orders_user_status ON orders(user_id, status);
|
|
108
|
+
|
|
109
|
+
-- JSONB 用 GIN
|
|
110
|
+
CREATE INDEX idx_products_attrs ON products USING GIN(attrs);
|
|
111
|
+
|
|
112
|
+
-- 软删除用部分索引(不索引已删数据)
|
|
113
|
+
CREATE INDEX idx_active_users ON users(email) WHERE deleted_at IS NULL;
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Migration 规范
|
|
117
|
+
|
|
118
|
+
```sql
|
|
119
|
+
-- 0001_init.sql — 初始 schema
|
|
120
|
+
-- 0002_add_indexes.sql — 索引优化
|
|
121
|
+
-- 0003_add_audit_log.sql — 新功能
|
|
122
|
+
|
|
123
|
+
-- 每个文件包含 UP + DOWN
|
|
124
|
+
-- === UP ===
|
|
125
|
+
CREATE TABLE products (...);
|
|
126
|
+
-- === DOWN ===
|
|
127
|
+
DROP TABLE IF EXISTS products;
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### Migration 原则
|
|
131
|
+
1. **向前兼容** — 新代码能读旧 schema,旧代码能读新 schema
|
|
132
|
+
2. **分步迁移** — 先加列(有默认值)→ 部署代码 → 回填数据 → 加约束
|
|
133
|
+
3. **不锁表** — 大表加列用 `DEFAULT ... NOT NULL`(PG 11+ 不锁表)
|
|
134
|
+
4. **回滚方案** — 每个 migration 有对应的 down migration
|
|
135
|
+
|
|
136
|
+
## 数据类型选择
|
|
137
|
+
|
|
138
|
+
| 场景 | 类型 | 说明 |
|
|
139
|
+
|------|------|------|
|
|
140
|
+
| 主键 | UUID | 全局唯一,无顺序泄露 |
|
|
141
|
+
| 金额 | NUMERIC(19,2) 或 INTEGER(分) | 浮点数会丢精度 |
|
|
142
|
+
| 状态 | TEXT + CHECK | `CHECK (status IN ('active','inactive'))` |
|
|
143
|
+
| 时间 | TIMESTAMPTZ | 带时区,统一存 UTC |
|
|
144
|
+
| 布尔 | BOOLEAN | 不用 INTEGER 0/1 |
|
|
145
|
+
| JSON | JSONB | 可索引,灵活字段 |
|
|
146
|
+
| 枚举 | TEXT + CHECK | 比 ENUM 类型灵活(加值不需 migration) |
|
|
147
|
+
| 大文本 | TEXT | 无长度限制 |
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
---
|
|
2
|
+
id: postgresql-optimization-quick
|
|
3
|
+
title: PostgreSQL 快速优化指南
|
|
4
|
+
domain: database
|
|
5
|
+
category: 02-playbooks
|
|
6
|
+
difficulty: intermediate
|
|
7
|
+
tags: [postgresql, database, optimization]
|
|
8
|
+
quality_score: 90
|
|
9
|
+
maintainer: dba-team@umadev.com
|
|
10
|
+
last_updated: 2026-03-29
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PostgreSQL 快速优化指南
|
|
14
|
+
|
|
15
|
+
## 栅查看询性能
|
|
16
|
+
|
|
17
|
+
### 1. 启用性能监控
|
|
18
|
+
```sql
|
|
19
|
+
CREATE EXTENSION pg_stat_statements;
|
|
20
|
+
SELECT * FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 10;
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
### 2. 分析执行计划
|
|
24
|
+
```sql
|
|
25
|
+
EXPLAIN ANALYZE SELECT * FROM orders WHERE user_id = 123;
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## 索引优化
|
|
29
|
+
|
|
30
|
+
### 创建关键索引
|
|
31
|
+
```sql
|
|
32
|
+
CREATE INDEX CONCURRENTLY idx_users_email ON users(email);
|
|
33
|
+
CREATE INDEX idx_orders_date ON orders(created_at);
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### 检查未使用索引
|
|
37
|
+
```sql
|
|
38
|
+
SELECT * FROM pg_stat_user_indexes WHERE idx_scan = 0;
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## 配置优化
|
|
42
|
+
### 关键参数
|
|
43
|
+
```sql
|
|
44
|
+
ALTER SYSTEM SET shared_buffers = '256MB';
|
|
45
|
+
ALTER SYSTEM SET work_mem = '16MB';
|
|
46
|
+
ALTER SYSTEM SET max_connections = 200;
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### 重载配置
|
|
50
|
+
```sql
|
|
51
|
+
SELECT pg_reload_conf();
|
|
52
|
+
```
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
---
|
|
2
|
+
id: postgresql-performance-optimization
|
|
3
|
+
title: PostgreSQL 性能优化实战
|
|
4
|
+
domain: database
|
|
5
|
+
category: 02-playbooks
|
|
6
|
+
difficulty: advanced
|
|
7
|
+
tags: [postgresql, database, performance, optimization]
|
|
8
|
+
quality_score: 93
|
|
9
|
+
maintainer: dba-team@umadev.com
|
|
10
|
+
last_updated: 2026-03-29
|
|
11
|
+
version: 2.0
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
# PostgreSQL 性能优化实战
|
|
15
|
+
|
|
16
|
+
## 概述
|
|
17
|
+
|
|
18
|
+
本实战指南提供系统化的 PostgreSQL 性能优化方法,从慢查询分析到索引优化,从配置调优到监控告警。
|
|
19
|
+
|
|
20
|
+
## 场景 1: 慢查询分析与优化
|
|
21
|
+
|
|
22
|
+
### 识别慢查询
|
|
23
|
+
```sql
|
|
24
|
+
SELECT
|
|
25
|
+
query,
|
|
26
|
+
calls,
|
|
27
|
+
round(total_exec_time::numeric, 2) as total_ms,
|
|
28
|
+
round(mean_exec_time::numeric, 2) as avg_ms
|
|
29
|
+
FROM pg_stat_statements
|
|
30
|
+
ORDER by total_exec_time desc
|
|
31
|
+
limit 10;
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### 查询优化策略
|
|
35
|
+
|
|
36
|
+
```sql
|
|
37
|
+
-- 使用 EXPLAIN ANALYZE
|
|
38
|
+
EXPLAIN ANALYZE SELECT * from orders where user_id = 123;
|
|
39
|
+
|
|
40
|
+
-- 添加缺失的索引
|
|
41
|
+
CREATE index idx_orders_user_id on orders(user_id);
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## 场景 2: 知识库优化
|
|
45
|
+
### 创建知识库
|
|
46
|
+
```sql
|
|
47
|
+
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
|
|
48
|
+
```
|
|
49
|
+
### 查询优化示例
|
|
50
|
+
```sql
|
|
51
|
+
-- 优化前
|
|
52
|
+
SELECT * FROM orders WHERE created_at > '2024-01-01';
|
|
53
|
+
|
|
54
|
+
-- 优化后
|
|
55
|
+
SELECT * FROM orders
|
|
56
|
+
WHERE created_at > '2024-01-01'
|
|
57
|
+
LIMIT 100;
|
|
58
|
+
```
|