@gordon.gan/specflow 1.3.3-beta → 1.4.1

package/skills/database/elasticsearch/examples/02-aggregation-report.md

@@ -0,0 +1,206 @@
+# 示例: 聚合报表 — 电商销售分析
+
+## 场景
+
+基于订单数据构建销售分析仪表盘：销售额趋势、TOP 品类、品牌分布、价格区段。
+
+## 步骤
+
+### 1. 创建索引
+
+```json
+PUT /orders
+{
+  "settings": {
+    "number_of_shards": 3,
+    "number_of_replicas": 1
+  },
+  "mappings": {
+    "properties": {
+      "order_id": { "type": "keyword" },
+      "user_id": { "type": "keyword" },
+      "product_id": { "type": "keyword" },
+      "product_name": { "type": "text", "fields": { "keyword": { "type": "keyword" } } },
+      "category": { "type": "keyword" },
+      "brand": { "type": "keyword" },
+      "amount": { "type": "float" },
+      "quantity": { "type": "integer" },
+      "status": { "type": "keyword" },
+      "region": { "type": "keyword" },
+      "order_date": { "type": "date" },
+      "payment_date": { "type": "date" }
+    }
+  }
+}
+```
+
+### 2. 按品类统计销售额 (terms + sum)
+
+```bash
+# 每个品类的总销售额, 降序排列
+GET /orders/_search
+{
+  "size": 0,
+  "aggs": {
+    "by_category": {
+      "terms": {
+        "field": "category",
+        "size": 20,
+        "order": { "total_sales": "desc" }
+      },
+      "aggs": {
+        "total_sales": { "sum": { "field": "amount" } },
+        "avg_order": { "avg": { "field": "amount" } },
+        "order_count": { "value_count": { "field": "order_id" } }
+      }
+    }
+  }
+}
+```
+
+### 3. 每日销售趋势 (date_histogram + sum)
+
+```bash
+# 按天统计销售额
+GET /orders/_search
+{
+  "size": 0,
+  "query": {
+    "range": {
+      "order_date": { "gte": "now-30d", "lte": "now" }
+    }
+  },
+  "aggs": {
+    "daily_sales": {
+      "date_histogram": {
+        "field": "order_date",
+        "calendar_interval": "day",
+        "format": "yyyy-MM-dd",
+        "min_doc_count": 0,
+        "extended_bounds": {
+          "min": "2024-01-01",
+          "max": "2024-12-31"
+        }
+      },
+      "aggs": {
+        "revenue": { "sum": { "field": "amount" } },
+        "orders": { "value_count": { "field": "order_id" } }
+      }
+    }
+  }
+}
+```
+
+### 4. 价格区段分布 (range + stats)
+
+```bash
+# 价格区段 + 每个区段的统计
+GET /orders/_search
+{
+  "size": 0,
+  "aggs": {
+    "price_ranges": {
+      "range": {
+        "field": "amount",
+        "ranges": [
+          { "key": "低价 (<100)", "to": 100 },
+          { "key": "中低价 (100-500)", "from": 100, "to": 500 },
+          { "key": "中价 (500-2000)", "from": 500, "to": 2000 },
+          { "key": "高价 (2000-10000)", "from": 2000, "to": 10000 },
+          { "key": "超高 (>10000)", "from": 10000 }
+        ]
+      },
+      "aggs": {
+        "amount_stats": { "stats": { "field": "amount" } },
+        "brand_distribution": {
+          "terms": { "field": "brand", "size": 5 }
+        }
+      }
+    }
+  }
+}
+```
+
+### 5. 地区分布 + TOP 品牌 (嵌套聚合)
+
+```bash
+# 各地区销售额, 每个地区 TOP 品牌
+GET /orders/_search
+{
+  "size": 0,
+  "aggs": {
+    "by_region": {
+      "terms": {
+        "field": "region",
+        "size": 10,
+        "order": { "revenue": "desc" }
+      },
+      "aggs": {
+        "revenue": { "sum": { "field": "amount" } },
+        "order_count": { "value_count": { "field": "order_id" } },
+        "top_brands": {
+          "terms": {
+            "field": "brand",
+            "size": 5,
+            "order": { "brand_revenue": "desc" }
+          },
+          "aggs": {
+            "brand_revenue": { "sum": { "field": "amount" } }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+### 6. 综合仪表盘查询 (一次查询完成多项分析)
+
+```bash
+# 一次查询返回多种聚合结果
+GET /orders/_search
+{
+  "size": 0,
+  "query": {
+    "bool": {
+      "filter": [
+        { "term": { "status": "completed" } },
+        { "range": { "order_date": { "gte": "now-30d/d", "lte": "now" } } }
+      ]
+    }
+  },
+  "aggs": {
+    "overview": {        // 总体指标
+      "stats": { "field": "amount" }
+    },
+    "total_revenue": {
+      "sum": { "field": "amount" }
+    },
+    "unique_customers": {
+      "cardinality": { "field": "user_id" }
+    },
+    "sales_trend": {     // 日销售趋势
+      "date_histogram": {
+        "field": "order_date",
+        "calendar_interval": "day"
+      },
+      "aggs": {
+        "daily_revenue": { "sum": { "field": "amount" } }
+      }
+    },
+    "top_categories": {  // TOP 品类
+      "terms": { "field": "category", "size": 10 },
+      "aggs": {
+        "cat_revenue": { "sum": { "field": "amount" } }
+      }
+    },
+    "price_distribution": {  // 价格分布
+      "percentiles": {
+        "field": "amount",
+        "percents": [25, 50, 75, 90, 99]
+      }
+    }
+  }
+}
+# 一次请求返回: 总览统计 + 总销售额 + 去重用户数 + 日趋势 + TOP 品类 + 价格百分位
+```

package/skills/database/elasticsearch/examples/03-reindex-zero-downtime.md

@@ -0,0 +1,200 @@
+# 示例: 零停机重建索引 (Zero-Downtime Reindex)
+
+## 场景
+
+需要修改已有索引的 mapping（如添加字段、修改分词器），但生产环境不能停机。
+
+## 核心思路
+
+Alias（别名） + Reindex（数据迁移） + 原子切换。
+
+## 步骤
+
+### 1. 假设已有索引和别名
+
+```bash
+# 初始状态: 已有 products_v1 索引, 通过 products 别名访问
+GET /products/_search
+{
+  "query": { "match_all": {} }
+}
+# 别名可以透明访问
+```
+
+### 2. 创建新索引 (v2) 使用新的 mapping
+
+```json
+PUT /products_v2
+{
+  "settings": {
+    "number_of_shards": 5,
+    "number_of_replicas": 1,
+    "refresh_interval": "-1"
+  },
+  "mappings": {
+    "properties": {
+      "title": {
+        "type": "text",
+        "analyzer": "ik_max_word",
+        "fields": {
+          "keyword": { "type": "keyword", "ignore_above": 256 },
+          "pinyin": { "type": "text", "analyzer": "pinyin" }
+        }
+      },
+      "description": {
+        "type": "text",
+        "analyzer": "ik_max_word"
+      },
+      "price": { "type": "double" },
+      "new_field": { "type": "keyword" },
+      "search_all": {
+        "type": "text",
+        "analyzer": "ik_max_word"
+      }
+    }
+  }
+}
+```
+
+### 3. 执行 Reindex 数据迁移
+
+```bash
+# 使用 slices=auto 并行加速
+POST /_reindex?slices=auto&wait_for_completion=false
+{
+  "source": {
+    "index": "products_v1"
+  },
+  "dest": {
+    "index": "products_v2"
+  },
+  "script": {
+    "source": """
+      // 如果新索引有 copy_to 字段, 在脚本中处理
+      if (ctx._source.title != null && ctx._source.description != null) {
+        ctx._source.search_all = ctx._source.title + ' ' + ctx._source.description;
+      }
+      // 添加新字段默认值
+      ctx._source.new_field = 'migrated';
+    """
+  }
+}
+# 返回 task id, 可用 _tasks API 查看进度
+GET /_tasks/<task_id>
+```
+
+### 4. 原子切换别名 (零停机)
+
+```bash
+# 关键步骤: 一次操作移除旧别名 + 添加新别名
+# 这期间已有查询不会中断
+POST /_aliases
+{
+  "actions": [
+    { "remove": { "index": "products_v1", "alias": "products" } },
+    { "add": { "index": "products_v2", "alias": "products" } }
+  ]
+}
+# 原子操作, 切换完成后所有通过 products 别名的查询自动指向 v2
+```
+
+### 5. 验证并删除旧索引
+
+```bash
+# 验证新索引数据完整
+GET /products/_search
+{
+  "query": { "match_all": {} },
+  "size": 0
+}
+
+# 确认无误后删除旧索引
+DELETE /products_v1
+```
+
+## Reindex 进阶技巧
+
+### 跨集群 Reindex
+
+```bash
+POST /_reindex
+{
+  "source": {
+    "remote": {
+      "host": "http://old-cluster:9200",
+      "username": "elastic",
+      "password": "pass"
+    },
+    "index": "products"
+  },
+  "dest": {
+    "index": "products_v2"
+  }
+}
+```
+
+### 只迁移部分数据
+
+```bash
+POST /_reindex
+{
+  "source": {
+    "index": "products_v1",
+    "query": {
+      "term": { "status": "active" }
+    }
+  },
+  "dest": {
+    "index": "products_active"
+  }
+}
+```
+
+### 冲突处理
+
+```bash
+# 跳过已存在的文档 (op_type=create)
+POST /_reindex
+{
+  "source": { "index": "products_v1" },
+  "dest": {
+    "index": "products_v2",
+    "op_type": "create"
+  }
+}
+
+# 或者使用 version_type=external 保留旧版本
+```
+
+### Reindex 大小和速度
+
+```bash
+# 限制批次大小 (默认 1000)
+POST /_reindex
+{
+  "source": {
+    "index": "products_v1",
+    "size": 5000
+  },
+  "dest": {
+    "index": "products_v2"
+  }
+}
+
+# 限制速率 (每秒文档数)
+PUT /_cluster/settings
+{
+  "transient": {
+    "indices.recovery.max_bytes_per_sec": "200mb"
+  }
+}
+```
+
+## 注意事项
+
+1. **大索引 Reindex**：建议 `slices=auto` 并行执行，可根据数据节点数自动调整并行度
+2. **源索引不影响**：Reindex 期间源索引可继续读写服务
+3. **Refresh 优化**：新索引设 `refresh_interval: -1`，迁移完再恢复
+4. **副本优化**：新索引设 `number_of_replicas: 0`，迁移完再调回
+5. **验证数据完整性**：迁移后对比文档数 (`_cat/count`)
+6. **善后清理**：确认新索引正常后删除旧索引释放空间

package/skills/database/elasticsearch/examples/04-cluster-monitoring.md

@@ -0,0 +1,204 @@
+# 示例: 集群监控实战
+
+## 场景
+
+日常运维需要监控 Elasticsearch 集群的健康状态、性能指标和资源使用情况。
+
+## 1. 集群健康检查
+
+### 快速健康状态
+
+```bash
+# 绿色 = 全部正常, 黄色 = 副本未分配, 红色 = 主分片丢失
+GET /_cluster/health?pretty
+```
+
+```json
+{
+  "cluster_name": "production",
+  "status": "yellow",
+  "timed_out": false,
+  "number_of_nodes": 5,
+  "number_of_data_nodes": 3,
+  "active_primary_shards": 125,
+  "active_shards": 240,
+  "relocating_shards": 0,
+  "initializing_shards": 0,
+  "unassigned_shards": 10,
+  "delayed_unassigned_shards": 0,
+  "active_shards_percent_as_number": 96.0
+}
+```
+
+**解读**：unassigned_shards=10, active_shards_percent=96% → 有副本未分配。检查节点是否宕机。
+
+### 详细健康诊断
+
+```bash
+# 查看所有未分配分片的原因
+GET /_cluster/allocation/explain?pretty
+
+# 重点关注字段:
+# - current_node: 当前所在节点
+# - can_remain_on_current_node: 能否保留
+# - can_rebalance_cluster: 能否再平衡
+# - node_decision: 节点决策
+```
+
+## 2. 节点监控
+
+### 节点概览
+
+```bash
+# 节点角色和资源使用
+GET /_cat/nodes?v&h=name,node.role,heap.percent,ram.percent,cpu,load_1m,disk.used_percent,master
+
+# 示例输出:
+# name     node.role heap.percent ram.percent cpu load_1m disk.used_percent master
+# master-1 m                  42          35   8     2.3               42 *
+# data-1   d                  67          72  35    12.5               56 -
+# data-2   d                  55          60  28     9.8               52 -
+# coord-1  -                  48          45  15     4.1               38 -
+```
+
+**告警阈值**：
+- heap.percent > 85 → GC 压力大
+- cpu > 80 → 需要扩容
+- disk.used_percent > 85 → 需要清理或 ILM
+
+### 热点线程排查 (CPU 突增)
+
+```bash
+# 当集群 CPU 突然飙高时
+GET /_nodes/hot_threads
+
+# 返回每个节点的热点线程堆栈
+# 重点关注:
+# - bulk 线程 → 写入压力大
+# - search 线程 → 查询压力大
+# - merge 线程 → 段合并 (大批量写入后)
+# - GC 线程 → 内存问题
+```
+
+### 节点统计
+
+```bash
+# 获取节点级别详细统计
+GET /_nodes/stats/indices,os,process,jvm,fs,transport,http
+
+# 关注指标:
+# - indices.indexing: 写入速率
+# - indices.search: 查询速率
+# - jvm.mem.heap_used_percent: 堆内存使用
+# - jvm.gc.collectors.young.collection_time_in_millis: GC 时间
+# - os.cpu.percent: CPU
+# - fs.total.available_in_bytes: 磁盘空间
+```
+
+## 3. 索引监控
+
+### 索引概览
+
+```bash
+# 查看所有索引的大小和文档数
+GET /_cat/indices?v&h=index,docs.count,store.size,pri.store.size
+
+# 查看索引分片分布
+GET /_cat/shards?v
+
+# 按大小排序
+GET /_cat/indices?v&s=store.size:desc
+```
+
+### 段 (Segment) 监控
+
+```bash
+# 检查段的统计
+GET /my_index/_segments
+
+# 查看各索引段数
+GET /_cat/segments?v&h=index,shard,segment,size,committed,search
+```
+
+**段过多 (>100)** 表示需要 force_merge：`POST /my_index/_forcemerge?max_num_segments=1`
+
+## 4. 性能监控脚本
+
+### Shell 健康检查脚本
+
+```bash
+#!/bin/bash
+# es_health_check.sh
+ES_HOST="http://localhost:9200"
+
+# 集群健康
+echo "=== 集群健康 ==="
+curl -s "$ES_HOST/_cluster/health?pretty" | python3 -c "
+import json, sys
+h = json.load(sys.stdin)
+print(f'状态: {h[\"status\"]}')
+print(f'节点: {h[\"number_of_nodes\"]} (数据节点: {h[\"number_of_data_nodes\"]})')
+print(f'活跃分片: {h[\"active_shards\"]}/{h[\"active_primary_shards\"]}p')
+print(f'未分配: {h[\"unassigned_shards\"]}')
+print(f'活跃率: {h[\"active_shards_percent_as_number\"]:.1f}%')
+"
+
+# 节点资源
+echo -e "\n=== 节点资源 ==="
+curl -s "$ES_HOST/_cat/nodes?v&h=name,node.role,heap.percent,cpu,load_1m,disk.used_percent"
+
+# 索引 TOP 10 大索引
+echo -e "\n=== TOP 10 大索引 ==="
+curl -s "$ES_HOST/_cat/indices?v&h=index,docs.count,store.size&s=store.size:desc&limit=10"
+
+# JVM 堆内存 Top 节点
+echo -e "\n=== JVM 内存 (Top 5) ==="
+curl -s "$ES_HOST/_nodes/stats/jvm" | python3 -c "
+import json, sys
+nodes = json.load(sys.stdin)['nodes']
+for node_id, info in sorted(nodes.items(), key=lambda x: x[1]['jvm']['mem']['heap_used_percent'], reverse=True)[:5]:
+    name = info['name']
+    mem = info['jvm']['mem']
+    gc = info['jvm']['gc']['collectors']
+    print(f'{name}: heap={mem[\"heap_used_percent\"]}%  old_gc={gc[\"old\"][\"collection_count\"]}次({gc[\"old\"][\"collection_time_in_millis\"]//1000}s)')
+"
+```
+
+## 5. 告警规则建议
+
+| 指标 | 警告阈值 | 严重阈值 | 检查间隔 |
+|------|---------|---------|---------|
+| cluster health status | yellow > 5min | red | 1min |
+| heap usage | > 80% | > 90% | 1min |
+| CPU | > 70% | > 85% | 5min |
+| disk usage | > 80% | > 90% | 1min |
+| unassigned shards | > 0 | > 5 | 1min |
+| search latency P99 | > 1s | > 5s | 5min |
+| indexing latency P99 | > 500ms | > 2s | 5min |
+| GC old gen count | > 5/min | > 10/min | 1min |
+
+## 6. Kibana 监控配置
+
+```yaml
+# 启用 Kibana 监控 UI (Stack Monitoring)
+# Management → Stack Monitoring
+# 可查看:
+# - 集群概览 (节点/索引/分片)
+# - 节点 CPU/内存/磁盘/IO
+# - 索引搜索/写入速率
+# - GC 统计
+# - 慢查询 Top N
+```
+
+## 性能排查清单
+
+```
+1. 是否 Yellow/Red → 检查未分配分片 (allocation/explain)
+2. CPU 高 → hot_threads 排查
+3. 查询慢 → Profile API + 慢查询日志
+4. 写入慢 → 检查 refresh_interval + Bulk 优化
+5. 磁盘满 → ILM 清理 + force_merge
+6. GC 频繁 → 减少分片/字段, 增加堆内存
+7. Segments 多 → force_merge (只读索引)
+8. 深度分页 → 改用 search_after
+```