@clickzetta/cz-cli-linux-x64 0.3.0 → 0.3.1

package/bin/skills/clickzetta-data-science/references/setup.md

@@ -0,0 +1,160 @@
+# 环境搭建与项目配置
+
+## 环境搭建
+
+```bash
+# 方式 1：venv（推荐）
+python3.12 -m venv .venv
+source .venv/bin/activate          # macOS/Linux
+pip install clickzetta_zettapark_python clickzetta-connector-python \
+    python-dotenv pandas numpy scikit-learn pyarrow jupyterlab matplotlib seaborn \
+    -i https://pypi.tuna.tsinghua.edu.cn/simple
+
+# 方式 2：pyenv（需要切换 Python 版本时）
+pyenv install 3.12.9 && pyenv local 3.12.9
+python -m venv .venv && source .venv/bin/activate
+pip install clickzetta_zettapark_python clickzetta-connector-python \
+    python-dotenv pandas numpy scikit-learn pyarrow jupyterlab matplotlib seaborn \
+    -i https://pypi.tuna.tsinghua.edu.cn/simple
+
+# 方式 3：conda
+conda create -n lakehouse-ds python=3.12 -y && conda activate lakehouse-ds
+pip install clickzetta_zettapark_python clickzetta-connector-python \
+    python-dotenv pandas numpy scikit-learn pyarrow jupyterlab matplotlib seaborn \
+    -i https://pypi.tuna.tsinghua.edu.cn/simple
+```
+
+| 问题 | 修复 |
+|------|------|
+| Python 3.8/3.9 | `pyenv install 3.12.9` 或 `python3.12 -m venv .venv` |
+| `pyarrow` 版本冲突 | `pip install pyarrow==14.0.0` |
+| M1/M2 Mac 报错 | `pip install --no-binary :all:` 或改用 conda |
+| 连接超时 | VCluster 未启动，在 Studio 中手动启动 |
+
+---
+
+## Jupyter Kernel 配置
+
+```bash
+# 注册 venv 为 Jupyter kernel（关键步骤，否则 notebook 用系统 Python）
+source .venv/bin/activate
+pip install ipykernel jupyterlab
+python -m ipykernel install --user --name lakehouse-ds --display-name "Python (lakehouse-ds)"
+
+# 启动 JupyterLab
+jupyter lab --port=8888
+```
+
+VS Code / Cursor：打开 `.ipynb` → 右上角 "Select Kernel" → 选 "Python (lakehouse-ds)"
+
+| 问题 | 修复 |
+|------|------|
+| `ModuleNotFoundError: clickzetta` | kernel 未选对，切换到注册的 venv kernel |
+| `.env` 读不到 | `load_dotenv(dotenv_path='../.env')` 指定路径 |
+| `to_pandas()` OOM | 加 `TABLESAMPLE ROW(1)` 或 `LIMIT` |
+| 图表不显示 | notebook 开头加 `%matplotlib inline` |
+
+---
+
+## src/config.py 模板
+
+```python
+import os, sys
+from pathlib import Path
+from dotenv import load_dotenv
+from clickzetta.zettapark.session import Session
+import clickzetta
+
+# 多位置查找 .env
+for _p in [
+    Path(__file__).parent.parent / ".env",
+    Path.home() / ".config" / "kilo" / ".env",
+    Path.home() / ".czcode" / ".env",
+    Path.home() / ".env",
+]:
+    if _p.exists():
+        load_dotenv(dotenv_path=_p)
+        break
+
+def check_environment():
+    """在 00-env-check.ipynb 里调用，打印环境诊断。"""
+    ver = sys.version_info
+    if ver < (3, 10):
+        raise RuntimeError(
+            f"Python {ver.major}.{ver.minor} 不满足要求。ZettaPark 需要 Python 3.10+。\n"
+            "升级：brew install pyenv && pyenv install 3.12.9 && pyenv local 3.12.9"
+        )
+    print(f"✅ Python {ver.major}.{ver.minor}.{ver.micro}")
+    for pkg, mod in [
+        ("clickzetta_zettapark_python", "clickzetta.zettapark"),
+        ("clickzetta-connector-python", "clickzetta"),
+        ("pandas", "pandas"), ("python-dotenv", "dotenv"),
+    ]:
+        try:
+            m = __import__(mod.split(".")[0])
+            print(f"✅ {pkg}: {getattr(m, '__version__', 'ok')}")
+        except ImportError:
+            print(f"❌ {pkg}: 未安装 → pip install {pkg}")
+    try:
+        s = get_session()
+        print(f"✅ Lakehouse: {s.sql('SELECT current_workspace(), current_user()').collect()}")
+    except Exception as e:
+        print(f"❌ Lakehouse 连接失败: {e}")
+
+def get_session() -> Session:
+    return Session.builder.configs({
+        "service":   os.environ["CLICKZETTA_SERVICE"],
+        "instance":  os.environ["CLICKZETTA_INSTANCE"],
+        "workspace": os.environ["CLICKZETTA_WORKSPACE"],
+        "username":  os.environ["CLICKZETTA_USERNAME"],
+        "password":  os.environ["CLICKZETTA_PASSWORD"],
+        "vcluster":  os.environ.get("CLICKZETTA_VCLUSTER", "default_ap"),
+        "schema":    os.environ.get("CLICKZETTA_SCHEMA", "public"),
+    }).create()
+
+def get_connector_connection():
+    """仅用于 pd.read_sql。禁止用于 df.to_sql()。"""
+    return clickzetta.connect(
+        service=os.environ["CLICKZETTA_SERVICE"],
+        instance=os.environ["CLICKZETTA_INSTANCE"],
+        workspace=os.environ["CLICKZETTA_WORKSPACE"],
+        username=os.environ["CLICKZETTA_USERNAME"],
+        password=os.environ["CLICKZETTA_PASSWORD"],
+        vcluster=os.environ.get("CLICKZETTA_VCLUSTER", "default_ap"),
+        schema=os.environ.get("CLICKZETTA_SCHEMA", "public"),
+    )
+```
+
+---
+
+## .env 模板
+
+```bash
+CLICKZETTA_SERVICE=cn-shanghai-alicloud.api.clickzetta.com
+CLICKZETTA_INSTANCE=<instance-id>
+CLICKZETTA_WORKSPACE=<workspace>
+CLICKZETTA_USERNAME=<username>
+CLICKZETTA_PASSWORD=<password>
+CLICKZETTA_VCLUSTER=default_ap
+CLICKZETTA_SCHEMA=ds_workspace
+```
+
+## pyproject.toml
+
+```toml
+[project]
+name = "my-lakehouse-ds-project"
+requires-python = ">=3.10"
+dependencies = [
+    "clickzetta_zettapark_python>=0.1.2",
+    "clickzetta-connector-python>=1.0.0",
+    "python-dotenv>=1.0.0",
+    "pandas>=2.0.0",
+    "numpy>=1.24.0",
+    "scikit-learn>=1.3.0",
+    "pyarrow>=14.0.0",
+    "jupyterlab>=4.0.0",
+    "matplotlib>=3.7.0",
+    "seaborn>=0.12.0",
+]
+```

package/bin/skills/clickzetta-data-science/references/stats-functions.md

@@ -0,0 +1,195 @@
+# 数据科学统计分析函数参考
+
+---
+
+## 近似聚合函数（大表高效统计）
+
+### approx_count_distinct — 近似 UV
+
+```sql
+-- 使用 HyperLogLog 算法，误差约 2%，比 COUNT(DISTINCT) 快 10x+
+SELECT approx_count_distinct(user_id) AS approx_uv
+FROM my_schema.events;
+
+-- 按天统计 DAU
+SELECT
+    DATE(event_time) AS dt,
+    approx_count_distinct(user_id) AS dau
+FROM my_schema.events
+GROUP BY 1
+ORDER BY 1;
+```
+
+### approx_percentile — 近似分位数
+
+```sql
+-- 中位数、四分位数、P95、P99
+SELECT
+    approx_percentile(amount, 0.25) AS p25,
+    approx_percentile(amount, 0.50) AS median,
+    approx_percentile(amount, 0.75) AS p75,
+    approx_percentile(amount, 0.95) AS p95,
+    approx_percentile(amount, 0.99) AS p99
+FROM my_schema.orders;
+
+-- 分组分位数
+SELECT
+    category,
+    approx_percentile(price, 0.5) AS median_price
+FROM my_schema.products
+GROUP BY category;
+```
+
+### approx_histogram — 近似直方图
+
+```sql
+-- 返回结构体数组：[{min, max, count}, ...]
+SELECT approx_histogram(amount, 10) AS hist
+FROM my_schema.orders;
+
+-- 解析直方图（展开为行）
+SELECT
+    bucket.min AS bucket_min,
+    bucket.max AS bucket_max,
+    bucket.count AS bucket_count
+FROM (
+    SELECT EXPLODE(approx_histogram(amount, 10)) AS bucket
+    FROM my_schema.orders
+);
+```
+
+### approx_top_k — 近似 TOP-K 高频值
+
+```sql
+-- 找出出现最多的前 10 个城市
+SELECT approx_top_k(city, 10) AS top_cities
+FROM my_schema.orders;
+
+-- 返回结构体数组：[{value, count}, ...]
+-- 解析展开（字段名是 value 和 count）
+SELECT item.value AS city, item.count AS cnt
+FROM (
+    SELECT EXPLODE(approx_top_k(city, 10)) AS item
+    FROM my_schema.orders
+)
+ORDER BY cnt DESC;
+```
+
+---
+
+## 精确统计函数
+
+### percentile / median
+
+```sql
+-- 精确中位数（小表用，大表用 approx_percentile）
+SELECT
+    percentile(amount, 0.5)  AS exact_median,
+    median(amount)           AS median_alias  -- 等价写法
+FROM my_schema.orders;
+
+-- 多分位数
+SELECT percentile(amount, ARRAY(0.25, 0.5, 0.75, 0.9, 0.99))
+FROM my_schema.orders;
+```
+
+---
+
+## TABLESAMPLE 采样
+
+```sql
+-- ROW 模式：精确行级采样（适合 ML 训练集，< 1000万行）
+SELECT * FROM my_schema.events TABLESAMPLE ROW (10);      -- 精确 10%
+SELECT * FROM my_schema.events TABLESAMPLE ROW (5 ROWS);  -- 精确 5 行
+
+-- SYSTEM 模式：文件级采样（适合大表快速预览，> 1000万行）
+SELECT * FROM my_schema.events TABLESAMPLE SYSTEM (0.1) LIMIT 50000;  -- 约 0.1%
+
+-- 分层采样（按类别等比例采样）
+SELECT * FROM (
+    SELECT *,
+           ROW_NUMBER() OVER (PARTITION BY category ORDER BY RAND()) AS rn,
+           COUNT(*) OVER (PARTITION BY category) AS cat_total
+    FROM my_schema.products
+)
+WHERE rn <= CEIL(cat_total * 0.1);  -- 每类取 10%
+```
+
+| 场景 | 推荐模式 | 说明 |
+|---|---|---|
+| 快速数据预览 | SYSTEM | 极快，适合 > 100万行 |
+| ML 训练集构建 | ROW | 精确随机，保证代表性 |
+| 数据质量抽检 | SYSTEM | 快速抽样验证 |
+| 统计分析 | ROW | 精确概率采样 |
+
+> ⚠️ **注意**：TABLESAMPLE 在小表（< 数万行）上可能返回全部数据，百分比采样不精确。小表直接用 `LIMIT` 即可。
+
+---
+
+## 窗口函数（时序/排名特征）
+
+```sql
+-- 移动平均（7日）
+SELECT
+    dt,
+    revenue,
+    AVG(revenue) OVER (
+        ORDER BY dt
+        ROWS BETWEEN 6 PRECEDING AND CURRENT ROW
+    ) AS revenue_7d_ma
+FROM daily_stats;
+
+-- 环比增长率
+SELECT
+    dt,
+    revenue,
+    LAG(revenue, 1) OVER (ORDER BY dt)  AS prev_revenue,
+    ROUND(100.0 * (revenue - LAG(revenue, 1) OVER (ORDER BY dt))
+          / NULLIF(LAG(revenue, 1) OVER (ORDER BY dt), 0), 2) AS mom_growth_pct
+FROM daily_stats;
+
+-- 用户行为排名（RFM 分析）
+SELECT
+    user_id,
+    total_amount,
+    NTILE(5) OVER (ORDER BY total_amount DESC)  AS monetary_quintile,
+    NTILE(5) OVER (ORDER BY order_cnt DESC)     AS frequency_quintile,
+    NTILE(5) OVER (ORDER BY last_order_date DESC) AS recency_quintile
+FROM user_rfm;
+
+-- 去重保留最新（数据清洗常用）
+SELECT * FROM (
+    SELECT *,
+           ROW_NUMBER() OVER (
+               PARTITION BY user_id
+               ORDER BY update_time DESC
+           ) AS rn
+    FROM my_schema.users_raw
+) WHERE rn = 1;
+```
+
+---
+
+## 数据质量检查模板
+
+```sql
+-- 一次性输出所有关键质量指标
+SELECT
+    COUNT(*)                                                    AS total_rows,
+    COUNT(DISTINCT user_id)                                     AS unique_users,
+    -- 缺失率
+    ROUND(100.0 * COUNT(*) FILTER (WHERE user_id IS NULL)
+          / COUNT(*), 2)                                        AS user_id_null_pct,
+    ROUND(100.0 * COUNT(*) FILTER (WHERE amount IS NULL)
+          / COUNT(*), 2)                                        AS amount_null_pct,
+    -- 异常值
+    SUM(CASE WHEN amount < 0 THEN 1 ELSE 0 END)                AS negative_amount_cnt,
+    SUM(CASE WHEN amount > 1000000 THEN 1 ELSE 0 END)          AS extreme_amount_cnt,
+    -- 时间范围
+    MIN(order_date)                                             AS earliest_date,
+    MAX(order_date)                                             AS latest_date,
+    -- 分布
+    approx_percentile(amount, 0.5)                             AS median_amount,
+    approx_percentile(amount, 0.99)                            AS p99_amount
+FROM my_schema.orders;
+```

package/bin/skills/clickzetta-data-science/references/write-and-infer.md

@@ -0,0 +1,122 @@
+# 数据写入、特征工程、模型推理示例
+
+## 数据写入
+
+| 场景 | 方式 |
+|------|------|
+| ZettaPark 可用（Python 3.10+） | `save_as_table()` 或 `create_dataframe().write` |
+| 本地 CSV/pandas 写入 | `session.create_dataframe(df).write.save_as_table()` |
+| Python 3.9 / ZettaPark 不可用 | cursor 批量 INSERT（见下方） |
+| **禁止** | `df.to_sql()`、SQLAlchemy `clickzetta://...` |
+
+```python
+# 方式 A：ZettaPark（推荐）
+session.sql("""
+    SELECT o.*, u.age_group FROM my_schema.orders_raw o
+    LEFT JOIN my_schema.users u ON o.user_id = u.user_id
+    WHERE o.amount > 0
+""").write.mode("overwrite").save_as_table("ds_workspace.orders_clean")
+
+# 方式 B：pandas → Lakehouse
+session.create_dataframe(local_df).write.mode("append").save_as_table("ds_workspace.features_v1")
+
+# 方式 C：cursor 批量 INSERT（fallback）
+import clickzetta, os
+conn = clickzetta.connect(
+    service=os.environ["CLICKZETTA_SERVICE"], instance=os.environ["CLICKZETTA_INSTANCE"],
+    workspace=os.environ["CLICKZETTA_WORKSPACE"], username=os.environ["CLICKZETTA_USERNAME"],
+    password=os.environ["CLICKZETTA_PASSWORD"],
+    vcluster=os.environ.get("CLICKZETTA_VCLUSTER", "default_ap"),
+    schema=os.environ.get("CLICKZETTA_SCHEMA", "public"),
+)
+cursor = conn.cursor()
+cursor.execute("CREATE TABLE IF NOT EXISTS ds_workspace.my_table (col1 STRING, col2 BIGINT, col3 DOUBLE)")
+rows = local_df.values.tolist()
+for i in range(0, len(rows), 500):
+    batch = rows[i:i+500]
+    vals = ",".join(f"({','.join(repr(v) for v in row)})" for row in batch)
+    cursor.execute(f"INSERT INTO ds_workspace.my_table VALUES {vals}")
+conn.close()
+```
+
+```sql
+-- 设置中间表生命周期（30 天自动清理）
+ALTER TABLE ds_workspace.orders_clean SET PROPERTIES ('data_lifecycle' = '30');
+```
+
+---
+
+## 特征工程
+
+```sql
+-- SQL 侧（利用 Lakehouse 算力，推荐）
+SELECT
+    user_id,
+    COUNT(*)                                                    AS order_cnt_30d,
+    SUM(amount)                                                 AS total_amount_30d,
+    AVG(amount)                                                 AS avg_amount_30d,
+    STDDEV(amount)                                              AS std_amount_30d,
+    DATEDIFF('day', MIN(order_date), MAX(order_date))           AS active_days,
+    COUNT(DISTINCT DATE(order_date))                            AS active_day_cnt,
+    NTILE(10) OVER (ORDER BY SUM(amount) DESC)                  AS revenue_decile
+FROM my_schema.orders
+WHERE order_date >= CURRENT_DATE - INTERVAL 30 DAY
+GROUP BY user_id;
+```
+
+```python
+# ZettaPark 侧（Python 逻辑）
+from clickzetta.zettapark.functions import col, when
+
+features = session.table("ds_workspace.orders_clean") \
+    .with_column("is_high_value", when(col("amount") > 1000, 1).otherwise(0))
+
+df = features.to_pandas()
+
+from sklearn.preprocessing import StandardScaler
+df[['amount_scaled']] = StandardScaler().fit_transform(df[['amount']])
+
+session.create_dataframe(df).write.mode("overwrite").save_as_table("ds_workspace.features_final")
+```
+
+---
+
+## 模型推理上线
+
+### BITMAP 用户画像
+
+```sql
+CREATE TABLE ds_workspace.user_tags AS
+SELECT tag_name, group_bitmap_state(user_id) AS user_bitmap
+FROM my_schema.user_behavior GROUP BY tag_name;
+
+-- 人群交集
+SELECT bitmap_count(bitmap_and(
+    (SELECT user_bitmap FROM ds_workspace.user_tags WHERE tag_name = '高消费'),
+    (SELECT user_bitmap FROM ds_workspace.user_tags WHERE tag_name = '近30天活跃')
+)) AS target_user_count;
+```
+
+### SQL UDF 批量推理
+
+```sql
+-- 调用已部署的模型 UDF（必须用完整 schema 路径）
+INSERT INTO ds_workspace.predictions
+SELECT user_id,
+       ds_workspace.credit_score_model(total_amount_30d, order_cnt_30d, active_days, avg_amount_30d) AS score,
+       CURRENT_TIMESTAMP() AS predict_time
+FROM ds_workspace.features_final;
+```
+
+### 向量检索
+
+```sql
+SELECT candidate_id,
+       cosine_distance(
+           (SELECT embedding FROM ds_workspace.user_embeddings WHERE user_id = 'target'),
+           embedding
+       ) AS similarity
+FROM ds_workspace.user_embeddings
+WHERE user_id != 'target'
+ORDER BY similarity LIMIT 10;
+```

package/bin/skills/clickzetta-data-science/references/zettapark-api.md

@@ -0,0 +1,156 @@
+# ZettaPark API 数据科学常用操作
+
+> 来源：https://www.yunqi.tech/documents/ZettaparkQuickStart
+> **Python 版本**：推荐 3.12（最低 3.10）。安装：`python3.12 -m venv .venv && pip install clickzetta_zettapark_python`
+
+---
+
+## Session 创建
+
+```python
+from clickzetta.zettapark.session import Session
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+session = Session.builder.configs({
+    "service":   os.environ["CLICKZETTA_SERVICE"],
+    "instance":  os.environ["CLICKZETTA_INSTANCE"],
+    "workspace": os.environ["CLICKZETTA_WORKSPACE"],
+    "username":  os.environ["CLICKZETTA_USERNAME"],
+    "password":  os.environ["CLICKZETTA_PASSWORD"],
+    "vcluster":  os.environ["CLICKZETTA_VCLUSTER"],
+    "schema":    os.environ.get("CLICKZETTA_SCHEMA", "public"),
+    "hints": {
+        "sdk.job.timeout": 300,
+        "query_tag": "ds_notebook"
+    }
+}).create()
+```
+
+---
+
+## 数据读取
+
+```python
+# 读取整张表
+df = session.table("my_schema.orders")
+
+# 执行 SQL 查询
+df = session.sql("SELECT * FROM my_schema.orders WHERE amount > 100")
+
+# 转为 pandas（小数据集）
+pandas_df = df.to_pandas()
+
+# 分批读取大表（避免 OOM）
+pandas_df = session.sql("""
+    SELECT * FROM my_schema.events
+    TABLESAMPLE ROW (1)   -- 1% 精确采样
+""").to_pandas()
+
+# 只获取前 N 行
+pandas_df = df.limit(10000).to_pandas()
+```
+
+---
+
+## DataFrame 变换
+
+```python
+from clickzetta.zettapark.functions import col, when, lit, sum as F_sum, count as F_count, avg as F_avg
+
+# 过滤
+df_filtered = df.filter(col("amount") > 0)
+df_filtered = df.filter((col("status") == "COMPLETED") & (col("amount") > 100))
+
+# 选择列
+df_selected = df.select("user_id", "amount", "order_date")
+
+# 新增列
+df = df.with_column("log_amount", col("amount").cast("double"))
+df = df.with_column("is_high_value", when(col("amount") > 1000, 1).otherwise(0))
+
+# 聚合
+agg_df = df.group_by("user_id").agg(
+    F_sum("amount").as_("total_amount"),
+    F_count("order_id").as_("order_cnt"),
+    F_avg("amount").as_("avg_amount")
+)
+
+# JOIN
+result = orders.join(users, orders["user_id"] == users["user_id"], "left")
+
+# 排序
+df_sorted = df.sort(col("amount").desc())
+```
+
+---
+
+## 数据写回
+
+```python
+# 覆盖写入（常用于特征表更新）
+df.write.mode("overwrite").save_as_table("ds_workspace.features_v1")
+
+# 追加写入（常用于预测结果）
+df.write.mode("append").save_as_table("ds_workspace.predictions")
+
+# pandas DataFrame 写回
+import pandas as pd
+local_df = pd.DataFrame({"user_id": [1, 2], "score": [0.8, 0.6]})
+session.create_dataframe(local_df).write.mode("overwrite") \
+    .save_as_table("ds_workspace.model_scores")
+```
+
+---
+
+## 与 pandas/scikit-learn 集成
+
+```python
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import StandardScaler
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import GradientBoostingClassifier
+
+# 1. 从 Lakehouse 拉特征
+features_df = session.sql("""
+    SELECT user_id, total_amount_30d, order_cnt_30d,
+           active_days, avg_amount_30d, label
+    FROM ds_workspace.features_final
+""").to_pandas()
+
+# 2. 本地处理
+X = features_df.drop(["user_id", "label"], axis=1)
+y = features_df["label"]
+
+scaler = StandardScaler()
+X_scaled = scaler.fit_transform(X)
+
+X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2)
+
+# 3. 训练模型
+model = GradientBoostingClassifier(n_estimators=100)
+model.fit(X_train, y_train)
+
+# 4. 预测并写回
+features_df["predicted_score"] = model.predict_proba(X_scaled)[:, 1]
+session.create_dataframe(
+    features_df[["user_id", "predicted_score"]]
+).write.mode("overwrite").save_as_table("ds_workspace.predictions")
+
+# 5. 保存模型
+import joblib
+joblib.dump(model, "models/gbm_model.pkl")
+joblib.dump(scaler, "models/scaler.pkl")
+```
+
+---
+
+## 注意事项
+
+- `to_pandas()` 会把数据全部拉到本地内存，大表必须先 `TABLESAMPLE` 或 `LIMIT`
+- `collect()` 返回 Row 对象列表，`to_pandas()` 返回 DataFrame，数据科学场景用后者
+- ZettaPark 的 DataFrame 操作是懒执行，只有 `to_pandas()`/`collect()`/`show()`/`save_as_table()` 才真正触发计算
+- 写回时推荐用 `ds_workspace` 这样的专属 Schema，与生产数据隔离