@clickzetta/cz-cli-linux-x64 0.3.4 → 0.3.5

package/bin/skills/clickzetta-data-science/references/setup.md

@@ -1,160 +0,0 @@
-# 环境搭建与项目配置
-
-## 环境搭建
-
-```bash
-# 方式 1：venv（推荐）
-python3.12 -m venv .venv
-source .venv/bin/activate          # macOS/Linux
-pip install clickzetta_zettapark_python clickzetta-connector-python \
-    python-dotenv pandas numpy scikit-learn pyarrow jupyterlab matplotlib seaborn \
-    -i https://pypi.tuna.tsinghua.edu.cn/simple
-
-# 方式 2：pyenv（需要切换 Python 版本时）
-pyenv install 3.12.9 && pyenv local 3.12.9
-python -m venv .venv && source .venv/bin/activate
-pip install clickzetta_zettapark_python clickzetta-connector-python \
-    python-dotenv pandas numpy scikit-learn pyarrow jupyterlab matplotlib seaborn \
-    -i https://pypi.tuna.tsinghua.edu.cn/simple
-
-# 方式 3：conda
-conda create -n lakehouse-ds python=3.12 -y && conda activate lakehouse-ds
-pip install clickzetta_zettapark_python clickzetta-connector-python \
-    python-dotenv pandas numpy scikit-learn pyarrow jupyterlab matplotlib seaborn \
-    -i https://pypi.tuna.tsinghua.edu.cn/simple
-```
-
-| 问题 | 修复 |
-|------|------|
-| Python 3.8/3.9 | `pyenv install 3.12.9` 或 `python3.12 -m venv .venv` |
-| `pyarrow` 版本冲突 | `pip install pyarrow==14.0.0` |
-| M1/M2 Mac 报错 | `pip install --no-binary :all:` 或改用 conda |
-| 连接超时 | VCluster 未启动，在 Studio 中手动启动 |
-
----
-
-## Jupyter Kernel 配置
-
-```bash
-# 注册 venv 为 Jupyter kernel（关键步骤，否则 notebook 用系统 Python）
-source .venv/bin/activate
-pip install ipykernel jupyterlab
-python -m ipykernel install --user --name lakehouse-ds --display-name "Python (lakehouse-ds)"
-
-# 启动 JupyterLab
-jupyter lab --port=8888
-```
-
-VS Code / Cursor：打开 `.ipynb` → 右上角 "Select Kernel" → 选 "Python (lakehouse-ds)"
-
-| 问题 | 修复 |
-|------|------|
-| `ModuleNotFoundError: clickzetta` | kernel 未选对，切换到注册的 venv kernel |
-| `.env` 读不到 | `load_dotenv(dotenv_path='../.env')` 指定路径 |
-| `to_pandas()` OOM | 加 `TABLESAMPLE ROW(1)` 或 `LIMIT` |
-| 图表不显示 | notebook 开头加 `%matplotlib inline` |
-
----
-
-## src/config.py 模板
-
-```python
-import os, sys
-from pathlib import Path
-from dotenv import load_dotenv
-from clickzetta.zettapark.session import Session
-import clickzetta
-
-# 多位置查找 .env
-for _p in [
-    Path(__file__).parent.parent / ".env",
-    Path.home() / ".config" / "kilo" / ".env",
-    Path.home() / ".czcode" / ".env",
-    Path.home() / ".env",
-]:
-    if _p.exists():
-        load_dotenv(dotenv_path=_p)
-        break
-
-def check_environment():
-    """在 00-env-check.ipynb 里调用，打印环境诊断。"""
-    ver = sys.version_info
-    if ver < (3, 10):
-        raise RuntimeError(
-            f"Python {ver.major}.{ver.minor} 不满足要求。ZettaPark 需要 Python 3.10+。\n"
-            "升级：brew install pyenv && pyenv install 3.12.9 && pyenv local 3.12.9"
-        )
-    print(f"✅ Python {ver.major}.{ver.minor}.{ver.micro}")
-    for pkg, mod in [
-        ("clickzetta_zettapark_python", "clickzetta.zettapark"),
-        ("clickzetta-connector-python", "clickzetta"),
-        ("pandas", "pandas"), ("python-dotenv", "dotenv"),
-    ]:
-        try:
-            m = __import__(mod.split(".")[0])
-            print(f"✅ {pkg}: {getattr(m, '__version__', 'ok')}")
-        except ImportError:
-            print(f"❌ {pkg}: 未安装 → pip install {pkg}")
-    try:
-        s = get_session()
-        print(f"✅ Lakehouse: {s.sql('SELECT current_workspace(), current_user()').collect()}")
-    except Exception as e:
-        print(f"❌ Lakehouse 连接失败: {e}")
-
-def get_session() -> Session:
-    return Session.builder.configs({
-        "service":   os.environ["CLICKZETTA_SERVICE"],
-        "instance":  os.environ["CLICKZETTA_INSTANCE"],
-        "workspace": os.environ["CLICKZETTA_WORKSPACE"],
-        "username":  os.environ["CLICKZETTA_USERNAME"],
-        "password":  os.environ["CLICKZETTA_PASSWORD"],
-        "vcluster":  os.environ.get("CLICKZETTA_VCLUSTER", "default_ap"),
-        "schema":    os.environ.get("CLICKZETTA_SCHEMA", "public"),
-    }).create()
-
-def get_connector_connection():
-    """仅用于 pd.read_sql。禁止用于 df.to_sql()。"""
-    return clickzetta.connect(
-        service=os.environ["CLICKZETTA_SERVICE"],
-        instance=os.environ["CLICKZETTA_INSTANCE"],
-        workspace=os.environ["CLICKZETTA_WORKSPACE"],
-        username=os.environ["CLICKZETTA_USERNAME"],
-        password=os.environ["CLICKZETTA_PASSWORD"],
-        vcluster=os.environ.get("CLICKZETTA_VCLUSTER", "default_ap"),
-        schema=os.environ.get("CLICKZETTA_SCHEMA", "public"),
-    )
-```
-
----
-
-## .env 模板
-
-```bash
-CLICKZETTA_SERVICE=cn-shanghai-alicloud.api.clickzetta.com
-CLICKZETTA_INSTANCE=<instance-id>
-CLICKZETTA_WORKSPACE=<workspace>
-CLICKZETTA_USERNAME=<username>
-CLICKZETTA_PASSWORD=<password>
-CLICKZETTA_VCLUSTER=default_ap
-CLICKZETTA_SCHEMA=ds_workspace
-```
-
-## pyproject.toml
-
-```toml
-[project]
-name = "my-lakehouse-ds-project"
-requires-python = ">=3.10"
-dependencies = [
-    "clickzetta_zettapark_python>=0.1.2",
-    "clickzetta-connector-python>=1.0.0",
-    "python-dotenv>=1.0.0",
-    "pandas>=2.0.0",
-    "numpy>=1.24.0",
-    "scikit-learn>=1.3.0",
-    "pyarrow>=14.0.0",
-    "jupyterlab>=4.0.0",
-    "matplotlib>=3.7.0",
-    "seaborn>=0.12.0",
-]
-```

package/bin/skills/clickzetta-data-science/references/stats-functions.md

@@ -1,195 +0,0 @@
-# 数据科学统计分析函数参考
-
----
-
-## 近似聚合函数（大表高效统计）
-
-### approx_count_distinct — 近似 UV
-
-```sql
--- 使用 HyperLogLog 算法，误差约 2%，比 COUNT(DISTINCT) 快 10x+
-SELECT approx_count_distinct(user_id) AS approx_uv
-FROM my_schema.events;
-
--- 按天统计 DAU
-SELECT
-    DATE(event_time) AS dt,
-    approx_count_distinct(user_id) AS dau
-FROM my_schema.events
-GROUP BY 1
-ORDER BY 1;
-```
-
-### approx_percentile — 近似分位数
-
-```sql
--- 中位数、四分位数、P95、P99
-SELECT
-    approx_percentile(amount, 0.25) AS p25,
-    approx_percentile(amount, 0.50) AS median,
-    approx_percentile(amount, 0.75) AS p75,
-    approx_percentile(amount, 0.95) AS p95,
-    approx_percentile(amount, 0.99) AS p99
-FROM my_schema.orders;
-
--- 分组分位数
-SELECT
-    category,
-    approx_percentile(price, 0.5) AS median_price
-FROM my_schema.products
-GROUP BY category;
-```
-
-### approx_histogram — 近似直方图
-
-```sql
--- 返回结构体数组：[{min, max, count}, ...]
-SELECT approx_histogram(amount, 10) AS hist
-FROM my_schema.orders;
-
--- 解析直方图（展开为行）
-SELECT
-    bucket.min AS bucket_min,
-    bucket.max AS bucket_max,
-    bucket.count AS bucket_count
-FROM (
-    SELECT EXPLODE(approx_histogram(amount, 10)) AS bucket
-    FROM my_schema.orders
-);
-```
-
-### approx_top_k — 近似 TOP-K 高频值
-
-```sql
--- 找出出现最多的前 10 个城市
-SELECT approx_top_k(city, 10) AS top_cities
-FROM my_schema.orders;
-
--- 返回结构体数组：[{value, count}, ...]
--- 解析展开（字段名是 value 和 count）
-SELECT item.value AS city, item.count AS cnt
-FROM (
-    SELECT EXPLODE(approx_top_k(city, 10)) AS item
-    FROM my_schema.orders
-)
-ORDER BY cnt DESC;
-```
-
----
-
-## 精确统计函数
-
-### percentile / median
-
-```sql
--- 精确中位数（小表用，大表用 approx_percentile）
-SELECT
-    percentile(amount, 0.5)  AS exact_median,
-    median(amount)           AS median_alias  -- 等价写法
-FROM my_schema.orders;
-
--- 多分位数
-SELECT percentile(amount, ARRAY(0.25, 0.5, 0.75, 0.9, 0.99))
-FROM my_schema.orders;
-```
-
----
-
-## TABLESAMPLE 采样
-
-```sql
--- ROW 模式：精确行级采样（适合 ML 训练集，< 1000万行）
-SELECT * FROM my_schema.events TABLESAMPLE ROW (10);      -- 精确 10%
-SELECT * FROM my_schema.events TABLESAMPLE ROW (5 ROWS);  -- 精确 5 行
-
--- SYSTEM 模式：文件级采样（适合大表快速预览，> 1000万行）
-SELECT * FROM my_schema.events TABLESAMPLE SYSTEM (0.1) LIMIT 50000;  -- 约 0.1%
-
--- 分层采样（按类别等比例采样）
-SELECT * FROM (
-    SELECT *,
-           ROW_NUMBER() OVER (PARTITION BY category ORDER BY RAND()) AS rn,
-           COUNT(*) OVER (PARTITION BY category) AS cat_total
-    FROM my_schema.products
-)
-WHERE rn <= CEIL(cat_total * 0.1);  -- 每类取 10%
-```
-
-| 场景 | 推荐模式 | 说明 |
-|---|---|---|
-| 快速数据预览 | SYSTEM | 极快，适合 > 100万行 |
-| ML 训练集构建 | ROW | 精确随机，保证代表性 |
-| 数据质量抽检 | SYSTEM | 快速抽样验证 |
-| 统计分析 | ROW | 精确概率采样 |
-
-> ⚠️ **注意**：TABLESAMPLE 在小表（< 数万行）上可能返回全部数据，百分比采样不精确。小表直接用 `LIMIT` 即可。
-
----
-
-## 窗口函数（时序/排名特征）
-
-```sql
--- 移动平均（7日）
-SELECT
-    dt,
-    revenue,
-    AVG(revenue) OVER (
-        ORDER BY dt
-        ROWS BETWEEN 6 PRECEDING AND CURRENT ROW
-    ) AS revenue_7d_ma
-FROM daily_stats;
-
--- 环比增长率
-SELECT
-    dt,
-    revenue,
-    LAG(revenue, 1) OVER (ORDER BY dt)  AS prev_revenue,
-    ROUND(100.0 * (revenue - LAG(revenue, 1) OVER (ORDER BY dt))
-          / NULLIF(LAG(revenue, 1) OVER (ORDER BY dt), 0), 2) AS mom_growth_pct
-FROM daily_stats;
-
--- 用户行为排名（RFM 分析）
-SELECT
-    user_id,
-    total_amount,
-    NTILE(5) OVER (ORDER BY total_amount DESC)  AS monetary_quintile,
-    NTILE(5) OVER (ORDER BY order_cnt DESC)     AS frequency_quintile,
-    NTILE(5) OVER (ORDER BY last_order_date DESC) AS recency_quintile
-FROM user_rfm;
-
--- 去重保留最新（数据清洗常用）
-SELECT * FROM (
-    SELECT *,
-           ROW_NUMBER() OVER (
-               PARTITION BY user_id
-               ORDER BY update_time DESC
-           ) AS rn
-    FROM my_schema.users_raw
-) WHERE rn = 1;
-```
-
----
-
-## 数据质量检查模板
-
-```sql
--- 一次性输出所有关键质量指标
-SELECT
-    COUNT(*)                                                    AS total_rows,
-    COUNT(DISTINCT user_id)                                     AS unique_users,
-    -- 缺失率
-    ROUND(100.0 * COUNT(*) FILTER (WHERE user_id IS NULL)
-          / COUNT(*), 2)                                        AS user_id_null_pct,
-    ROUND(100.0 * COUNT(*) FILTER (WHERE amount IS NULL)
-          / COUNT(*), 2)                                        AS amount_null_pct,
-    -- 异常值
-    SUM(CASE WHEN amount < 0 THEN 1 ELSE 0 END)                AS negative_amount_cnt,
-    SUM(CASE WHEN amount > 1000000 THEN 1 ELSE 0 END)          AS extreme_amount_cnt,
-    -- 时间范围
-    MIN(order_date)                                             AS earliest_date,
-    MAX(order_date)                                             AS latest_date,
-    -- 分布
-    approx_percentile(amount, 0.5)                             AS median_amount,
-    approx_percentile(amount, 0.99)                            AS p99_amount
-FROM my_schema.orders;
-```

package/bin/skills/clickzetta-data-science/references/write-and-infer.md

@@ -1,122 +0,0 @@
-# 数据写入、特征工程、模型推理示例
-
-## 数据写入
-
-| 场景 | 方式 |
-|------|------|
-| ZettaPark 可用（Python 3.10+） | `save_as_table()` 或 `create_dataframe().write` |
-| 本地 CSV/pandas 写入 | `session.create_dataframe(df).write.save_as_table()` |
-| Python 3.9 / ZettaPark 不可用 | cursor 批量 INSERT（见下方） |
-| **禁止** | `df.to_sql()`、SQLAlchemy `clickzetta://...` |
-
-```python
-# 方式 A：ZettaPark（推荐）
-session.sql("""
-    SELECT o.*, u.age_group FROM my_schema.orders_raw o
-    LEFT JOIN my_schema.users u ON o.user_id = u.user_id
-    WHERE o.amount > 0
-""").write.mode("overwrite").save_as_table("ds_workspace.orders_clean")
-
-# 方式 B：pandas → Lakehouse
-session.create_dataframe(local_df).write.mode("append").save_as_table("ds_workspace.features_v1")
-
-# 方式 C：cursor 批量 INSERT（fallback）
-import clickzetta, os
-conn = clickzetta.connect(
-    service=os.environ["CLICKZETTA_SERVICE"], instance=os.environ["CLICKZETTA_INSTANCE"],
-    workspace=os.environ["CLICKZETTA_WORKSPACE"], username=os.environ["CLICKZETTA_USERNAME"],
-    password=os.environ["CLICKZETTA_PASSWORD"],
-    vcluster=os.environ.get("CLICKZETTA_VCLUSTER", "default_ap"),
-    schema=os.environ.get("CLICKZETTA_SCHEMA", "public"),
-)
-cursor = conn.cursor()
-cursor.execute("CREATE TABLE IF NOT EXISTS ds_workspace.my_table (col1 STRING, col2 BIGINT, col3 DOUBLE)")
-rows = local_df.values.tolist()
-for i in range(0, len(rows), 500):
-    batch = rows[i:i+500]
-    vals = ",".join(f"({','.join(repr(v) for v in row)})" for row in batch)
-    cursor.execute(f"INSERT INTO ds_workspace.my_table VALUES {vals}")
-conn.close()
-```
-
-```sql
--- 设置中间表生命周期（30 天自动清理）
-ALTER TABLE ds_workspace.orders_clean SET PROPERTIES ('data_lifecycle' = '30');
-```
-
----
-
-## 特征工程
-
-```sql
--- SQL 侧（利用 Lakehouse 算力，推荐）
-SELECT
-    user_id,
-    COUNT(*)                                                    AS order_cnt_30d,
-    SUM(amount)                                                 AS total_amount_30d,
-    AVG(amount)                                                 AS avg_amount_30d,
-    STDDEV(amount)                                              AS std_amount_30d,
-    DATEDIFF('day', MIN(order_date), MAX(order_date))           AS active_days,
-    COUNT(DISTINCT DATE(order_date))                            AS active_day_cnt,
-    NTILE(10) OVER (ORDER BY SUM(amount) DESC)                  AS revenue_decile
-FROM my_schema.orders
-WHERE order_date >= CURRENT_DATE - INTERVAL 30 DAY
-GROUP BY user_id;
-```
-
-```python
-# ZettaPark 侧（Python 逻辑）
-from clickzetta.zettapark.functions import col, when
-
-features = session.table("ds_workspace.orders_clean") \
-    .with_column("is_high_value", when(col("amount") > 1000, 1).otherwise(0))
-
-df = features.to_pandas()
-
-from sklearn.preprocessing import StandardScaler
-df[['amount_scaled']] = StandardScaler().fit_transform(df[['amount']])
-
-session.create_dataframe(df).write.mode("overwrite").save_as_table("ds_workspace.features_final")
-```
-
----
-
-## 模型推理上线
-
-### BITMAP 用户画像
-
-```sql
-CREATE TABLE ds_workspace.user_tags AS
-SELECT tag_name, group_bitmap_state(user_id) AS user_bitmap
-FROM my_schema.user_behavior GROUP BY tag_name;
-
--- 人群交集
-SELECT bitmap_count(bitmap_and(
-    (SELECT user_bitmap FROM ds_workspace.user_tags WHERE tag_name = '高消费'),
-    (SELECT user_bitmap FROM ds_workspace.user_tags WHERE tag_name = '近30天活跃')
-)) AS target_user_count;
-```
-
-### SQL UDF 批量推理
-
-```sql
--- 调用已部署的模型 UDF（必须用完整 schema 路径）
-INSERT INTO ds_workspace.predictions
-SELECT user_id,
-       ds_workspace.credit_score_model(total_amount_30d, order_cnt_30d, active_days, avg_amount_30d) AS score,
-       CURRENT_TIMESTAMP() AS predict_time
-FROM ds_workspace.features_final;
-```
-
-### 向量检索
-
-```sql
-SELECT candidate_id,
-       cosine_distance(
-           (SELECT embedding FROM ds_workspace.user_embeddings WHERE user_id = 'target'),
-           embedding
-       ) AS similarity
-FROM ds_workspace.user_embeddings
-WHERE user_id != 'target'
-ORDER BY similarity LIMIT 10;
-```

package/bin/skills/clickzetta-data-science/references/zettapark-api.md

@@ -1,156 +0,0 @@
-# ZettaPark API 数据科学常用操作
-
-> 来源：https://www.yunqi.tech/documents/ZettaparkQuickStart
-> **Python 版本**：推荐 3.12（最低 3.10）。安装：`python3.12 -m venv .venv && pip install clickzetta_zettapark_python`
-
----
-
-## Session 创建
-
-```python
-from clickzetta.zettapark.session import Session
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-
-session = Session.builder.configs({
-    "service":   os.environ["CLICKZETTA_SERVICE"],
-    "instance":  os.environ["CLICKZETTA_INSTANCE"],
-    "workspace": os.environ["CLICKZETTA_WORKSPACE"],
-    "username":  os.environ["CLICKZETTA_USERNAME"],
-    "password":  os.environ["CLICKZETTA_PASSWORD"],
-    "vcluster":  os.environ["CLICKZETTA_VCLUSTER"],
-    "schema":    os.environ.get("CLICKZETTA_SCHEMA", "public"),
-    "hints": {
-        "sdk.job.timeout": 300,
-        "query_tag": "ds_notebook"
-    }
-}).create()
-```
-
----
-
-## 数据读取
-
-```python
-# 读取整张表
-df = session.table("my_schema.orders")
-
-# 执行 SQL 查询
-df = session.sql("SELECT * FROM my_schema.orders WHERE amount > 100")
-
-# 转为 pandas（小数据集）
-pandas_df = df.to_pandas()
-
-# 分批读取大表（避免 OOM）
-pandas_df = session.sql("""
-    SELECT * FROM my_schema.events
-    TABLESAMPLE ROW (1)   -- 1% 精确采样
-""").to_pandas()
-
-# 只获取前 N 行
-pandas_df = df.limit(10000).to_pandas()
-```
-
----
-
-## DataFrame 变换
-
-```python
-from clickzetta.zettapark.functions import col, when, lit, sum as F_sum, count as F_count, avg as F_avg
-
-# 过滤
-df_filtered = df.filter(col("amount") > 0)
-df_filtered = df.filter((col("status") == "COMPLETED") & (col("amount") > 100))
-
-# 选择列
-df_selected = df.select("user_id", "amount", "order_date")
-
-# 新增列
-df = df.with_column("log_amount", col("amount").cast("double"))
-df = df.with_column("is_high_value", when(col("amount") > 1000, 1).otherwise(0))
-
-# 聚合
-agg_df = df.group_by("user_id").agg(
-    F_sum("amount").as_("total_amount"),
-    F_count("order_id").as_("order_cnt"),
-    F_avg("amount").as_("avg_amount")
-)
-
-# JOIN
-result = orders.join(users, orders["user_id"] == users["user_id"], "left")
-
-# 排序
-df_sorted = df.sort(col("amount").desc())
-```
-
----
-
-## 数据写回
-
-```python
-# 覆盖写入（常用于特征表更新）
-df.write.mode("overwrite").save_as_table("ds_workspace.features_v1")
-
-# 追加写入（常用于预测结果）
-df.write.mode("append").save_as_table("ds_workspace.predictions")
-
-# pandas DataFrame 写回
-import pandas as pd
-local_df = pd.DataFrame({"user_id": [1, 2], "score": [0.8, 0.6]})
-session.create_dataframe(local_df).write.mode("overwrite") \
-    .save_as_table("ds_workspace.model_scores")
-```
-
----
-
-## 与 pandas/scikit-learn 集成
-
-```python
-import pandas as pd
-import numpy as np
-from sklearn.preprocessing import StandardScaler
-from sklearn.model_selection import train_test_split
-from sklearn.ensemble import GradientBoostingClassifier
-
-# 1. 从 Lakehouse 拉特征
-features_df = session.sql("""
-    SELECT user_id, total_amount_30d, order_cnt_30d,
-           active_days, avg_amount_30d, label
-    FROM ds_workspace.features_final
-""").to_pandas()
-
-# 2. 本地处理
-X = features_df.drop(["user_id", "label"], axis=1)
-y = features_df["label"]
-
-scaler = StandardScaler()
-X_scaled = scaler.fit_transform(X)
-
-X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2)
-
-# 3. 训练模型
-model = GradientBoostingClassifier(n_estimators=100)
-model.fit(X_train, y_train)
-
-# 4. 预测并写回
-features_df["predicted_score"] = model.predict_proba(X_scaled)[:, 1]
-session.create_dataframe(
-    features_df[["user_id", "predicted_score"]]
-).write.mode("overwrite").save_as_table("ds_workspace.predictions")
-
-# 5. 保存模型
-import joblib
-joblib.dump(model, "models/gbm_model.pkl")
-joblib.dump(scaler, "models/scaler.pkl")
-```
-
----
-
-## 注意事项
-
-- `to_pandas()` 会把数据全部拉到本地内存，大表必须先 `TABLESAMPLE` 或 `LIMIT`
-- `collect()` 返回 Row 对象列表，`to_pandas()` 返回 DataFrame，数据科学场景用后者
-- ZettaPark 的 DataFrame 操作是懒执行，只有 `to_pandas()`/`collect()`/`show()`/`save_as_table()` 才真正触发计算
-- 写回时推荐用 `ds_workspace` 这样的专属 Schema，与生产数据隔离