@clickzetta/cz-cli-linux-x64 0.3.2 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/bin/cz-cli +0 -0
  2. package/package.json +1 -1
  3. package/bin/skills/clickzetta-access-control/SKILL.md +0 -243
  4. package/bin/skills/clickzetta-access-control/references/dynamic-masking.md +0 -86
  5. package/bin/skills/clickzetta-access-control/references/grant-revoke.md +0 -103
  6. package/bin/skills/clickzetta-access-control/references/role-management.md +0 -66
  7. package/bin/skills/clickzetta-access-control/references/user-management.md +0 -61
  8. package/bin/skills/clickzetta-ai-vector-search/SKILL.md +0 -160
  9. package/bin/skills/clickzetta-ai-vector-search/references/vector-search.md +0 -155
  10. package/bin/skills/clickzetta-app-python-sdk/SKILL.md +0 -153
  11. package/bin/skills/clickzetta-app-python-sdk/references/bulkload.md +0 -196
  12. package/bin/skills/clickzetta-app-python-sdk/references/connector.md +0 -143
  13. package/bin/skills/clickzetta-app-python-sdk/references/realtime.md +0 -122
  14. package/bin/skills/clickzetta-batch-sync-pipeline/SKILL.md +0 -293
  15. package/bin/skills/clickzetta-bi-connect/SKILL.md +0 -176
  16. package/bin/skills/clickzetta-bi-connect/references/bi-tools.md +0 -170
  17. package/bin/skills/clickzetta-cdc-sync-pipeline/SKILL.md +0 -450
  18. package/bin/skills/clickzetta-concepts/SKILL.md +0 -282
  19. package/bin/skills/clickzetta-concepts/references/brands-and-endpoints.md +0 -79
  20. package/bin/skills/clickzetta-concepts/references/object-model.md +0 -311
  21. package/bin/skills/clickzetta-data-ingest-pipeline/SKILL.md +0 -165
  22. package/bin/skills/clickzetta-data-lifecycle/SKILL.md +0 -211
  23. package/bin/skills/clickzetta-data-lifecycle/references/lifecycle-reference.md +0 -175
  24. package/bin/skills/clickzetta-data-recovery/SKILL.md +0 -215
  25. package/bin/skills/clickzetta-data-recovery/evals/evals.json +0 -35
  26. package/bin/skills/clickzetta-data-science/SKILL.md +0 -125
  27. package/bin/skills/clickzetta-data-science/references/bitmap-profile.md +0 -146
  28. package/bin/skills/clickzetta-data-science/references/data-patterns.md +0 -110
  29. package/bin/skills/clickzetta-data-science/references/setup.md +0 -160
  30. package/bin/skills/clickzetta-data-science/references/stats-functions.md +0 -195
  31. package/bin/skills/clickzetta-data-science/references/write-and-infer.md +0 -122
  32. package/bin/skills/clickzetta-data-science/references/zettapark-api.md +0 -156
  33. package/bin/skills/clickzetta-data-sharing/SKILL.md +0 -160
  34. package/bin/skills/clickzetta-data-sharing/references/share-ddl.md +0 -134
  35. package/bin/skills/clickzetta-dba-guide/SKILL.md +0 -540
  36. package/bin/skills/clickzetta-dw-modeling/SKILL.md +0 -259
  37. package/bin/skills/clickzetta-dw-modeling/references/modeling-patterns.md +0 -100
  38. package/bin/skills/clickzetta-dynamic-table/SKILL.md +0 -86
  39. package/bin/skills/clickzetta-dynamic-table/best-practices/dimension-table-join-guide.md +0 -257
  40. package/bin/skills/clickzetta-dynamic-table/best-practices/medallion-and-stream-patterns.md +0 -124
  41. package/bin/skills/clickzetta-dynamic-table/best-practices/non-partitioned-merge-into-warning.md +0 -96
  42. package/bin/skills/clickzetta-dynamic-table/best-practices/performance-optimization.md +0 -109
  43. package/bin/skills/clickzetta-dynamic-table/dt-creator/SKILL.md +0 -15
  44. package/bin/skills/clickzetta-dynamic-table/dt-creator/references/dt-declaration-strategy.md +0 -185
  45. package/bin/skills/clickzetta-dynamic-table/dt-creator/references/incremental-config-reference.md +0 -429
  46. package/bin/skills/clickzetta-dynamic-table/dt-creator/references/refresh-history-guide.md +0 -268
  47. package/bin/skills/clickzetta-dynamic-table/dt-creator/references/sql-limitations.md +0 -80
  48. package/bin/skills/clickzetta-dynamic-table/dynamic-table-alter/SKILL.md +0 -190
  49. package/bin/skills/clickzetta-external-catalog/SKILL.md +0 -120
  50. package/bin/skills/clickzetta-external-catalog/references/external-catalog-ddl.md +0 -130
  51. package/bin/skills/clickzetta-external-function/SKILL.md +0 -203
  52. package/bin/skills/clickzetta-external-function/references/external-function-ddl.md +0 -171
  53. package/bin/skills/clickzetta-file-import-pipeline/SKILL.md +0 -117
  54. package/bin/skills/clickzetta-index-manager/SKILL.md +0 -140
  55. package/bin/skills/clickzetta-index-manager/references/bloomfilter-index.md +0 -67
  56. package/bin/skills/clickzetta-index-manager/references/index-management.md +0 -73
  57. package/bin/skills/clickzetta-index-manager/references/inverted-index.md +0 -80
  58. package/bin/skills/clickzetta-index-manager/references/vector-index.md +0 -81
  59. package/bin/skills/clickzetta-information-schema/SKILL.md +0 -367
  60. package/bin/skills/clickzetta-information-schema/references/instance-views-reference.md +0 -276
  61. package/bin/skills/clickzetta-information-schema/references/metering-views-reference.md +0 -137
  62. package/bin/skills/clickzetta-information-schema/references/views-reference.md +0 -271
  63. package/bin/skills/clickzetta-java-sdk/SKILL.md +0 -186
  64. package/bin/skills/clickzetta-java-sdk/references/bulkload.md +0 -163
  65. package/bin/skills/clickzetta-java-sdk/references/realtime.md +0 -212
  66. package/bin/skills/clickzetta-kafka-ingest-pipeline/SKILL.md +0 -531
  67. package/bin/skills/clickzetta-kafka-ingest-pipeline/references/kafka-pipe-syntax.md +0 -186
  68. package/bin/skills/clickzetta-lakehouse-connect/SKILL.md +0 -218
  69. package/bin/skills/clickzetta-lakehouse-connect/evals/evals.json +0 -35
  70. package/bin/skills/clickzetta-lakehouse-connect/references/config-file.md +0 -435
  71. package/bin/skills/clickzetta-lakehouse-connect/references/jdbc.md +0 -478
  72. package/bin/skills/clickzetta-lakehouse-connect/references/python-sdk.md +0 -225
  73. package/bin/skills/clickzetta-lakehouse-connect/references/sqlalchemy.md +0 -468
  74. package/bin/skills/clickzetta-lakehouse-connect/references/zettapark-session.md +0 -445
  75. package/bin/skills/clickzetta-manage-comments/SKILL.md +0 -219
  76. package/bin/skills/clickzetta-metadata-query/SKILL.md +0 -298
  77. package/bin/skills/clickzetta-metadata-query/references/show-desc-reference.md +0 -326
  78. package/bin/skills/clickzetta-monitoring/SKILL.md +0 -199
  79. package/bin/skills/clickzetta-monitoring/references/job-history-analysis.md +0 -97
  80. package/bin/skills/clickzetta-monitoring/references/show-jobs.md +0 -48
  81. package/bin/skills/clickzetta-oss-ingest-pipeline/SKILL.md +0 -402
  82. package/bin/skills/clickzetta-query-optimizer/SKILL.md +0 -156
  83. package/bin/skills/clickzetta-query-optimizer/references/explain.md +0 -56
  84. package/bin/skills/clickzetta-query-optimizer/references/hints-and-sortkey.md +0 -78
  85. package/bin/skills/clickzetta-query-optimizer/references/optimize.md +0 -65
  86. package/bin/skills/clickzetta-query-optimizer/references/result-cache.md +0 -49
  87. package/bin/skills/clickzetta-query-optimizer/references/show-jobs.md +0 -42
  88. package/bin/skills/clickzetta-realtime-sync-pipeline/SKILL.md +0 -197
  89. package/bin/skills/clickzetta-semantic-view/SKILL.md +0 -207
  90. package/bin/skills/clickzetta-semantic-view/references/semantic-view-reference.md +0 -167
  91. package/bin/skills/clickzetta-spark-flink-connector/SKILL.md +0 -92
  92. package/bin/skills/clickzetta-spark-flink-connector/references/flink.md +0 -147
  93. package/bin/skills/clickzetta-spark-flink-connector/references/spark.md +0 -132
  94. package/bin/skills/clickzetta-sql-pipeline-manager/SKILL.md +0 -353
  95. package/bin/skills/clickzetta-sql-pipeline-manager/evals/evals.json +0 -166
  96. package/bin/skills/clickzetta-sql-pipeline-manager/references/dynamic-table.md +0 -173
  97. package/bin/skills/clickzetta-sql-pipeline-manager/references/materialized-view.md +0 -129
  98. package/bin/skills/clickzetta-sql-pipeline-manager/references/pipe.md +0 -160
  99. package/bin/skills/clickzetta-sql-pipeline-manager/references/table-stream.md +0 -123
  100. package/bin/skills/clickzetta-sql-syntax-guide/SKILL.md +0 -172
  101. package/bin/skills/clickzetta-sql-syntax-guide/references/ddl-reference.md +0 -350
  102. package/bin/skills/clickzetta-sql-syntax-guide/references/dml-reference.md +0 -279
  103. package/bin/skills/clickzetta-sql-syntax-guide/references/dql-reference.md +0 -504
  104. package/bin/skills/clickzetta-sql-syntax-guide/references/functions-reference.md +0 -372
  105. package/bin/skills/clickzetta-sql-syntax-guide/references/migration-databricks.md +0 -260
  106. package/bin/skills/clickzetta-sql-syntax-guide/references/migration-snowflake.md +0 -382
  107. package/bin/skills/clickzetta-sql-syntax-guide/references/vs-snowflake.md +0 -346
  108. package/bin/skills/clickzetta-sql-syntax-guide/references/vs-spark.md +0 -229
  109. package/bin/skills/clickzetta-studio-overview/SKILL.md +0 -170
  110. package/bin/skills/clickzetta-studio-overview/references/studio-modules.md +0 -173
  111. package/bin/skills/clickzetta-table-stream-pipeline/SKILL.md +0 -155
  112. package/bin/skills/clickzetta-vcluster-manager/SKILL.md +0 -212
  113. package/bin/skills/clickzetta-vcluster-manager/references/vc-cache.md +0 -54
  114. package/bin/skills/clickzetta-vcluster-manager/references/vcluster-ddl.md +0 -150
  115. package/bin/skills/clickzetta-volume-manager/SKILL.md +0 -249
  116. package/bin/skills/clickzetta-volume-manager/references/volume-ddl.md +0 -194
  117. package/bin/skills/clickzetta-zettapark/SKILL.md +0 -248
  118. package/bin/skills/clickzetta-zettapark/references/zettapark-api.md +0 -283
@@ -1,173 +0,0 @@
1
- # Dynamic Table(动态表)SQL 参考
2
-
3
- > **⚠️ ClickZetta 特有语法**
4
- > - 刷新调度写法:`REFRESH interval 5 MINUTE VCLUSTER default`(不是 `TARGET_LAG`)
5
- > - 修改调度周期或计算集群必须用 `CREATE OR REPLACE`,`ALTER` 不支持
6
- > - `ALTER DYNAMIC TABLE` 只支持:SUSPEND / RESUME / SET COMMENT / RENAME COLUMN
7
-
8
- 动态表是 ClickZetta Lakehouse 的核心增量计算对象。通过 SQL 查询定义,自动增量刷新,无需手动调度。
9
-
10
- ## CREATE DYNAMIC TABLE
11
-
12
- ```sql
13
- CREATE [ OR REPLACE ] DYNAMIC TABLE <name>
14
- [ (<column_list>) ]
15
- [ PARTITIONED BY (<col_name>) ]
16
- [ CLUSTERED BY (<col_name>) ]
17
- [ COMMENT <comment> ]
18
- [ PROPERTIES ( data_lifecycle = <day_num> ) ]
19
- REFRESH [ START WITH <timestamp_expr> ] interval <n> { SECOND | MINUTE | HOUR | DAY }
20
- VCLUSTER <vcluster_name>
21
- AS
22
- <query>;
23
- ```
24
-
25
- **关键参数:**
26
- - `REFRESH interval <n> MINUTE`:刷新间隔,最小 1 分钟
27
- - `VCLUSTER`:运行刷新任务的计算集群名称(直接跟名称,不带等号和引号)
28
- - `OR REPLACE`:若同名动态表已存在则替换(修改 SQL 逻辑或调度配置必须用此方式)
29
-
30
- **示例:**
31
- ```sql
32
- -- 基础示例:每 5 分钟刷新一次订单汇总
33
- CREATE OR REPLACE DYNAMIC TABLE dw.order_summary
34
- REFRESH interval 5 MINUTE
35
- VCLUSTER default
36
- AS
37
- SELECT
38
- date_trunc('hour', created_at) AS hour,
39
- region,
40
- COUNT(*) AS order_cnt,
41
- SUM(amount) AS total_amount
42
- FROM ods.orders
43
- GROUP BY 1, 2;
44
-
45
- -- 修改调度周期(必须用 CREATE OR REPLACE)
46
- CREATE OR REPLACE DYNAMIC TABLE dw.order_summary
47
- REFRESH interval 10 MINUTE
48
- VCLUSTER default
49
- AS
50
- SELECT
51
- date_trunc('hour', created_at) AS hour,
52
- region,
53
- COUNT(*) AS order_cnt,
54
- SUM(amount) AS total_amount
55
- FROM ods.orders
56
- GROUP BY 1, 2;
57
- ```
58
-
59
- ## ALTER DYNAMIC TABLE
60
-
61
- ```sql
62
- -- 暂停刷新
63
- ALTER DYNAMIC TABLE <name> SUSPEND;
64
-
65
- -- 恢复刷新
66
- ALTER DYNAMIC TABLE <name> RESUME;
67
-
68
- -- 修改注释
69
- ALTER DYNAMIC TABLE <name> SET COMMENT <comment>;
70
-
71
- -- 修改列名
72
- ALTER DYNAMIC TABLE <name> RENAME COLUMN <old_col> TO <new_col>;
73
-
74
- -- 修改列注释
75
- ALTER DYNAMIC TABLE <name> CHANGE COLUMN <col_name> COMMENT <comment>;
76
-
77
- -- 修改属性
78
- ALTER DYNAMIC TABLE <name> SET PROPERTIES ( key = value );
79
- ALTER DYNAMIC TABLE <name> UNSET PROPERTIES ( key );
80
- ```
81
-
82
- > 注意:修改调度周期、计算集群、SQL 查询逻辑,必须用 `CREATE OR REPLACE DYNAMIC TABLE`,ALTER 不支持这些操作。
83
-
84
- ## REFRESH DYNAMIC TABLE(手动触发)
85
-
86
- ```sql
87
- -- 手动触发一次刷新
88
- REFRESH DYNAMIC TABLE <name>;
89
- ```
90
-
91
- ## DROP DYNAMIC TABLE
92
-
93
- ```sql
94
- DROP DYNAMIC TABLE [ IF EXISTS ] <name>;
95
- ```
96
-
97
- ## SHOW / DESC
98
-
99
- ```sql
100
- -- 列出当前 schema 下所有动态表
101
- SHOW TABLES WHERE is_dynamic = true;
102
-
103
- -- 列出指定 schema 下的动态表
104
- SHOW TABLES IN <schema_name> WHERE is_dynamic = true;
105
-
106
- -- 查看动态表结构和调度配置
107
- DESC TABLE <name>;
108
- DESCRIBE DYNAMIC TABLE <name> EXTENDED;
109
-
110
- -- 查看完整建表语句
111
- SHOW CREATE TABLE <name>;
112
-
113
- -- 查看刷新历史(状态、耗时、触发方式、增量行数)
114
- SHOW DYNAMIC TABLE REFRESH HISTORY <name>;
115
- SHOW DYNAMIC TABLE REFRESH HISTORY <name> LIMIT 20;
116
- ```
117
-
118
- ## 注意事项
119
-
120
- - 修改 SQL 逻辑、调度周期、计算集群 → 用 `CREATE OR REPLACE`,不能用 `ALTER`
121
- - 刷新间隔最小 1 分钟
122
- - 动态表支持 Time Travel(`TIMESTAMP AS OF`)和 UNDROP
123
- - 刷新失败不影响表的可查询性(返回上次成功版本的数据)
124
- - 非简单加列/减列的 `CREATE OR REPLACE` 会触发一次全量刷新
125
-
126
- ## 参数化动态表(SESSION_CONFIGS)
127
-
128
- 通过 `SESSION_CONFIGS()` 函数定义参数化查询,在刷新时传入分区值控制刷新范围:
129
-
130
- ```sql
131
- -- 创建参数化动态表
132
- CREATE OR REPLACE DYNAMIC TABLE dwd.orders_partitioned
133
- REFRESH interval 30 MINUTE
134
- VCLUSTER default
135
- AS
136
- SELECT order_id, user_id, amount, dt
137
- FROM ods.orders
138
- WHERE dt = SESSION_CONFIGS('target_date', CAST(CURRENT_DATE() AS STRING));
139
-
140
- -- 手动触发刷新并传入参数
141
- REFRESH DYNAMIC TABLE dwd.orders_partitioned
142
- WITH PROPERTIES ('target_date' = '2024-06-15');
143
- ```
144
-
145
- 适用场景:传统按天全量 ETL 改造为增量任务,用 SESSION_CONFIGS 替换调度变量。
146
-
147
- ## 动态表 DML 操作
148
-
149
- 动态表默认不支持 DML,需先开启参数:
150
-
151
- ```sql
152
- -- 开启 DML 支持
153
- ALTER DYNAMIC TABLE <name> SET PROPERTIES ('cz.dynamic.table.enable.dml' = 'true');
154
-
155
- -- 支持的 DML 操作
156
- INSERT INTO <name> VALUES (...);
157
- DELETE FROM <name> WHERE ...;
158
- ```
159
-
160
- > ⚠️ 执行 DML 后,下一次自动刷新会触发**全量刷新**(而非增量),仅在数据修正等特殊场景使用。
161
-
162
- ## 参考文档
163
-
164
- - [CREATE DYNAMIC TABLE](https://www.yunqi.tech/documents/create-dynamic-table)
165
- - [ALTER DYNAMIC TABLE](https://www.yunqi.tech/documents/alter-dynamic-table)
166
- - [DROP DYNAMIC TABLE](https://www.yunqi.tech/documents/drop-dynamic-table)
167
- - [SHOW DYNAMIC TABLES](https://www.yunqi.tech/documents/show-dynamic-table)
168
- - [SHOW DYNAMIC TABLE REFRESH HISTORY](https://www.yunqi.tech/documents/refresh-history)
169
- - [动态表简介](https://www.yunqi.tech/documents/dynamic_table_summary)
170
- - [查看动态表刷新模式](https://www.yunqi.tech/documents/dynamic-table-incre)
171
- - [传统离线任务转增量实践](https://www.yunqi.tech/documents/transformt-dt)
172
- - [动态表支持参数化定义](https://www.yunqi.tech/documents/dynamicTable-parmaters)
173
- - [动态表支持DML语句修改](https://www.yunqi.tech/documents/dynamicTable-dml)
@@ -1,129 +0,0 @@
1
- # Materialized View(物化视图)SQL 参考
2
-
3
- > **⚠️ ClickZetta 特有语法**
4
- > - 定时刷新:`REFRESH INTERVAL 10 MINUTE vcluster default`(与动态表语法相同)
5
- > - 手动刷新:`REFRESH MATERIALIZED VIEW <name>;`
6
- > - 修改注释用 `ALTER TABLE`,不是 `ALTER MATERIALIZED VIEW`
7
-
8
- 物化视图将查询结果预计算并物理存储,适合固定维度的聚合加速场景。与动态表的区别:物化视图支持手动或定时刷新,不支持增量刷新。
9
-
10
- ## CREATE MATERIALIZED VIEW
11
-
12
- ```sql
13
- CREATE [ OR REPLACE ] MATERIALIZED VIEW <name>
14
- [ COMMENT = '<comment>' ]
15
- [ BUILD DEFERRED ]
16
- [ REFRESH INTERVAL <N> { SECOND | MINUTE | HOUR | DAY } vcluster <vcluster_name> ]
17
- [ DISABLE QUERY REWRITE ]
18
- AS
19
- <query>;
20
- ```
21
-
22
- **关键参数:**
23
- - `REFRESH INTERVAL 10 MINUTE vcluster default`:定时自动刷新(与动态表语法相同)
24
- - 不写 REFRESH 子句:只能手动触发 `REFRESH MATERIALIZED VIEW <name>;`
25
- - `BUILD DEFERRED`:延迟构建,创建时不立即计算结果
26
- - `DISABLE QUERY REWRITE`:禁用查询改写(不自动用 MV 加速查询)
27
-
28
- **示例:**
29
- ```sql
30
- -- 定时自动刷新的物化视图(每 10 分钟)
31
- CREATE MATERIALIZED VIEW mv_dept_stats
32
- REFRESH INTERVAL 10 MINUTE vcluster default
33
- AS
34
- SELECT
35
- d.dept_id,
36
- d.dept_name,
37
- COUNT(e.emp_id) AS emp_count,
38
- AVG(e.salary) AS avg_salary
39
- FROM departments d
40
- JOIN employees e ON d.dept_id = e.dept_id
41
- GROUP BY d.dept_id, d.dept_name;
42
-
43
- -- 修改刷新周期(需要 CREATE OR REPLACE)
44
- CREATE OR REPLACE MATERIALIZED VIEW mv_dept_stats
45
- BUILD DEFERRED
46
- REFRESH INTERVAL 20 MINUTE vcluster default
47
- DISABLE QUERY REWRITE
48
- AS
49
- SELECT
50
- d.dept_id,
51
- d.dept_name,
52
- d.location,
53
- ANY_VALUE(d.col1) AS col1,
54
- COUNT(e.emp_id) AS emp_count,
55
- AVG(e.salary) AS avg_salary
56
- FROM departments d
57
- JOIN employees e ON d.dept_id = e.dept_id
58
- GROUP BY d.dept_id, d.dept_name, d.location;
59
-
60
- -- 手动刷新
61
- REFRESH MATERIALIZED VIEW mv_dept_stats;
62
- ```
63
-
64
- ## ALTER MATERIALIZED VIEW
65
-
66
- ```sql
67
- -- 暂停自动刷新
68
- ALTER MATERIALIZED VIEW <name> SUSPEND;
69
-
70
- -- 恢复自动刷新
71
- ALTER MATERIALIZED VIEW <name> RESUME;
72
-
73
- -- 修改注释
74
- ALTER TABLE <mv_name> SET COMMENT '<comment>';
75
-
76
- -- 修改列注释(物化视图用 ALTER TABLE 语法)
77
- ALTER TABLE <mv_name> CHANGE COLUMN <col_name> COMMENT '<comment>';
78
- ```
79
-
80
- > 注意:物化视图的注释修改使用 `ALTER TABLE`,不是 `ALTER MATERIALIZED VIEW`。
81
-
82
- ## REFRESH MATERIALIZED VIEW
83
-
84
- ```sql
85
- -- 手动触发全量刷新
86
- REFRESH MATERIALIZED VIEW <name>;
87
- ```
88
-
89
- ## DROP MATERIALIZED VIEW
90
-
91
- ```sql
92
- DROP MATERIALIZED VIEW [ IF EXISTS ] <name>;
93
- ```
94
-
95
- ## SHOW / DESC
96
-
97
- ```sql
98
- -- 列出当前 schema 下所有物化视图
99
- SHOW TABLES WHERE is_materialized_view = true;
100
-
101
- -- 按名称过滤
102
- SHOW TABLES LIKE 'mv_%' WHERE is_materialized_view = true;
103
-
104
- -- 查看物化视图结构
105
- DESC MATERIALIZED VIEW <name>;
106
- DESCRIBE MATERIALIZED VIEW <name> EXTENDED;
107
-
108
- -- 查看完整建表语句
109
- SHOW CREATE TABLE <name>;
110
- ```
111
-
112
- ## 动态表 vs 物化视图 选择指南
113
-
114
- | 场景 | 推荐 |
115
- |---|---|
116
- | 需要秒/分钟级自动增量刷新 | Dynamic Table |
117
- | 固定聚合,手动或低频刷新 | Materialized View |
118
- | 需要 CDC 变更感知 | Dynamic Table + Table Stream |
119
- | 加速 BI 查询,数据不要求实时 | Materialized View |
120
-
121
- ## 参考文档
122
-
123
- - [CREATE MATERIALIZED VIEW](https://www.yunqi.tech/documents/CREATEMATERIALIZEDVIEW)
124
- - [ALTER MATERIALIZED VIEW](https://www.yunqi.tech/documents/alter-materialzied-view)
125
- - [REFRESH MATERIALIZED VIEW](https://www.yunqi.tech/documents/REFRESH)
126
- - [DROP MATERIALIZED VIEW](https://www.yunqi.tech/documents/DROPMATERIALIZEDVIEW)
127
- - [SHOW MATERIALIZED VIEWS](https://www.yunqi.tech/documents/show-materialized-view)
128
- - [物化视图概念与场景](https://www.yunqi.tech/documents/MATERIALIZEDVIEW)
129
- - [物化视图 DDL 汇总](https://www.yunqi.tech/documents/materialized_ddl)
@@ -1,160 +0,0 @@
1
- # Pipe SQL 参考
2
-
3
- > **⚠️ ClickZetta 特有语法**
4
- > - Kafka 读取函数是 `READ_KAFKA(...)`,不是 `KAFKA_SOURCE(...)` 或其他写法
5
- > - 参数使用 `=>` 命名参数语法:`KAFKA_BROKER => 'host:port'`
6
- > - JSON 字段提取用 `$1:field_name::TYPE` 语法(`$1` 表示整行 JSON)
7
- > - Pipe 创建后默认自动启动,无需手动 RESUME
8
-
9
- Pipe 是 ClickZetta Lakehouse 的持续数据导入对象,通过 SQL 定义从 Kafka 或对象存储(OSS/S3/COS)自动、持续地将数据导入目标表,无需外部调度。
10
-
11
- ## CREATE PIPE — 从 Kafka 导入
12
-
13
- ```sql
14
- CREATE [ OR REPLACE ] PIPE <pipe_name>
15
- [ COMMENT = '<comment>' ]
16
- [ AUTO_INGEST = { TRUE | FALSE } ]
17
- AS
18
- INSERT INTO <target_table> [ ( <col1>, <col2>, ... ) ]
19
- SELECT <expr> [, ...]
20
- FROM TABLE(
21
- READ_KAFKA(
22
- KAFKA_BROKER => '<broker_host>:<port>',
23
- KAFKA_TOPIC => '<topic_name>',
24
- KAFKA_GROUP_ID => '<consumer_group>',
25
- KAFKA_OFFSET => '<earliest | latest | <offset_value>>',
26
- KAFKA_DATA_FORMAT => '<json | csv | avro>',
27
- [ KAFKA_SASL_USERNAME => '<username>', ]
28
- [ KAFKA_SASL_PASSWORD => '<password>', ]
29
- [ KAFKA_SCHEMA_REGISTRY_URL => '<url>' ]
30
- )
31
- );
32
- ```
33
-
34
- **示例:**
35
- ```sql
36
- -- 从 Kafka 持续导入 JSON 数据
37
- CREATE OR REPLACE PIPE kafka_orders_pipe
38
- COMMENT '从 Kafka 持续导入订单数据'
39
- AS
40
- INSERT INTO ods.orders (order_id, user_id, amount, created_at)
41
- SELECT
42
- $1:order_id::STRING,
43
- $1:user_id::STRING,
44
- $1:amount::DECIMAL(10,2),
45
- $1:created_at::TIMESTAMP
46
- FROM TABLE(
47
- READ_KAFKA(
48
- KAFKA_BROKER => 'kafka.example.com:9092',
49
- KAFKA_TOPIC => 'orders',
50
- KAFKA_GROUP_ID => 'lakehouse_consumer',
51
- KAFKA_OFFSET => 'latest',
52
- KAFKA_DATA_FORMAT => 'json'
53
- )
54
- );
55
- ```
56
-
57
- ## CREATE PIPE — 从对象存储导入
58
-
59
- ```sql
60
- CREATE [ OR REPLACE ] PIPE [ IF NOT EXISTS ] <pipe_name>
61
- VIRTUAL_CLUSTER = <virtual_cluster_name>
62
- INGEST_MODE = { LIST_PURGE | EVENT_NOTIFICATION }
63
- [ COPY_JOB_HINT = '<hint>' ]
64
- AS
65
- COPY INTO <target_table>
66
- FROM VOLUME <volume_name>
67
- USING <csv | parquet | orc | json>
68
- [ OPTIONS ('<key>' = '<value>', ...) ];
69
- ```
70
-
71
- **关键参数:**
72
- - `VIRTUAL_CLUSTER`:指定虚拟集群名称(OSS Pipe 必填)
73
- - `INGEST_MODE = LIST_PURGE`:通用模式,定期扫描文件列表
74
- - `INGEST_MODE = EVENT_NOTIFICATION`:事件通知模式,低延迟(仅阿里云 OSS + AWS S3)
75
- - `FROM VOLUME <volume_name>`:引用已创建的 External Volume(不是 `FROM '@path/'`)
76
- - PIPE 中的 COPY 语句不支持 `files`、`regexp`、`subdirectory` 参数
77
-
78
- **示例:**
79
- ```sql
80
- -- 从 OSS Volume 持续导入 Parquet 文件(LIST_PURGE 模式)
81
- CREATE OR REPLACE PIPE oss_events_pipe
82
- VIRTUAL_CLUSTER = default
83
- INGEST_MODE = LIST_PURGE
84
- AS
85
- COPY INTO ods.events
86
- FROM VOLUME my_oss_volume
87
- USING PARQUET;
88
-
89
- -- EVENT_NOTIFICATION 模式(仅阿里云 OSS + AWS S3)
90
- CREATE OR REPLACE PIPE oss_events_event_pipe
91
- VIRTUAL_CLUSTER = default
92
- INGEST_MODE = EVENT_NOTIFICATION
93
- ALICLOUD_MNS_QUEUE = 'my-mns-queue-name'
94
- AS
95
- COPY INTO ods.events
96
- FROM VOLUME my_oss_event_volume
97
- USING PARQUET;
98
- ```
99
-
100
- ## 启停 Pipe
101
-
102
- ```sql
103
- -- 暂停 Pipe
104
- ALTER PIPE <pipe_name> SET PIPE_EXECUTION_PAUSED = true;
105
-
106
- -- 恢复 Pipe
107
- ALTER PIPE <pipe_name> SET PIPE_EXECUTION_PAUSED = false;
108
- ```
109
-
110
- ## DROP PIPE
111
-
112
- ```sql
113
- DROP PIPE [ IF EXISTS ] <pipe_name>;
114
- ```
115
-
116
- ## SHOW PIPE
117
-
118
- ```sql
119
- -- 列出当前 schema 下所有 Pipe
120
- SHOW PIPES;
121
-
122
- -- 按名称过滤
123
- SHOW PIPES LIKE 'kafka%';
124
-
125
- -- 查看 Pipe 详情
126
- DESC PIPE <pipe_name>;
127
- ```
128
-
129
- ## 验证 Kafka 连接(创建 Pipe 前)
130
-
131
- ```sql
132
- -- 先用 READ_KAFKA 函数验证连接和数据格式
133
- SELECT *
134
- FROM TABLE(
135
- READ_KAFKA(
136
- KAFKA_BROKER => 'kafka.example.com:9092',
137
- KAFKA_TOPIC => 'orders',
138
- KAFKA_GROUP_ID => 'test_group',
139
- KAFKA_OFFSET => 'earliest',
140
- KAFKA_DATA_FORMAT => 'json'
141
- )
142
- )
143
- LIMIT 10;
144
- ```
145
-
146
- ## 注意事项
147
-
148
- - Pipe 创建后默认自动启动,无需手动 RESUME
149
- - Kafka Pipe 使用 consumer group 管理 offset,重建 Pipe 时注意 group_id 和 offset 设置
150
- - 对象存储 Pipe 通过文件列表或事件通知(EVENT_NOTIFICATION)检测新文件,避免重复导入
151
- - Pipe 不支持修改 AS 子句,需要 `CREATE OR REPLACE`
152
-
153
- ## 参考文档
154
-
155
- - [PIPE 导入语法](https://www.yunqi.tech/documents/pipe-syntax)
156
- - [Pipe 简介](https://www.yunqi.tech/documents/pipe-summary)
157
- - [借助 read_kafka 函数持续导入](https://www.yunqi.tech/documents/pipe-kafka)
158
- - [借助 Kafka 外表 Table Stream 持续导入](https://www.yunqi.tech/documents/pipe-kafka-table-stream)
159
- - [最佳实践:使用 Pipe 高效接入 Kafka 数据](https://www.yunqi.tech/documents/pipe-kafka-bestpractice-1)
160
- - [使用 Pipe 持续导入对象存储数据](https://www.yunqi.tech/documents/pipe-storage-object)
@@ -1,123 +0,0 @@
1
- # Table Stream(表流)SQL 参考
2
-
3
- > **⚠️ ClickZetta 特有语法**
4
- > - 创建语法是 `CREATE TABLE STREAM`,参数放在 `WITH PROPERTIES (...)` 里
5
- > - 元数据字段是 `__change_type`(双下划线),值:`INSERT` / `UPDATE_BEFORE` / `UPDATE_AFTER` / `DELETE`
6
- > - UPDATE 产生两条记录:`UPDATE_BEFORE`(更新前)和 `UPDATE_AFTER`(更新后)
7
- > - 通常只需要 `UPDATE_AFTER` 和 `INSERT`,忽略 `UPDATE_BEFORE`
8
-
9
- Table Stream 捕获源表的变更数据(INSERT / UPDATE / DELETE),是构建 CDC 管道的核心对象。通常与 Dynamic Table 或 SQL 任务配合消费变更数据。
10
-
11
- ## CREATE TABLE STREAM
12
-
13
- ```sql
14
- CREATE [ OR REPLACE ] TABLE STREAM [ IF NOT EXISTS ] <stream_name>
15
- ON TABLE <source_name>
16
- [ TIMESTAMP AS OF <timestamp_expr> ]
17
- [ COMMENT '<comment>' ]
18
- WITH PROPERTIES (
19
- 'TABLE_STREAM_MODE' = 'STANDARD | APPEND_ONLY',
20
- 'SHOW_INITIAL_ROWS' = 'TRUE | FALSE'
21
- );
22
- ```
23
-
24
- **关键参数:**
25
- - `TABLE_STREAM_MODE = STANDARD`(默认):捕获 INSERT、UPDATE、DELETE 所有变更,每行附带 `__change_type` 字段(`INSERT` / `UPDATE_BEFORE` / `UPDATE_AFTER` / `DELETE`)
26
- - `TABLE_STREAM_MODE = APPEND_ONLY`:只捕获 INSERT,性能更好,适合仅追加写入的源表
27
- - `SHOW_INITIAL_ROWS = TRUE`:首次消费返回建 Stream 时表中已有行;`FALSE`(默认)仅返回建 Stream 后的新变更
28
- - `TIMESTAMP AS OF`:指定 Stream 从哪个时间点开始捕获变更
29
-
30
- **示例:**
31
- ```sql
32
- -- 在普通表上创建标准流(捕获所有变更,需先开启 change_tracking)
33
- ALTER TABLE ods.orders SET PROPERTIES ('change_tracking' = 'true');
34
-
35
- CREATE TABLE STREAM orders_stream
36
- ON TABLE ods.orders
37
- WITH PROPERTIES ('TABLE_STREAM_MODE' = 'STANDARD');
38
-
39
- -- 仅追加流
40
- CREATE TABLE STREAM events_stream
41
- ON TABLE dw.events
42
- COMMENT '事件流,仅追加'
43
- WITH PROPERTIES ('TABLE_STREAM_MODE' = 'APPEND_ONLY');
44
-
45
- -- 从指定时间点开始捕获
46
- CREATE TABLE STREAM orders_stream_from_ts
47
- ON TABLE ods.orders
48
- TIMESTAMP AS OF '2024-01-01 00:00:00'
49
- WITH PROPERTIES ('TABLE_STREAM_MODE' = 'STANDARD', 'SHOW_INITIAL_ROWS' = 'TRUE');
50
- ```
51
-
52
- ## 消费 Table Stream
53
-
54
- Table Stream 是一次性消费的:**每次 SELECT 后,已读取的数据会被标记为已消费**,下次 SELECT 只返回新增变更。
55
-
56
- ```sql
57
- -- 查看当前未消费的变更数据
58
- SELECT * FROM orders_stream;
59
-
60
- -- 变更数据包含的系统字段
61
- -- __change_type: INSERT | UPDATE_BEFORE | UPDATE_AFTER | DELETE
62
- -- __commit_version: 变更版本号
63
- -- __commit_timestamp: 变更发生时间
64
-
65
- -- 典型用法:将变更数据 MERGE 到目标表(忽略 UPDATE_BEFORE)
66
- MERGE INTO dw.orders_dim AS target
67
- USING (
68
- SELECT * FROM orders_stream
69
- WHERE __change_type IN ('INSERT', 'UPDATE_AFTER', 'DELETE')
70
- ) AS src
71
- ON target.order_id = src.order_id
72
- WHEN MATCHED AND src.__change_type = 'UPDATE_AFTER' THEN UPDATE SET target.status = src.status, target.amount = src.amount
73
- WHEN MATCHED AND src.__change_type = 'DELETE' THEN DELETE
74
- WHEN NOT MATCHED AND src.__change_type = 'INSERT' THEN INSERT (order_id, status, amount) VALUES (src.order_id, src.status, src.amount);
75
-
76
- -- 配合 Dynamic Table 自动消费(推荐)
77
- CREATE OR REPLACE DYNAMIC TABLE dw.orders_processed
78
- REFRESH interval 1 MINUTE
79
- VCLUSTER default
80
- AS
81
- SELECT order_id, status, amount, __change_type, __commit_timestamp
82
- FROM orders_stream
83
- WHERE __change_type IN ('INSERT', 'UPDATE_AFTER');
84
- ```
85
-
86
- ## DROP TABLE STREAM
87
-
88
- ```sql
89
- DROP TABLE STREAM [ IF EXISTS ] <stream_name>;
90
- ```
91
-
92
- ## SHOW / DESC
93
-
94
- ```sql
95
- -- 列出当前 schema 下所有 Table Stream
96
- SHOW TABLE STREAMS;
97
-
98
- -- 列出指定 schema 下的 Table Stream
99
- SHOW TABLE STREAMS IN <schema_name>;
100
-
101
- -- 按名称过滤
102
- SHOW TABLE STREAMS LIKE 'orders%';
103
-
104
- -- 查看 Table Stream 详情(源表、模式、创建时间)
105
- DESC TABLE STREAM <stream_name>;
106
- ```
107
-
108
- ## 注意事项
109
-
110
- - Stream 数据**只能消费一次**,SELECT 后即标记为已读
111
- - 若长时间不消费,超出源表的 `data_retention_days` 后数据会丢失
112
- - `STANDARD` 模式下 UPDATE 会产生两条记录:`UPDATE_BEFORE`(更新前)和 `UPDATE_AFTER`(更新后)
113
- - 消费时通常过滤 `__change_type IN ('INSERT', 'UPDATE_AFTER', 'DELETE')`,忽略 `UPDATE_BEFORE`
114
-
115
- ## 参考文档
116
-
117
- - [CREATE TABLE STREAM](https://www.yunqi.tech/documents/create-table-stream)
118
- - [DESC TABLE STREAM](https://www.yunqi.tech/documents/desc-table-stream)
119
- - [SHOW TABLE STREAMS](https://www.yunqi.tech/documents/show-table-streams)
120
- - [DROP TABLE STREAM](https://www.yunqi.tech/documents/drop-table-stream)
121
- - [TABLE STREAM 简介](https://www.yunqi.tech/documents/tablestream_summary)
122
- - [Table Stream 变化数据捕获](https://www.yunqi.tech/documents/table_stream)
123
- - [Table Stream 最佳实践](https://www.yunqi.tech/documents/lakehouse-table-stream-best-practices)