ops-tune-workbench-plugin 0.0.0-stage → 0.0.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE.md +14 -0
- package/README.md +39 -2
- package/cordis.patch.yml +20 -0
- package/lib/client.js +4253 -0
- package/lib/host.js +2518 -0
- package/package.json +53 -4
- package/skills/tool-ops-tune/INSTALL.md +52 -0
- package/skills/tool-ops-tune/README.md +167 -0
- package/skills/tool-ops-tune/SKILL.md +93 -0
- package/skills/tool-ops-tune/agent/AGENTS.md +195 -0
- package/skills/tool-ops-tune/agent/agents/perf-implementer.md +68 -0
- package/skills/tool-ops-tune/agent/agents/perf-reviewer.md +86 -0
- package/skills/tool-ops-tune/agent/workflows/task-prompts.md +362 -0
- package/skills/tool-ops-tune/agent/workflows/troubleshooting.md +43 -0
- package/skills/tool-ops-tune/docs/figures/opstune-architecture.png +0 -0
- package/skills/tool-ops-tune/docs/figures/opstune-architecture.svg +253 -0
- package/skills/tool-ops-tune/docs/ops-tune-design.md +404 -0
- package/skills/tool-ops-tune/docs//347/224/250/344/276/213/346/236/204/345/273/272.md +714 -0
- package/skills/tool-ops-tune/docs//347/237/245/350/257/206/345/272/223/350/247/204/350/214/203.md +103 -0
- package/skills/tool-ops-tune/evals/evals.json +242 -0
- package/skills/tool-ops-tune/ops-collect/SKILL.md +146 -0
- package/skills/tool-ops-tune/ops-collect/references/tool-routing.md +86 -0
- package/skills/tool-ops-tune/ops-collect/scripts/collect_board.sh +444 -0
- package/skills/tool-ops-tune/ops-collect/scripts/collect_cases.py +368 -0
- package/skills/tool-ops-tune/ops-collect/scripts/collect_sim.sh +134 -0
- package/skills/tool-ops-tune/ops-collect/scripts/csv_to_db.py +314 -0
- package/skills/tool-ops-tune/ops-collect/scripts/db_bench.py +545 -0
- package/skills/tool-ops-tune/ops-collect/scripts/facts_bridge.py +1063 -0
- package/skills/tool-ops-tune/ops-collect/scripts/isolation_test.py +454 -0
- package/skills/tool-ops-tune/ops-collect/scripts/jsonl_to_cases.py +163 -0
- package/skills/tool-ops-tune/ops-family/SKILL.md +76 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/advanced_tiling.md +92 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/broadcast_design.md +160 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/broadcast_add_kernel.cpp +126 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/broadcast_add_tiling.cpp +144 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/broadcast_tiling_common.h +401 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/onedim_add_kernel.cpp +107 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/onedim_add_tiling.cpp +64 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/datacopypad_design.md +46 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/guide.md +14 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/nddma_design.md +53 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/onedim_design.md +114 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/ub_broadcast_design.md +67 -0
- package/skills/tool-ops-tune/ops-family/references/common/datacopy_optimization_design.md +351 -0
- package/skills/tool-ops-tune/ops-family/references/common/index.md +9 -0
- package/skills/tool-ops-tune/ops-family/references/common/tail_block_design.md +268 -0
- package/skills/tool-ops-tune/ops-family/references/common/timing_harness/index.md +38 -0
- package/skills/tool-ops-tune/ops-family/references/common/timing_harness/timing_harness_aclrt_event.cpp +199 -0
- package/skills/tool-ops-tune/ops-family/references/common/timing_harness/timing_harness_torch_event.py +130 -0
- package/skills/tool-ops-tune/ops-family/references/common/ub_resident_design.md +342 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/guide.md +69 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/index.md +4 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_base.template +81 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_big_dim.template +217 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_cut_one_axis.template +336 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_cut_two_axis.template +781 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_n_last.template +340 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_small_shape.template +321 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_tensor_move.template +102 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_transdata_5hd.template +480 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_transdata_5hd_021axis.template +533 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_with_gather.template +688 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/transpose_tiling_data.template +192 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_big_dim.md +220 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_cut_one_axis.md +175 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_cut_two_axis.md +177 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_fusion_design.md +398 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_gather.md +191 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_n_last.md +276 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_small_shape.md +315 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_tensor_move.md +203 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_tiling_data.md +176 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_vconv_021.md +206 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_vconv_5hd.md +179 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/double_buffer_design.md +288 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/guide.md +46 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/index.md +10 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/CMakeLists.txt +131 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/README.md +55 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_extension/add_custom_torch.cpp +80 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_extension/ops.h +23 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_extension/register.cpp +30 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_host/add_custom.asc +149 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_host/data_utils.h +97 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_kernel/add_custom_kernel.asc +136 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_kernel/add_custom_tiling.h +31 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/run.sh +86 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/scripts/gen_data.py +70 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/scripts/golden.py +34 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/scripts/test_torch.py +99 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/scripts/verify_result.py +56 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/usage_guide.md +61 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/vector_efficiency_design.md +203 -0
- package/skills/tool-ops-tune/ops-family/references/fa/fa_mixed_kernel_design.md +174 -0
- package/skills/tool-ops-tune/ops-family/references/fa/fa_roofline_design.md +141 -0
- package/skills/tool-ops-tune/ops-family/references/fa/guide.md +87 -0
- package/skills/tool-ops-tune/ops-family/references/knowledge_gaps.json +56 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/best-practices-guide.md +14 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/fullload_design.md +344 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/guide.md +52 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/mte2_preload_design.md +320 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/pingpong_design.md +254 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/scale_coalescing_design.md +282 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/streamk_design.md +374 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/swat_design.md +423 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/templates/matmul_basic_blaze.template +165 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/templates/usage_guide.md +29 -0
- package/skills/tool-ops-tune/ops-family/references/mc2/best-practices-guide.md +43 -0
- package/skills/tool-ops-tune/ops-family/references/mc2/guide.md +42 -0
- package/skills/tool-ops-tune/ops-family/references/mc2/local_matmul_design.md +130 -0
- package/skills/tool-ops-tune/ops-family/references/mc2/pipeline_balancing_design.md +502 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/guide.md +178 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/kernel_utils.template +75 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ar_full_load.template +429 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ar_recompute.template +485 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ar_small_r.template +348 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ara_full_load.template +574 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ara_online.template +324 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ara_recompute.template +436 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_base.template +950 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/fused_attention_online_softmax_design.md +428 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ar_full_load.md +137 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ar_recompute.md +135 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ar_small_r.md +122 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ara_full_load.md +144 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ara_online.md +185 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ara_recompute.md +163 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_tiling.template +460 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_tiling_data.template +278 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/state_resident_design.md +337 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/usage_guide.md +253 -0
- package/skills/tool-ops-tune/ops-family/references/scalar/coding_principles.md +476 -0
- package/skills/tool-ops-tune/ops-family/references/scalar/conv_scalar.md +163 -0
- package/skills/tool-ops-tune/ops-family/references/scalar/guide.md +30 -0
- package/skills/tool-ops-tune/ops-family/references/simt/guide.md +91 -0
- package/skills/tool-ops-tune/ops-family/references/simt/optimization-guide.md +110 -0
- package/skills/tool-ops-tune/ops-family/references/sort/guide.md +83 -0
- package/skills/tool-ops-tune/ops-family/references/sort/radix_sort.md +354 -0
- package/skills/tool-ops-tune/ops-family/scripts/family_asset_audit.py +68 -0
- package/skills/tool-ops-tune/ops-family/scripts/tiling_candidates.py +419 -0
- package/skills/tool-ops-tune/ops-mechanism/SKILL.md +23 -0
- package/skills/tool-ops-tune/ops-mechanism/cases/ascendc/case_groupedmatmul_pipeline.md +23 -0
- package/skills/tool-ops-tune/ops-mechanism/cases/ascendc/case_triton_atomic_revert.md +19 -0
- package/skills/tool-ops-tune/ops-mechanism/cases/mc2/case_mc2_fusion.md +19 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/api-usage.md +70 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/ascendc-regbase-vf.md +45 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/ascendc-tiling.md +51 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/data-copy.md +67 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/index.md +45 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/mc2-balancing.md +77 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/mc2.md +103 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/memory-hierarchy.md +40 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/pipeline.md +95 -0
- package/skills/tool-ops-tune/ops-mechanism/scripts/case_backfill.py +170 -0
- package/skills/tool-ops-tune/ops-mechanism/scripts/log_gap.py +145 -0
- package/skills/tool-ops-tune/ops-mechanism/scripts/mc2_search.py +367 -0
- package/skills/tool-ops-tune/ops-report/SKILL.md +77 -0
- package/skills/tool-ops-tune/ops-verdict/evals/run_evals.py +837 -0
- package/skills/tool-ops-tune/ops-verdict/evals/smoke/install-hooks.sh +65 -0
- package/skills/tool-ops-tune/ops-verdict/evals/smoke/run_smoke.sh +405 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/core_tail_a5.csv +25 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/few_cores_a5.csv +5 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/mte2_bound_a2.csv +17 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/multi_kernel_mix_a5.csv +7 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/no_bound_a5.csv +9 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/serial_no_overlap_a2.csv +9 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/vec_bound_a5.csv +9 -0
- package/skills/tool-ops-tune/ops-verdict/references/io-contracts.md +94 -0
- package/skills/tool-ops-tune/ops-verdict/references/rule-schema.md +130 -0
- package/skills/tool-ops-tune/ops-verdict/references/scope-schema.md +61 -0
- package/skills/tool-ops-tune/ops-verdict/references/soc-spec.yaml +140 -0
- package/skills/tool-ops-tune/ops-verdict/rules/L0-instruction.yaml +67 -0
- package/skills/tool-ops-tune/ops-verdict/rules/L1-single-core.yaml +316 -0
- package/skills/tool-ops-tune/ops-verdict/rules/L2-inter-core.yaml +173 -0
- package/skills/tool-ops-tune/ops-verdict/rules/L3-inter-card.yaml +156 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/batch_pipeline.py +444 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/calibrate.py +249 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/delta_compare.py +244 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/render_html.py +274 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/snapshot.py +200 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/trace_bubble.py +331 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/verdict.py +1072 -0
- package/skills/tool-ops-tune/scripts/diff_audit.py +281 -0
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
|
3
|
+
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
|
4
|
+
* CANN Open Software License Agreement Version 2.0 (the "License").
|
|
5
|
+
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
|
6
|
+
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
|
7
|
+
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
|
8
|
+
* See LICENSE in the root of the software repository for the full text of the License.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/*!
|
|
12
|
+
* \file transpose_big_dim.h
|
|
13
|
+
* \brief transpose_big_dim
|
|
14
|
+
*/
|
|
15
|
+
#ifndef TRANSPOSE_BIG_DIM_H
|
|
16
|
+
#define TRANSPOSE_BIG_DIM_H
|
|
17
|
+
|
|
18
|
+
#include "transpose_base.h"
|
|
19
|
+
|
|
20
|
+
namespace Transpose {
|
|
21
|
+
using namespace AscendC;
|
|
22
|
+
|
|
23
|
+
template <typename T>
|
|
24
|
+
class TransposeBigDim : public TransposeBase<T> {
|
|
25
|
+
public:
|
|
26
|
+
__aicore__ inline TransposeBigDim(){};
|
|
27
|
+
__aicore__ inline void Init(GM_ADDR x, GM_ADDR y, const TransposeOpTilingData* tilingData, TPipe* pipe);
|
|
28
|
+
__aicore__ inline void Process();
|
|
29
|
+
|
|
30
|
+
private:
|
|
31
|
+
__aicore__ inline void DecimalToMixed(int64_t num, int64_t bases[], int64_t mixedBase[]);
|
|
32
|
+
__aicore__ inline void SetLoopInfo(MultiCopyLoopInfo<NDDMA_MAX_DIM_NUM>& loopInfo);
|
|
33
|
+
__aicore__ inline void CopyIn(int64_t loopIdx, MultiCopyParams<T, NDDMA_MAX_DIM_NUM>& params);
|
|
34
|
+
__aicore__ inline void CopyOut(int64_t loopIdx, DataCopyExtParams& copyOutParams);
|
|
35
|
+
__aicore__ inline void ProcessPerCore(MultiCopyParams<T, NDDMA_MAX_DIM_NUM>& paramsMain,
|
|
36
|
+
MultiCopyParams<T, NDDMA_MAX_DIM_NUM>& paramsTail,
|
|
37
|
+
DataCopyExtParams& copyOutParamsMain, DataCopyExtParams& copyOutParamsTail);
|
|
38
|
+
|
|
39
|
+
private:
|
|
40
|
+
int64_t blockIdx_;
|
|
41
|
+
|
|
42
|
+
// buffer
|
|
43
|
+
TQueBind<QuePosition::VECIN, QuePosition::VECOUT, 1> vecQue_;
|
|
44
|
+
GlobalTensor<T> inputGM_;
|
|
45
|
+
GlobalTensor<T> outputGM_;
|
|
46
|
+
|
|
47
|
+
// tiling params
|
|
48
|
+
const TransposeOpTilingData* tiling_;
|
|
49
|
+
|
|
50
|
+
// core params
|
|
51
|
+
int64_t blkProcessNum_ = 0;
|
|
52
|
+
int64_t blkProcessIdxStart_ = 0;
|
|
53
|
+
int64_t blkProcessIdxEnd_ = 0;
|
|
54
|
+
|
|
55
|
+
// addressOffset params
|
|
56
|
+
int64_t dstAddressOffsetMixedBase_[TRANSPOSE_MAX_AXIS_NUM] = {1, 1, 1, 1, 1, 1, 1, 1};
|
|
57
|
+
int64_t dstAddressOffsetMixedBaseRes_[TRANSPOSE_MAX_AXIS_NUM] = {0, 0, 0, 0, 0, 0, 0, 0};
|
|
58
|
+
int64_t dstLoopStride_[TRANSPOSE_MAX_AXIS_NUM] = {1, 1, 1, 1, 1, 1, 1, 1};
|
|
59
|
+
int64_t srcLoopStride_[TRANSPOSE_MAX_AXIS_NUM] = {1, 1, 1, 1, 1, 1, 1, 1};
|
|
60
|
+
|
|
61
|
+
int64_t cutIdx_ = 0;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
template <typename T>
|
|
65
|
+
__aicore__ inline void TransposeBigDim<T>::Init(GM_ADDR x, GM_ADDR y, const TransposeOpTilingData* tilingData,
|
|
66
|
+
TPipe* pipe)
|
|
67
|
+
{
|
|
68
|
+
blockIdx_ = GetBlockIdx();
|
|
69
|
+
tiling_ = tilingData;
|
|
70
|
+
inputGM_.SetGlobalBuffer((__gm__ T*)x);
|
|
71
|
+
outputGM_.SetGlobalBuffer((__gm__ T*)y);
|
|
72
|
+
pipe->InitBuffer(vecQue_, 1, tiling_->ubSize);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
template <typename T>
|
|
76
|
+
__aicore__ inline void TransposeBigDim<T>::DecimalToMixed(int64_t num, int64_t bases[], int64_t mixedBase[])
|
|
77
|
+
{
|
|
78
|
+
if (num == 0) {
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
for (int64_t i = 0; i < tiling_->permSize; i++) {
|
|
82
|
+
mixedBase[i] = num % bases[i];
|
|
83
|
+
num /= bases[i];
|
|
84
|
+
if (num == 0) {
|
|
85
|
+
break;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
template <typename T>
|
|
91
|
+
__aicore__ inline void TransposeBigDim<T>::SetLoopInfo(MultiCopyLoopInfo<NDDMA_MAX_DIM_NUM>& loopInfo)
|
|
92
|
+
{
|
|
93
|
+
int64_t nddmaFlag[NDDMA_MAX_DIM_NUM] = {0};
|
|
94
|
+
for (int64_t i = 0; i < NDDMA_MAX_DIM_NUM; i++) {
|
|
95
|
+
if (tiling_->nddmaIdx[i] == tiling_->perm[tiling_->outCutIndex]) {
|
|
96
|
+
nddmaFlag[i] = -1;
|
|
97
|
+
}
|
|
98
|
+
for (int64_t j = tiling_->outCutIndex + 1; j < tiling_->permSize; j++) {
|
|
99
|
+
if (tiling_->perm[j] == tiling_->nddmaIdx[i]) {
|
|
100
|
+
nddmaFlag[i] = 1;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
for (int64_t i = NDDMA_MAX_DIM_NUM - 1; i >= 0; i--) {
|
|
106
|
+
if (nddmaFlag[NDDMA_MAX_DIM_NUM - 1 - i] == 1) {
|
|
107
|
+
loopInfo.loopSize[i] = tiling_->inputShape[tiling_->nddmaIdx[NDDMA_MAX_DIM_NUM - 1 - i]];
|
|
108
|
+
loopInfo.loopSrcStride[i] = tiling_->baseInShape[tiling_->nddmaIdx[NDDMA_MAX_DIM_NUM - 1 - i]];
|
|
109
|
+
loopInfo.loopDstStride[i] = tiling_->baseNddmaShape[NDDMA_MAX_DIM_NUM - 1 - i];
|
|
110
|
+
} else if (nddmaFlag[NDDMA_MAX_DIM_NUM - 1 - i] == -1) {
|
|
111
|
+
cutIdx_ = i;
|
|
112
|
+
loopInfo.loopSize[i] = tiling_->outUbFactor; // not tail
|
|
113
|
+
loopInfo.loopSrcStride[i] = tiling_->baseInShape[tiling_->nddmaIdx[NDDMA_MAX_DIM_NUM - 1 - i]];
|
|
114
|
+
loopInfo.loopDstStride[i] = tiling_->baseNddmaShape[NDDMA_MAX_DIM_NUM - 1 - i];
|
|
115
|
+
} else {
|
|
116
|
+
loopInfo.loopSize[i] = 1;
|
|
117
|
+
loopInfo.loopSrcStride[i] = 1;
|
|
118
|
+
loopInfo.loopDstStride[i] = tiling_->baseNddmaShape[NDDMA_MAX_DIM_NUM - 1 - i];
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
loopInfo.loopLpSize[i] = 0;
|
|
122
|
+
loopInfo.loopRpSize[i] = 0;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
template <typename T>
|
|
127
|
+
__aicore__ inline void TransposeBigDim<T>::Process()
|
|
128
|
+
{
|
|
129
|
+
if (!ParseMultiCoreRange(blockIdx_, tiling_->realCoreNum, tiling_->blkFactor, tiling_->blkTailFactor,
|
|
130
|
+
blkProcessNum_, blkProcessIdxStart_, blkProcessIdxEnd_)) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
MultiCopyLoopInfo<NDDMA_MAX_DIM_NUM> loopInfo;
|
|
135
|
+
SetLoopInfo(loopInfo);
|
|
136
|
+
|
|
137
|
+
// calculate blockCount
|
|
138
|
+
DataCopyExtParams copyOutParamsMain{1, 0, 0, 0, 0};
|
|
139
|
+
copyOutParamsMain.blockLen = tiling_->totalNddmaNum * sizeof(T);
|
|
140
|
+
|
|
141
|
+
DataCopyExtParams copyOutParamsTail{1, 0, 0, 0, 0};
|
|
142
|
+
copyOutParamsTail.blockLen = tiling_->totalNddmaNum / tiling_->outUbFactor * tiling_->outTailFactor * sizeof(T);
|
|
143
|
+
|
|
144
|
+
T constValue = 0;
|
|
145
|
+
MultiCopyParams<T, NDDMA_MAX_DIM_NUM> paramsMain = {loopInfo, constValue};
|
|
146
|
+
if (tiling_->outTailFactor != 0) {
|
|
147
|
+
loopInfo.loopSize[cutIdx_] = tiling_->outTailFactor;
|
|
148
|
+
loopInfo.loopDstStride[cutIdx_] = tiling_->baseNddmaShape[NDDMA_MAX_DIM_NUM - 1 - cutIdx_];
|
|
149
|
+
}
|
|
150
|
+
MultiCopyParams<T, NDDMA_MAX_DIM_NUM> paramsTail = {loopInfo, constValue};
|
|
151
|
+
ProcessPerCore(paramsMain, paramsTail, copyOutParamsMain, copyOutParamsTail);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
template <typename T>
|
|
155
|
+
__aicore__ inline void TransposeBigDim<T>::CopyIn(int64_t loopIdx, MultiCopyParams<T, NDDMA_MAX_DIM_NUM>& params)
|
|
156
|
+
{
|
|
157
|
+
int64_t srcAddressOffset = 0;
|
|
158
|
+
DecimalToMixed(loopIdx, dstAddressOffsetMixedBase_, dstAddressOffsetMixedBaseRes_);
|
|
159
|
+
for (int64_t i = 0; i < tiling_->permSize; i++) {
|
|
160
|
+
srcAddressOffset += dstAddressOffsetMixedBaseRes_[i] *
|
|
161
|
+
srcLoopStride_[tiling_->permSize - 1 - tiling_->perm[tiling_->permSize - 1 - i]];
|
|
162
|
+
}
|
|
163
|
+
LocalTensor<T> bindLocalIn = vecQue_.AllocTensor<T>();
|
|
164
|
+
DataCopy<T, NDDMA_MAX_DIM_NUM, config>(bindLocalIn, inputGM_[srcAddressOffset], params);
|
|
165
|
+
vecQue_.EnQue(bindLocalIn);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
template <typename T>
|
|
169
|
+
__aicore__ inline void TransposeBigDim<T>::CopyOut(int64_t loopIdx, DataCopyExtParams& copyOutParams)
|
|
170
|
+
{
|
|
171
|
+
int64_t dstAddressOffset = 0;
|
|
172
|
+
for (int64_t i = 0; i < tiling_->permSize; i++) {
|
|
173
|
+
dstAddressOffset += dstAddressOffsetMixedBaseRes_[i] * dstLoopStride_[i];
|
|
174
|
+
}
|
|
175
|
+
LocalTensor<T> bindLocalOut = vecQue_.DeQue<T>();
|
|
176
|
+
DataCopyPad(outputGM_[dstAddressOffset], bindLocalOut, copyOutParams);
|
|
177
|
+
vecQue_.FreeTensor(bindLocalOut);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
template <typename T>
|
|
181
|
+
__aicore__ inline void TransposeBigDim<T>::ProcessPerCore(MultiCopyParams<T, NDDMA_MAX_DIM_NUM>& paramsMain,
|
|
182
|
+
MultiCopyParams<T, NDDMA_MAX_DIM_NUM>& paramsTail,
|
|
183
|
+
DataCopyExtParams& copyOutParamsMain,
|
|
184
|
+
DataCopyExtParams& copyOutParamsTail)
|
|
185
|
+
{
|
|
186
|
+
int64_t outCutLoopSize = Ops::Base::CeilDiv(tiling_->outputShape[tiling_->outCutIndex], tiling_->outUbFactor);
|
|
187
|
+
const int64_t permSize = tiling_->permSize;
|
|
188
|
+
for (int64_t i = 0; i < permSize; i++) {
|
|
189
|
+
if (permSize - 1 - i == tiling_->outCutIndex) {
|
|
190
|
+
dstAddressOffsetMixedBase_[i] = outCutLoopSize;
|
|
191
|
+
}
|
|
192
|
+
if (permSize - 1 - i < tiling_->outCutIndex) {
|
|
193
|
+
dstAddressOffsetMixedBase_[i] = tiling_->outputShape[permSize - 1 - i];
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (i >= 1) {
|
|
197
|
+
dstLoopStride_[i] = dstLoopStride_[i - 1] * tiling_->outputShape[permSize - i];
|
|
198
|
+
srcLoopStride_[i] = srcLoopStride_[i - 1] * tiling_->inputShape[permSize - i];
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
dstLoopStride_[permSize - 1 - tiling_->outCutIndex] *= tiling_->outUbFactor;
|
|
202
|
+
srcLoopStride_[permSize - 1 - tiling_->perm[tiling_->outCutIndex]] *= tiling_->outUbFactor;
|
|
203
|
+
|
|
204
|
+
for (int64_t loopIdx = blkProcessIdxStart_; loopIdx < blkProcessIdxEnd_; loopIdx++) {
|
|
205
|
+
if (tiling_->outTailFactor != 0 && (loopIdx + 1) % outCutLoopSize == 0) { // tail
|
|
206
|
+
CopyIn(loopIdx, paramsTail);
|
|
207
|
+
CopyOut(loopIdx, copyOutParamsTail);
|
|
208
|
+
} else { // main
|
|
209
|
+
CopyIn(loopIdx, paramsMain);
|
|
210
|
+
CopyOut(loopIdx, copyOutParamsMain);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
} // namespace Transpose
|
|
216
|
+
|
|
217
|
+
#endif // TRANSPOSE_BIG_DIM_H
|
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
|
3
|
+
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
|
4
|
+
* CANN Open Software License Agreement Version 2.0 (the "License").
|
|
5
|
+
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
|
6
|
+
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
|
7
|
+
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
|
8
|
+
* See LICENSE in the root of the software repository for the full text of the License.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/*!
|
|
12
|
+
* \file transpose_cut_one_axis.h
|
|
13
|
+
* \brief transpose_cut_one_axis
|
|
14
|
+
*/
|
|
15
|
+
#ifndef TRANSPOSE_CUT_ONE_AXIS_H
|
|
16
|
+
#define TRANSPOSE_CUT_ONE_AXIS_H
|
|
17
|
+
|
|
18
|
+
#include "transpose_base.h"
|
|
19
|
+
|
|
20
|
+
namespace Transpose {
|
|
21
|
+
using namespace AscendC;
|
|
22
|
+
|
|
23
|
+
template <typename T>
|
|
24
|
+
class TransposeCutOneAxis : public TransposeBase<T> {
|
|
25
|
+
public:
|
|
26
|
+
__aicore__ inline TransposeCutOneAxis(){};
|
|
27
|
+
__aicore__ inline void Init(GM_ADDR x, GM_ADDR y, const TransposeOpTilingData* tilingData, TPipe* pipe);
|
|
28
|
+
__aicore__ inline void Process();
|
|
29
|
+
|
|
30
|
+
private:
|
|
31
|
+
__aicore__ inline void ParseTilingData();
|
|
32
|
+
__aicore__ inline MultiCopyLoopInfo<NDDMA_MAX_DIM_NUM> SetupLoopInfo(const int64_t inUbSrcShape[],
|
|
33
|
+
const int64_t inUbDstShape[]);
|
|
34
|
+
__aicore__ inline void ProcessMain(int64_t loopidxEnd);
|
|
35
|
+
__aicore__ inline void ProcessTail();
|
|
36
|
+
__aicore__ inline void GetLoopParams(int64_t n);
|
|
37
|
+
__aicore__ inline void DecimalToMixed(int64_t num, int64_t bases[], int64_t mixedBase[]);
|
|
38
|
+
__aicore__ inline void GetLoopAndStride();
|
|
39
|
+
__aicore__ inline void CopyIn(int64_t loopIdx, MultiCopyParams<T, NDDMA_MAX_DIM_NUM>& params);
|
|
40
|
+
__aicore__ inline void CopyOut(int64_t loopIdx, int64_t loopSize[], int64_t loopSrcStride[],
|
|
41
|
+
int64_t loopDstStride[]);
|
|
42
|
+
__aicore__ inline void ProcessPerCore();
|
|
43
|
+
|
|
44
|
+
private:
|
|
45
|
+
int64_t blockIdx_;
|
|
46
|
+
|
|
47
|
+
// buffer
|
|
48
|
+
TQueBind<QuePosition::VECIN, QuePosition::VECOUT, 1> vecQue_;
|
|
49
|
+
GlobalTensor<T> inputGM_;
|
|
50
|
+
GlobalTensor<T> outputGM_;
|
|
51
|
+
|
|
52
|
+
// tiling params
|
|
53
|
+
const TransposeOpTilingData* tiling_;
|
|
54
|
+
|
|
55
|
+
// core params
|
|
56
|
+
int64_t blkProcessNum_ = 0;
|
|
57
|
+
int64_t blkProcessIdxStart_ = 0;
|
|
58
|
+
int64_t blkProcessIdxEnd_ = 0;
|
|
59
|
+
|
|
60
|
+
// blockCount and blockLen of copyOut
|
|
61
|
+
int64_t mainBlockLen_ = 1;
|
|
62
|
+
int64_t tailBlockLen_ = 1;
|
|
63
|
+
int64_t blockCount_ = 1;
|
|
64
|
+
int64_t dstStrideTotalLen_ = 1;
|
|
65
|
+
|
|
66
|
+
// expand output init
|
|
67
|
+
int64_t expandedOutputCutIndex_;
|
|
68
|
+
int64_t expandedInputCutIndex_;
|
|
69
|
+
|
|
70
|
+
// address offset
|
|
71
|
+
int64_t srcLoopSize_[NDDMA_MAX_DIM_NUM] = {0, 0, 0, 0, 0};
|
|
72
|
+
int64_t dstLoopSize_[NDDMA_MAX_DIM_NUM] = {0, 0, 0, 0, 0};
|
|
73
|
+
int64_t srcLoopStride_[NDDMA_MAX_DIM_NUM] = {0, 0, 0, 0, 0};
|
|
74
|
+
int64_t dstLoopStride_[NDDMA_MAX_DIM_NUM] = {0, 0, 0, 0, 0};
|
|
75
|
+
int64_t srcAddressOffsetMixedBase_[NDDMA_MAX_DIM_NUM] = {0, 0, 0, 0, 0};
|
|
76
|
+
int64_t dstAddressOffsetMixedBase_[NDDMA_MAX_DIM_NUM] = {0, 0, 0, 0, 0};
|
|
77
|
+
|
|
78
|
+
// dataCopy params
|
|
79
|
+
int64_t loopSizeMain_[NDDMA_MAX_DIM_NUM] = {1, 1, 1, 1, 1};
|
|
80
|
+
int64_t loopSrcStrideMain_[NDDMA_MAX_DIM_NUM] = {1, 0, 0, 0, 0};
|
|
81
|
+
int64_t loopDstStrideMain_[NDDMA_MAX_DIM_NUM] = {1, 0, 0, 0, 0};
|
|
82
|
+
int64_t loopSizeTail_[NDDMA_MAX_DIM_NUM] = {1, 1, 1, 1, 1};
|
|
83
|
+
int64_t loopSrcStrideTail_[NDDMA_MAX_DIM_NUM] = {1, 0, 0, 0, 0};
|
|
84
|
+
int64_t loopDstStrideTail_[NDDMA_MAX_DIM_NUM] = {1, 0, 0, 0, 0};
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
template <typename T>
|
|
88
|
+
__aicore__ inline void TransposeCutOneAxis<T>::Init(GM_ADDR x, GM_ADDR y, const TransposeOpTilingData* tilingData,
|
|
89
|
+
TPipe* pipe)
|
|
90
|
+
{
|
|
91
|
+
blockIdx_ = GetBlockIdx();
|
|
92
|
+
tiling_ = tilingData;
|
|
93
|
+
ParseTilingData();
|
|
94
|
+
inputGM_.SetGlobalBuffer((__gm__ T*)x);
|
|
95
|
+
outputGM_.SetGlobalBuffer((__gm__ T*)y);
|
|
96
|
+
pipe->InitBuffer(vecQue_, 1, tiling_->ubSize);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
template <typename T>
|
|
100
|
+
__aicore__ inline void TransposeCutOneAxis<T>::Process()
|
|
101
|
+
{
|
|
102
|
+
if (!ParseMultiCoreRange(blockIdx_, tiling_->realCoreNum, tiling_->blkFactor, tiling_->blkTailFactor,
|
|
103
|
+
blkProcessNum_, blkProcessIdxStart_, blkProcessIdxEnd_)) {
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
ProcessPerCore();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
template <typename T>
|
|
110
|
+
__aicore__ inline void TransposeCutOneAxis<T>::ParseTilingData()
|
|
111
|
+
{
|
|
112
|
+
expandedOutputCutIndex_ = tiling_->outCutIndex + NDDMA_MAX_DIM_NUM - tiling_->permSize;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
template <typename T>
|
|
116
|
+
__aicore__ inline MultiCopyLoopInfo<NDDMA_MAX_DIM_NUM> TransposeCutOneAxis<T>::SetupLoopInfo(
|
|
117
|
+
const int64_t inUbSrcShape[], const int64_t inUbDstShape[])
|
|
118
|
+
{
|
|
119
|
+
MultiCopyLoopInfo<NDDMA_MAX_DIM_NUM> loopInfo;
|
|
120
|
+
int64_t loopDstStrideTmp[NDDMA_MAX_DIM_NUM] = {0};
|
|
121
|
+
for (int64_t i = 0; i < NDDMA_MAX_DIM_NUM; i++) {
|
|
122
|
+
int64_t recumMultiSrc = 1;
|
|
123
|
+
int64_t recumMultiDst = 1;
|
|
124
|
+
loopInfo.loopSize[i] = inUbSrcShape[i];
|
|
125
|
+
for (int64_t j = i + 1; j < NDDMA_MAX_DIM_NUM; j++) {
|
|
126
|
+
recumMultiSrc *= tiling_->expandedInputShape[j];
|
|
127
|
+
recumMultiDst *= inUbDstShape[j];
|
|
128
|
+
}
|
|
129
|
+
loopInfo.loopSrcStride[i] = recumMultiSrc;
|
|
130
|
+
loopDstStrideTmp[i] = recumMultiDst;
|
|
131
|
+
loopInfo.loopLpSize[i] = 0;
|
|
132
|
+
loopInfo.loopRpSize[i] = 0;
|
|
133
|
+
}
|
|
134
|
+
loopInfo.loopSrcStride[NDDMA_MAX_DIM_NUM - 1] = 1;
|
|
135
|
+
loopDstStrideTmp[NDDMA_MAX_DIM_NUM - 1] = 1;
|
|
136
|
+
|
|
137
|
+
int64_t alignDstStride = 1;
|
|
138
|
+
if (expandedOutputCutIndex_ - 1 >= 0) {
|
|
139
|
+
alignDstStride = (loopDstStrideTmp[expandedOutputCutIndex_ - 1] * sizeof(T) + BLOCK_SIZE_BYTE - 1) /
|
|
140
|
+
BLOCK_SIZE_BYTE;
|
|
141
|
+
alignDstStride = alignDstStride * BLOCK_SIZE_BYTE / sizeof(T);
|
|
142
|
+
loopDstStrideTmp[expandedOutputCutIndex_ - 1] = alignDstStride;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
for (int64_t i = expandedOutputCutIndex_ - 2; i >= 0; i--) {
|
|
146
|
+
loopDstStrideTmp[i] = inUbDstShape[i + 1] * loopDstStrideTmp[i + 1];
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
for (int64_t i = 0; i < NDDMA_MAX_DIM_NUM; i++) {
|
|
150
|
+
loopInfo.loopDstStride[tiling_->expandedPerm[i]] = loopDstStrideTmp[i];
|
|
151
|
+
}
|
|
152
|
+
this->reverseArray(loopInfo.loopSize);
|
|
153
|
+
this->reverseArray(loopInfo.loopSrcStride);
|
|
154
|
+
this->reverseArray(loopInfo.loopDstStride);
|
|
155
|
+
|
|
156
|
+
return loopInfo;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
template <typename T>
|
|
160
|
+
__aicore__ inline void TransposeCutOneAxis<T>::DecimalToMixed(int64_t num, int64_t bases[], int64_t mixedBase[])
|
|
161
|
+
{
|
|
162
|
+
if (num == 0) {
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
for (int64_t i = NDDMA_MAX_DIM_NUM - 1; i >= 0; i--) {
|
|
166
|
+
mixedBase[i] = num % bases[i];
|
|
167
|
+
num /= bases[i];
|
|
168
|
+
if (num == 0) {
|
|
169
|
+
break;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
template <typename T>
|
|
175
|
+
__aicore__ inline void TransposeCutOneAxis<T>::GetLoopParams(int64_t n)
|
|
176
|
+
{
|
|
177
|
+
const int64_t reverseIdx = NDDMA_MAX_DIM_NUM - 1 - n;
|
|
178
|
+
// main and tail loopSize
|
|
179
|
+
loopSizeMain_[n] = tiling_->inUbMainDstShape[reverseIdx];
|
|
180
|
+
loopSizeTail_[n] = tiling_->inUbTailDstShape[reverseIdx];
|
|
181
|
+
// main loopSrcStride and loopDstStride
|
|
182
|
+
if (n > 0) {
|
|
183
|
+
if (n == NDDMA_MAX_DIM_NUM - expandedOutputCutIndex_) {
|
|
184
|
+
loopSrcStrideMain_[n] = Ops::Base::CeilAlign(
|
|
185
|
+
static_cast<int64_t>(tiling_->inUbMainDstShape[NDDMA_MAX_DIM_NUM - n] *
|
|
186
|
+
loopSrcStrideMain_[n - 1] * sizeof(T)),
|
|
187
|
+
BLOCK_SIZE_BYTE) /
|
|
188
|
+
sizeof(T);
|
|
189
|
+
} else {
|
|
190
|
+
loopSrcStrideMain_[n] = loopSrcStrideMain_[n - 1] * tiling_->inUbMainDstShape[NDDMA_MAX_DIM_NUM - n];
|
|
191
|
+
}
|
|
192
|
+
loopDstStrideMain_[n] = loopDstStrideMain_[n - 1] * tiling_->expandedOutputShape[NDDMA_MAX_DIM_NUM - n];
|
|
193
|
+
}
|
|
194
|
+
// tail loopSrcStride and loopDstStride
|
|
195
|
+
if (tiling_->outTailFactor != 0) {
|
|
196
|
+
if (n > 0) {
|
|
197
|
+
if (n == NDDMA_MAX_DIM_NUM - expandedOutputCutIndex_) {
|
|
198
|
+
loopSrcStrideTail_[n] = Ops::Base::CeilAlign(
|
|
199
|
+
static_cast<int64_t>(tiling_->inUbTailDstShape[NDDMA_MAX_DIM_NUM - n] *
|
|
200
|
+
loopSrcStrideTail_[n - 1] * sizeof(T)),
|
|
201
|
+
BLOCK_SIZE_BYTE) /
|
|
202
|
+
sizeof(T);
|
|
203
|
+
} else {
|
|
204
|
+
loopSrcStrideTail_[n] = loopSrcStrideTail_[n - 1] * tiling_->inUbTailDstShape[NDDMA_MAX_DIM_NUM - n];
|
|
205
|
+
}
|
|
206
|
+
loopDstStrideTail_[n] = loopDstStrideTail_[n - 1] * tiling_->expandedOutputShape[NDDMA_MAX_DIM_NUM - n];
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
template <typename T>
|
|
212
|
+
__aicore__ inline void TransposeCutOneAxis<T>::GetLoopAndStride()
|
|
213
|
+
{
|
|
214
|
+
// calculate the loopSize and stride of src and dst
|
|
215
|
+
expandedInputCutIndex_ = tiling_->expandedPerm[expandedOutputCutIndex_];
|
|
216
|
+
srcLoopStride_[NDDMA_MAX_DIM_NUM - 1] = 1;
|
|
217
|
+
dstLoopStride_[NDDMA_MAX_DIM_NUM - 1] = 1;
|
|
218
|
+
srcLoopStride_[NDDMA_MAX_DIM_NUM - 2] = tiling_->expandedInputShape[NDDMA_MAX_DIM_NUM - 1];
|
|
219
|
+
dstLoopStride_[NDDMA_MAX_DIM_NUM - 2] = tiling_->expandedOutputShape[NDDMA_MAX_DIM_NUM - 1];
|
|
220
|
+
for (int64_t i = NDDMA_MAX_DIM_NUM - 3; i >= 0; i--) {
|
|
221
|
+
srcLoopStride_[i] = srcLoopStride_[i + 1] * tiling_->expandedInputShape[i + 1];
|
|
222
|
+
dstLoopStride_[i] = dstLoopStride_[i + 1] * tiling_->expandedOutputShape[i + 1];
|
|
223
|
+
}
|
|
224
|
+
srcLoopStride_[expandedInputCutIndex_] *= tiling_->outUbFactor;
|
|
225
|
+
dstLoopStride_[expandedOutputCutIndex_] *= tiling_->outUbFactor;
|
|
226
|
+
|
|
227
|
+
for (int64_t i = 0; i < NDDMA_MAX_DIM_NUM; i++) {
|
|
228
|
+
srcLoopSize_[i] = Ops::Base::CeilDiv(tiling_->expandedInputShape[i], tiling_->inUbMainSrcShape[i]);
|
|
229
|
+
dstLoopSize_[i] = Ops::Base::CeilDiv(tiling_->expandedOutputShape[i], tiling_->inUbMainDstShape[i]);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
template <typename T>
|
|
234
|
+
__aicore__ inline void TransposeCutOneAxis<T>::CopyIn(int64_t loopIdx, MultiCopyParams<T, NDDMA_MAX_DIM_NUM>& params)
|
|
235
|
+
{
|
|
236
|
+
int64_t srcAddressOffset = 0;
|
|
237
|
+
DecimalToMixed(loopIdx, dstLoopSize_, dstAddressOffsetMixedBase_);
|
|
238
|
+
for (int64_t i = 0; i < NDDMA_MAX_DIM_NUM; i++) {
|
|
239
|
+
srcAddressOffsetMixedBase_[tiling_->expandedPerm[i]] = dstAddressOffsetMixedBase_[i];
|
|
240
|
+
srcAddressOffset += srcAddressOffsetMixedBase_[tiling_->expandedPerm[i]] *
|
|
241
|
+
srcLoopStride_[tiling_->expandedPerm[i]];
|
|
242
|
+
}
|
|
243
|
+
LocalTensor<T> bindLocalIn = vecQue_.AllocTensor<T>();
|
|
244
|
+
DataCopy<T, NDDMA_MAX_DIM_NUM, config>(bindLocalIn, inputGM_[srcAddressOffset], params);
|
|
245
|
+
vecQue_.EnQue(bindLocalIn);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
template <typename T>
|
|
249
|
+
__aicore__ inline void TransposeCutOneAxis<T>::CopyOut(int64_t loopIdx, int64_t loopSize[], int64_t loopSrcStride[],
|
|
250
|
+
int64_t loopDstStride[])
|
|
251
|
+
{
|
|
252
|
+
int64_t dstAddressOffset = 0;
|
|
253
|
+
for (int64_t i = 0; i < NDDMA_MAX_DIM_NUM; i++) {
|
|
254
|
+
dstAddressOffset += dstAddressOffsetMixedBase_[i] * dstLoopStride_[i];
|
|
255
|
+
}
|
|
256
|
+
// MTE3 params
|
|
257
|
+
DataCopyExtParams copyOutParams{1, 0, 0, 0, 0};
|
|
258
|
+
LoopModeParams loopParams;
|
|
259
|
+
|
|
260
|
+
copyOutParams.blockLen = sizeof(T);
|
|
261
|
+
int64_t endIndex = 0;
|
|
262
|
+
int64_t dstStride = 1;
|
|
263
|
+
for (int64_t i = NDDMA_MAX_DIM_NUM - 1; i >= 0; i--) {
|
|
264
|
+
copyOutParams.blockLen *= loopSize[NDDMA_MAX_DIM_NUM - 1 - i];
|
|
265
|
+
if (tiling_->expandedOutputShape[i] != tiling_->inUbMainDstShape[i]) {
|
|
266
|
+
endIndex = NDDMA_MAX_DIM_NUM - 1 - i;
|
|
267
|
+
break;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
if (endIndex + 1 < NDDMA_MAX_DIM_NUM) {
|
|
271
|
+
copyOutParams.blockCount = loopSize[endIndex + 1];
|
|
272
|
+
copyOutParams.dstStride = copyOutParams.blockCount == 1 ?
|
|
273
|
+
0 :
|
|
274
|
+
(loopDstStride[endIndex + 1] - copyOutParams.blockLen / sizeof(T)) * sizeof(T);
|
|
275
|
+
}
|
|
276
|
+
loopParams.loop1Size = 1;
|
|
277
|
+
if (endIndex + 2 < NDDMA_MAX_DIM_NUM) {
|
|
278
|
+
loopParams.loop1Size = loopSize[endIndex + 2];
|
|
279
|
+
loopParams.loop1SrcStride = loopSrcStride[endIndex + 2] * sizeof(T);
|
|
280
|
+
loopParams.loop1DstStride = loopDstStride[endIndex + 2] * sizeof(T);
|
|
281
|
+
}
|
|
282
|
+
loopParams.loop2Size = 1;
|
|
283
|
+
if (endIndex + 3 < NDDMA_MAX_DIM_NUM) {
|
|
284
|
+
loopParams.loop2Size = loopSize[endIndex + 3];
|
|
285
|
+
loopParams.loop2SrcStride = loopSrcStride[endIndex + 3] * sizeof(T);
|
|
286
|
+
loopParams.loop2DstStride = loopDstStride[endIndex + 3] * sizeof(T);
|
|
287
|
+
}
|
|
288
|
+
LocalTensor<T> bindLocalOut = vecQue_.DeQue<T>();
|
|
289
|
+
if (endIndex + 4 < NDDMA_MAX_DIM_NUM) {
|
|
290
|
+
for (int64_t loop4Idx = 0; loop4Idx < loopSize[4]; loop4Idx++) {
|
|
291
|
+
SetLoopModePara(loopParams, DataCopyMVType::UB_TO_OUT);
|
|
292
|
+
DataCopyPad(outputGM_[dstAddressOffset + loop4Idx * loopDstStride[4]],
|
|
293
|
+
bindLocalOut[loop4Idx * loopSrcStride[4]], copyOutParams);
|
|
294
|
+
ResetLoopModePara(DataCopyMVType::UB_TO_OUT);
|
|
295
|
+
}
|
|
296
|
+
} else {
|
|
297
|
+
SetLoopModePara(loopParams, DataCopyMVType::UB_TO_OUT);
|
|
298
|
+
DataCopyPad(outputGM_[dstAddressOffset], bindLocalOut, copyOutParams);
|
|
299
|
+
ResetLoopModePara(DataCopyMVType::UB_TO_OUT);
|
|
300
|
+
}
|
|
301
|
+
vecQue_.FreeTensor(bindLocalOut);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
template <typename T>
|
|
305
|
+
__aicore__ inline void TransposeCutOneAxis<T>::ProcessPerCore()
|
|
306
|
+
{
|
|
307
|
+
for (int64_t i = 0; i < tiling_->permSize; i++) {
|
|
308
|
+
GetLoopParams(i);
|
|
309
|
+
}
|
|
310
|
+
GetLoopAndStride();
|
|
311
|
+
|
|
312
|
+
// MTE2 params main
|
|
313
|
+
T constValue = 0;
|
|
314
|
+
MultiCopyLoopInfo<NDDMA_MAX_DIM_NUM> loopInfoMain = SetupLoopInfo(tiling_->inUbMainSrcShape,
|
|
315
|
+
tiling_->inUbMainDstShape);
|
|
316
|
+
MultiCopyParams<T, NDDMA_MAX_DIM_NUM> paramsMain = {loopInfoMain, constValue};
|
|
317
|
+
|
|
318
|
+
int64_t outCutLoopSize = Ops::Base::CeilDiv(tiling_->expandedOutputShape[expandedOutputCutIndex_],
|
|
319
|
+
tiling_->outUbFactor);
|
|
320
|
+
for (int64_t loopIdx = blkProcessIdxStart_; loopIdx < blkProcessIdxEnd_; loopIdx++) {
|
|
321
|
+
if (tiling_->outTailFactor != 0 && (loopIdx + 1) % outCutLoopSize == 0) { // tail
|
|
322
|
+
// MTE2 params tail
|
|
323
|
+
MultiCopyLoopInfo<NDDMA_MAX_DIM_NUM> loopInfoTail = SetupLoopInfo(tiling_->inUbTailSrcShape,
|
|
324
|
+
tiling_->inUbTailDstShape);
|
|
325
|
+
MultiCopyParams<T, NDDMA_MAX_DIM_NUM> paramsTail = {loopInfoTail, constValue};
|
|
326
|
+
CopyIn(loopIdx, paramsTail);
|
|
327
|
+
CopyOut(loopIdx, loopSizeTail_, loopSrcStrideTail_, loopDstStrideTail_);
|
|
328
|
+
} else {
|
|
329
|
+
CopyIn(loopIdx, paramsMain);
|
|
330
|
+
CopyOut(loopIdx, loopSizeMain_, loopSrcStrideMain_, loopDstStrideMain_);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
} // namespace Transpose
|
|
335
|
+
|
|
336
|
+
#endif // TRANSPOSE_CUT_ONE_AXIS
|