ops-tune-workbench-plugin 0.0.0-stage → 0.0.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE.md +14 -0
- package/README.md +39 -2
- package/cordis.patch.yml +20 -0
- package/lib/client.js +4253 -0
- package/lib/host.js +2518 -0
- package/package.json +53 -4
- package/skills/tool-ops-tune/INSTALL.md +52 -0
- package/skills/tool-ops-tune/README.md +167 -0
- package/skills/tool-ops-tune/SKILL.md +93 -0
- package/skills/tool-ops-tune/agent/AGENTS.md +195 -0
- package/skills/tool-ops-tune/agent/agents/perf-implementer.md +68 -0
- package/skills/tool-ops-tune/agent/agents/perf-reviewer.md +86 -0
- package/skills/tool-ops-tune/agent/workflows/task-prompts.md +362 -0
- package/skills/tool-ops-tune/agent/workflows/troubleshooting.md +43 -0
- package/skills/tool-ops-tune/docs/figures/opstune-architecture.png +0 -0
- package/skills/tool-ops-tune/docs/figures/opstune-architecture.svg +253 -0
- package/skills/tool-ops-tune/docs/ops-tune-design.md +404 -0
- package/skills/tool-ops-tune/docs//347/224/250/344/276/213/346/236/204/345/273/272.md +714 -0
- package/skills/tool-ops-tune/docs//347/237/245/350/257/206/345/272/223/350/247/204/350/214/203.md +103 -0
- package/skills/tool-ops-tune/evals/evals.json +242 -0
- package/skills/tool-ops-tune/ops-collect/SKILL.md +146 -0
- package/skills/tool-ops-tune/ops-collect/references/tool-routing.md +86 -0
- package/skills/tool-ops-tune/ops-collect/scripts/collect_board.sh +444 -0
- package/skills/tool-ops-tune/ops-collect/scripts/collect_cases.py +368 -0
- package/skills/tool-ops-tune/ops-collect/scripts/collect_sim.sh +134 -0
- package/skills/tool-ops-tune/ops-collect/scripts/csv_to_db.py +314 -0
- package/skills/tool-ops-tune/ops-collect/scripts/db_bench.py +545 -0
- package/skills/tool-ops-tune/ops-collect/scripts/facts_bridge.py +1063 -0
- package/skills/tool-ops-tune/ops-collect/scripts/isolation_test.py +454 -0
- package/skills/tool-ops-tune/ops-collect/scripts/jsonl_to_cases.py +163 -0
- package/skills/tool-ops-tune/ops-family/SKILL.md +76 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/advanced_tiling.md +92 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/broadcast_design.md +160 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/broadcast_add_kernel.cpp +126 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/broadcast_add_tiling.cpp +144 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/broadcast_tiling_common.h +401 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/onedim_add_kernel.cpp +107 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/code/onedim_add_tiling.cpp +64 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/datacopypad_design.md +46 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/guide.md +14 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/nddma_design.md +53 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/onedim_design.md +114 -0
- package/skills/tool-ops-tune/ops-family/references/broadcast/ub_broadcast_design.md +67 -0
- package/skills/tool-ops-tune/ops-family/references/common/datacopy_optimization_design.md +351 -0
- package/skills/tool-ops-tune/ops-family/references/common/index.md +9 -0
- package/skills/tool-ops-tune/ops-family/references/common/tail_block_design.md +268 -0
- package/skills/tool-ops-tune/ops-family/references/common/timing_harness/index.md +38 -0
- package/skills/tool-ops-tune/ops-family/references/common/timing_harness/timing_harness_aclrt_event.cpp +199 -0
- package/skills/tool-ops-tune/ops-family/references/common/timing_harness/timing_harness_torch_event.py +130 -0
- package/skills/tool-ops-tune/ops-family/references/common/ub_resident_design.md +342 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/guide.md +69 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/index.md +4 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_base.template +81 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_big_dim.template +217 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_cut_one_axis.template +336 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_cut_two_axis.template +781 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_n_last.template +340 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_small_shape.template +321 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_tensor_move.template +102 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_transdata_5hd.template +480 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_transdata_5hd_021axis.template +533 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/dav3510/transpose_with_gather.template +688 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/templates/transpose_tiling_data.template +192 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_big_dim.md +220 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_cut_one_axis.md +175 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_cut_two_axis.md +177 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_fusion_design.md +398 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_gather.md +191 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_n_last.md +276 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_small_shape.md +315 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_tensor_move.md +203 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_tiling_data.md +176 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_vconv_021.md +206 -0
- package/skills/tool-ops-tune/ops-family/references/conversion/transpose_vconv_5hd.md +179 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/double_buffer_design.md +288 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/guide.md +46 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/index.md +10 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/CMakeLists.txt +131 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/README.md +55 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_extension/add_custom_torch.cpp +80 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_extension/ops.h +23 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_extension/register.cpp +30 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_host/add_custom.asc +149 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_host/data_utils.h +97 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_kernel/add_custom_kernel.asc +136 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/op_kernel/add_custom_tiling.h +31 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/run.sh +86 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/scripts/gen_data.py +70 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/scripts/golden.py +34 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/scripts/test_torch.py +99 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/add_custom/scripts/verify_result.py +56 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/templates/usage_guide.md +61 -0
- package/skills/tool-ops-tune/ops-family/references/elementwise/vector_efficiency_design.md +203 -0
- package/skills/tool-ops-tune/ops-family/references/fa/fa_mixed_kernel_design.md +174 -0
- package/skills/tool-ops-tune/ops-family/references/fa/fa_roofline_design.md +141 -0
- package/skills/tool-ops-tune/ops-family/references/fa/guide.md +87 -0
- package/skills/tool-ops-tune/ops-family/references/knowledge_gaps.json +56 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/best-practices-guide.md +14 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/fullload_design.md +344 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/guide.md +52 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/mte2_preload_design.md +320 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/pingpong_design.md +254 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/scale_coalescing_design.md +282 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/streamk_design.md +374 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/swat_design.md +423 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/templates/matmul_basic_blaze.template +165 -0
- package/skills/tool-ops-tune/ops-family/references/matmul/templates/usage_guide.md +29 -0
- package/skills/tool-ops-tune/ops-family/references/mc2/best-practices-guide.md +43 -0
- package/skills/tool-ops-tune/ops-family/references/mc2/guide.md +42 -0
- package/skills/tool-ops-tune/ops-family/references/mc2/local_matmul_design.md +130 -0
- package/skills/tool-ops-tune/ops-family/references/mc2/pipeline_balancing_design.md +502 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/guide.md +178 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/kernel_utils.template +75 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ar_full_load.template +429 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ar_recompute.template +485 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ar_small_r.template +348 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ara_full_load.template +574 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ara_online.template +324 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_ara_recompute.template +436 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/dav310/softmax_v2_base.template +950 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/fused_attention_online_softmax_design.md +428 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ar_full_load.md +137 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ar_recompute.md +135 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ar_small_r.md +122 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ara_full_load.md +144 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ara_online.md +185 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_ara_recompute.md +163 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_tiling.template +460 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/softmax_v2_tiling_data.template +278 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/state_resident_design.md +337 -0
- package/skills/tool-ops-tune/ops-family/references/reduce/templates/usage_guide.md +253 -0
- package/skills/tool-ops-tune/ops-family/references/scalar/coding_principles.md +476 -0
- package/skills/tool-ops-tune/ops-family/references/scalar/conv_scalar.md +163 -0
- package/skills/tool-ops-tune/ops-family/references/scalar/guide.md +30 -0
- package/skills/tool-ops-tune/ops-family/references/simt/guide.md +91 -0
- package/skills/tool-ops-tune/ops-family/references/simt/optimization-guide.md +110 -0
- package/skills/tool-ops-tune/ops-family/references/sort/guide.md +83 -0
- package/skills/tool-ops-tune/ops-family/references/sort/radix_sort.md +354 -0
- package/skills/tool-ops-tune/ops-family/scripts/family_asset_audit.py +68 -0
- package/skills/tool-ops-tune/ops-family/scripts/tiling_candidates.py +419 -0
- package/skills/tool-ops-tune/ops-mechanism/SKILL.md +23 -0
- package/skills/tool-ops-tune/ops-mechanism/cases/ascendc/case_groupedmatmul_pipeline.md +23 -0
- package/skills/tool-ops-tune/ops-mechanism/cases/ascendc/case_triton_atomic_revert.md +19 -0
- package/skills/tool-ops-tune/ops-mechanism/cases/mc2/case_mc2_fusion.md +19 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/api-usage.md +70 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/ascendc-regbase-vf.md +45 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/ascendc-tiling.md +51 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/data-copy.md +67 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/index.md +45 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/mc2-balancing.md +77 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/mc2.md +103 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/memory-hierarchy.md +40 -0
- package/skills/tool-ops-tune/ops-mechanism/references/optimize/pipeline.md +95 -0
- package/skills/tool-ops-tune/ops-mechanism/scripts/case_backfill.py +170 -0
- package/skills/tool-ops-tune/ops-mechanism/scripts/log_gap.py +145 -0
- package/skills/tool-ops-tune/ops-mechanism/scripts/mc2_search.py +367 -0
- package/skills/tool-ops-tune/ops-report/SKILL.md +77 -0
- package/skills/tool-ops-tune/ops-verdict/evals/run_evals.py +837 -0
- package/skills/tool-ops-tune/ops-verdict/evals/smoke/install-hooks.sh +65 -0
- package/skills/tool-ops-tune/ops-verdict/evals/smoke/run_smoke.sh +405 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/core_tail_a5.csv +25 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/few_cores_a5.csv +5 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/mte2_bound_a2.csv +17 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/multi_kernel_mix_a5.csv +7 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/no_bound_a5.csv +9 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/serial_no_overlap_a2.csv +9 -0
- package/skills/tool-ops-tune/ops-verdict/fixtures/vec_bound_a5.csv +9 -0
- package/skills/tool-ops-tune/ops-verdict/references/io-contracts.md +94 -0
- package/skills/tool-ops-tune/ops-verdict/references/rule-schema.md +130 -0
- package/skills/tool-ops-tune/ops-verdict/references/scope-schema.md +61 -0
- package/skills/tool-ops-tune/ops-verdict/references/soc-spec.yaml +140 -0
- package/skills/tool-ops-tune/ops-verdict/rules/L0-instruction.yaml +67 -0
- package/skills/tool-ops-tune/ops-verdict/rules/L1-single-core.yaml +316 -0
- package/skills/tool-ops-tune/ops-verdict/rules/L2-inter-core.yaml +173 -0
- package/skills/tool-ops-tune/ops-verdict/rules/L3-inter-card.yaml +156 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/batch_pipeline.py +444 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/calibrate.py +249 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/delta_compare.py +244 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/render_html.py +274 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/snapshot.py +200 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/trace_bubble.py +331 -0
- package/skills/tool-ops-tune/ops-verdict/scripts/verdict.py +1072 -0
- package/skills/tool-ops-tune/scripts/diff_audit.py +281 -0
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2026 Huawei Technologies Co., Ltd.
|
|
3
|
+
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
|
4
|
+
* CANN Open Software License Agreement Version 2.0 (the "License").
|
|
5
|
+
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
|
6
|
+
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
|
7
|
+
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
|
8
|
+
* See LICENSE in the root of the software repository for the full text of the License.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/*!
|
|
12
|
+
* \file transpose_transdata_5hd_021.h
|
|
13
|
+
* \brief 3D 021 transpose using TransDataTo5HD: [N, H, W] -> [N, W, H]
|
|
14
|
+
* Supports 8-bit (int8/uint8), 16-bit (half/bfloat16), and 32-bit (float/int32) data types.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
#ifndef KERNEL_TRANSPOSE_TRANSDATA_5HD_021_H_
|
|
18
|
+
#define KERNEL_TRANSPOSE_TRANSDATA_5HD_021_H_
|
|
19
|
+
|
|
20
|
+
#include <type_traits>
|
|
21
|
+
#include "op_kernel/platform_util.h"
|
|
22
|
+
#include "transpose_base.h"
|
|
23
|
+
|
|
24
|
+
namespace Transpose {
|
|
25
|
+
using namespace AscendC;
|
|
26
|
+
static constexpr int64_t TRANSELEM_021 = 16;
|
|
27
|
+
static constexpr int64_t BLOCKELEM_8BIT = 32;
|
|
28
|
+
static constexpr int64_t BLOCKELEM_16BIT = 16;
|
|
29
|
+
static constexpr int64_t BLOCKELEM_32BIT = 8;
|
|
30
|
+
static constexpr int64_t DST_STRIDE_FACTOR_16BIT = 1;
|
|
31
|
+
static constexpr int64_t DST_STRIDE_FACTOR_32BIT = 2;
|
|
32
|
+
|
|
33
|
+
template <typename T>
|
|
34
|
+
class KernelTransDataTo5HD021 {
|
|
35
|
+
public:
|
|
36
|
+
__aicore__ inline KernelTransDataTo5HD021(){};
|
|
37
|
+
__aicore__ inline void Init(GM_ADDR x, GM_ADDR y, const Transpose021VCONVTilingData* tilingData, TPipe* pipe);
|
|
38
|
+
__aicore__ inline void Process();
|
|
39
|
+
|
|
40
|
+
private:
|
|
41
|
+
__aicore__ inline void ProcessOneBatch(int64_t nIdx);
|
|
42
|
+
__aicore__ inline void ProcessOneBatchHSplit(int64_t nIdx);
|
|
43
|
+
__aicore__ inline void CopyIn(int64_t nIdx, uint32_t ubLoop, uint32_t r, uint32_t c);
|
|
44
|
+
__aicore__ inline void CopyInWAligned(uint32_t actualValidRows, uint32_t r, uint32_t c);
|
|
45
|
+
__aicore__ inline void CopyInWUnaligned(uint32_t actualValidRows, uint32_t r, uint32_t c, uint32_t ubLoop);
|
|
46
|
+
__aicore__ inline void CopyInHSplit(int64_t nIdx, uint32_t ubLoop, uint32_t r, uint32_t c);
|
|
47
|
+
__aicore__ inline void Compute(uint32_t r, uint32_t c, uint32_t rAlign, uint32_t cAlign);
|
|
48
|
+
__aicore__ inline void ComputeRConv(uint32_t r, uint32_t c, uint32_t rAlign, uint32_t cAlign);
|
|
49
|
+
__aicore__ inline void Compute8BitCore(uint32_t r, uint32_t c, uint32_t rAlign);
|
|
50
|
+
__aicore__ inline void ComputeRConvGeneric(uint32_t r, uint32_t c, uint32_t rAlign, uint32_t cAlign);
|
|
51
|
+
__aicore__ inline void ComputeCConv(uint32_t r, uint32_t c, uint32_t rAlign);
|
|
52
|
+
__aicore__ inline void ComputeCConvGeneric(uint32_t r, uint32_t c, uint32_t rAlign);
|
|
53
|
+
__aicore__ inline void CopyOut(int64_t nIdx, uint32_t ubLoop, uint32_t r, uint32_t c);
|
|
54
|
+
__aicore__ inline void CopyOutHSplit(int64_t nIdx, uint32_t ubLoop, uint32_t r, uint32_t c);
|
|
55
|
+
__aicore__ inline void CopyOutAligned(uint32_t r, uint32_t c);
|
|
56
|
+
__aicore__ inline void CopyOutUnaligned(uint32_t r, uint32_t c, uint32_t ubLoop);
|
|
57
|
+
|
|
58
|
+
static constexpr int64_t blockElem = (sizeof(T) == 1) ? BLOCKELEM_8BIT :
|
|
59
|
+
(sizeof(T) == 4) ? BLOCKELEM_32BIT :
|
|
60
|
+
BLOCKELEM_16BIT;
|
|
61
|
+
static constexpr int64_t dstStrideFactor = (sizeof(T) == 4) ? DST_STRIDE_FACTOR_32BIT : DST_STRIDE_FACTOR_16BIT;
|
|
62
|
+
|
|
63
|
+
const Transpose021VCONVTilingData* tiling_ = nullptr;
|
|
64
|
+
TQue<TPosition::VECIN, 1> inQueueSrc;
|
|
65
|
+
TQue<TPosition::VECOUT, 1> outQueueDst;
|
|
66
|
+
GlobalTensor<T> srcGlobal, dstGlobal;
|
|
67
|
+
|
|
68
|
+
int64_t blockIdx_ = 0;
|
|
69
|
+
int64_t fullCoreNum_ = 0;
|
|
70
|
+
int64_t nStart_ = 0;
|
|
71
|
+
int64_t nCount_ = 0;
|
|
72
|
+
int64_t hStart_ = 0;
|
|
73
|
+
int64_t hCount_ = 0;
|
|
74
|
+
|
|
75
|
+
DataCopyExtParams copyInParams_;
|
|
76
|
+
DataCopyExtParams copyOutParams_;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
template <typename T>
|
|
80
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::Init(GM_ADDR x, GM_ADDR y,
|
|
81
|
+
const Transpose021VCONVTilingData* tilingData, TPipe* pipe)
|
|
82
|
+
{
|
|
83
|
+
tiling_ = tilingData;
|
|
84
|
+
blockIdx_ = GetBlockIdx();
|
|
85
|
+
srcGlobal.SetGlobalBuffer((__gm__ T*)x);
|
|
86
|
+
dstGlobal.SetGlobalBuffer((__gm__ T*)y);
|
|
87
|
+
|
|
88
|
+
if (tiling_->UseHSplit) {
|
|
89
|
+
fullCoreNum_ = tiling_->UsedCoreNum - 1;
|
|
90
|
+
if (blockIdx_ < fullCoreNum_) {
|
|
91
|
+
hStart_ = blockIdx_ * tiling_->HPerCore;
|
|
92
|
+
hCount_ = tiling_->HPerCore;
|
|
93
|
+
} else if (blockIdx_ < tiling_->UsedCoreNum) {
|
|
94
|
+
hStart_ = fullCoreNum_ * tiling_->HPerCore;
|
|
95
|
+
hCount_ = tiling_->HTailCore;
|
|
96
|
+
}
|
|
97
|
+
nStart_ = 0;
|
|
98
|
+
nCount_ = tiling_->NLen;
|
|
99
|
+
} else {
|
|
100
|
+
fullCoreNum_ = tiling_->UsedCoreNum - 1;
|
|
101
|
+
if (blockIdx_ < fullCoreNum_) {
|
|
102
|
+
nStart_ = blockIdx_ * tiling_->NPerCore;
|
|
103
|
+
nCount_ = tiling_->NPerCore;
|
|
104
|
+
} else if (blockIdx_ < tiling_->UsedCoreNum) {
|
|
105
|
+
nStart_ = fullCoreNum_ * tiling_->NPerCore;
|
|
106
|
+
nCount_ = tiling_->NTailCore;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
pipe->InitBuffer(inQueueSrc, BUFFER_NUM, tiling_->AvailableUbSize);
|
|
111
|
+
pipe->InitBuffer(outQueueDst, BUFFER_NUM, tiling_->AvailableUbSize);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
template <typename T>
|
|
115
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::ComputeRConv(uint32_t r, uint32_t c, uint32_t rAlign,
|
|
116
|
+
uint32_t cAlign)
|
|
117
|
+
{
|
|
118
|
+
if constexpr (sizeof(T) == 1) {
|
|
119
|
+
Compute8BitCore(r, c, rAlign);
|
|
120
|
+
} else {
|
|
121
|
+
ComputeRConvGeneric(r, c, rAlign, cAlign);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
template <typename T>
|
|
126
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::Compute8BitCore(uint32_t r, uint32_t c, uint32_t rAlign)
|
|
127
|
+
{
|
|
128
|
+
LocalTensor<T> srcLocal = inQueueSrc.DeQue<T>();
|
|
129
|
+
LocalTensor<T> dstLocal = outQueueDst.AllocTensor<T>();
|
|
130
|
+
uint16_t cAlignBlocks = c / blockElem;
|
|
131
|
+
uint8_t evenUbCount = (rAlign + 1) / 2;
|
|
132
|
+
uint8_t oddUbCount = rAlign / 2;
|
|
133
|
+
TransDataTo5HDParams evenParams{false, false, evenUbCount,
|
|
134
|
+
static_cast<uint16_t>((evenUbCount <= 1) ? 0 : dstStrideFactor),
|
|
135
|
+
static_cast<uint16_t>((evenUbCount <= 1) ? 0 : c)};
|
|
136
|
+
TransDataTo5HDParams oddParams{true, false, oddUbCount,
|
|
137
|
+
static_cast<uint16_t>((oddUbCount <= 1) ? 0 : dstStrideFactor),
|
|
138
|
+
static_cast<uint16_t>((oddUbCount <= 1) ? 0 : c)};
|
|
139
|
+
for (uint16_t j = 0; j < cAlignBlocks; j++) {
|
|
140
|
+
uint64_t srcList[TRANSELEM_021];
|
|
141
|
+
for (uint16_t i = 0; i < TRANSELEM_021; i++) {
|
|
142
|
+
srcList[i] = srcLocal[i * c + j * blockElem].GetPhyAddr();
|
|
143
|
+
}
|
|
144
|
+
uint64_t srcListOdd[TRANSELEM_021];
|
|
145
|
+
if (oddUbCount > 0) {
|
|
146
|
+
for (uint16_t i = 0; i < TRANSELEM_021; i++) {
|
|
147
|
+
srcListOdd[i] = srcLocal[(i + TRANSELEM_021) * c + j * blockElem].GetPhyAddr();
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
for (uint16_t srcHalfIdx = 0; srcHalfIdx < 2; srcHalfIdx++) {
|
|
151
|
+
bool srcHighHalf = (srcHalfIdx == 1);
|
|
152
|
+
uint16_t dstRowBase = j * blockElem + srcHalfIdx * TRANSELEM_021;
|
|
153
|
+
|
|
154
|
+
uint64_t dstList[TRANSELEM_021];
|
|
155
|
+
for (uint16_t k = 0; k < TRANSELEM_021; k++) {
|
|
156
|
+
dstList[k] = dstLocal[(dstRowBase + k) * r].GetPhyAddr();
|
|
157
|
+
}
|
|
158
|
+
evenParams.srcHighHalf = srcHighHalf;
|
|
159
|
+
if (evenUbCount > 0) {
|
|
160
|
+
TransDataTo5HD<T>(dstList, srcList, evenParams);
|
|
161
|
+
}
|
|
162
|
+
if (oddUbCount > 0) {
|
|
163
|
+
PipeBarrier<PIPE_V>();
|
|
164
|
+
oddParams.srcHighHalf = srcHighHalf;
|
|
165
|
+
TransDataTo5HD<T>(dstList, srcListOdd, oddParams);
|
|
166
|
+
}
|
|
167
|
+
if (srcHalfIdx == 0) {
|
|
168
|
+
PipeBarrier<PIPE_V>();
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
outQueueDst.EnQue<T>(dstLocal);
|
|
173
|
+
inQueueSrc.FreeTensor(srcLocal);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
template <typename T>
|
|
177
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::ComputeRConvGeneric(uint32_t r, uint32_t c, uint32_t rAlign,
|
|
178
|
+
uint32_t cAlign)
|
|
179
|
+
{
|
|
180
|
+
LocalTensor<T> srcLocal = inQueueSrc.DeQue<T>();
|
|
181
|
+
LocalTensor<T> dstLocal = outQueueDst.AllocTensor<T>();
|
|
182
|
+
|
|
183
|
+
TransDataTo5HDParams transDataParams;
|
|
184
|
+
transDataParams.dstHighHalf = false;
|
|
185
|
+
transDataParams.srcHighHalf = false;
|
|
186
|
+
transDataParams.repeatTimes = r / TRANSELEM_021;
|
|
187
|
+
transDataParams.dstRepStride = transDataParams.repeatTimes == 1 ? 0 : dstStrideFactor;
|
|
188
|
+
transDataParams.srcRepStride = transDataParams.repeatTimes == 1 ? 0 : cAlign * TRANSELEM_021;
|
|
189
|
+
|
|
190
|
+
for (uint16_t j = 0; j < cAlign; j++) {
|
|
191
|
+
uint64_t srcLocalList[TRANSELEM_021];
|
|
192
|
+
for (uint16_t i = 0; i < TRANSELEM_021; i++) {
|
|
193
|
+
srcLocalList[i] = srcLocal[i * c + j * blockElem].GetPhyAddr();
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
uint64_t dstLocalList[TRANSELEM_021];
|
|
197
|
+
if constexpr (sizeof(T) == 4) {
|
|
198
|
+
constexpr int64_t dstRows = TRANSELEM_021 / dstStrideFactor;
|
|
199
|
+
for (uint16_t k = 0; k < dstRows; k++) {
|
|
200
|
+
dstLocalList[dstStrideFactor * k] = dstLocal[(k + j * blockElem) * r].GetPhyAddr();
|
|
201
|
+
dstLocalList[dstStrideFactor * k + 1] = dstLocal[(k + j * blockElem) * r + blockElem].GetPhyAddr();
|
|
202
|
+
}
|
|
203
|
+
} else {
|
|
204
|
+
for (uint16_t i = 0; i < TRANSELEM_021; i++) {
|
|
205
|
+
dstLocalList[i] = dstLocal[i * r + j * TRANSELEM_021 * r].GetPhyAddr();
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
TransDataTo5HD<T>(dstLocalList, srcLocalList, transDataParams);
|
|
209
|
+
}
|
|
210
|
+
outQueueDst.EnQue<T>(dstLocal);
|
|
211
|
+
inQueueSrc.FreeTensor(srcLocal);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
template <typename T>
|
|
215
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::ComputeCConv(uint32_t r, uint32_t c, uint32_t rAlign)
|
|
216
|
+
{
|
|
217
|
+
if constexpr (sizeof(T) == 1) {
|
|
218
|
+
Compute8BitCore(r, c, rAlign);
|
|
219
|
+
} else {
|
|
220
|
+
ComputeCConvGeneric(r, c, rAlign);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
template <typename T>
|
|
225
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::ComputeCConvGeneric(uint32_t r, uint32_t c, uint32_t rAlign)
|
|
226
|
+
{
|
|
227
|
+
LocalTensor<T> srcLocal = inQueueSrc.DeQue<T>();
|
|
228
|
+
LocalTensor<T> dstLocal = outQueueDst.AllocTensor<T>();
|
|
229
|
+
|
|
230
|
+
TransDataTo5HDParams transDataParams;
|
|
231
|
+
transDataParams.dstHighHalf = false;
|
|
232
|
+
transDataParams.srcHighHalf = false;
|
|
233
|
+
transDataParams.repeatTimes = c / blockElem;
|
|
234
|
+
transDataParams.dstRepStride = transDataParams.repeatTimes == 1 ? 0 : rAlign * TRANSELEM_021;
|
|
235
|
+
transDataParams.srcRepStride = transDataParams.repeatTimes == 1 ? 0 : 1;
|
|
236
|
+
|
|
237
|
+
for (uint16_t j = 0; j < rAlign; j++) {
|
|
238
|
+
uint64_t srcLocalList[TRANSELEM_021];
|
|
239
|
+
for (uint16_t i = 0; i < TRANSELEM_021; i++) {
|
|
240
|
+
srcLocalList[i] = srcLocal[i * c + j * TRANSELEM_021 * c].GetPhyAddr();
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
uint64_t dstLocalList[TRANSELEM_021];
|
|
244
|
+
if constexpr (sizeof(T) == 4) {
|
|
245
|
+
constexpr int64_t dstRows = TRANSELEM_021 / dstStrideFactor;
|
|
246
|
+
for (uint16_t k = 0; k < dstRows; k++) {
|
|
247
|
+
dstLocalList[dstStrideFactor * k] = dstLocal[k * r + j * TRANSELEM_021].GetPhyAddr();
|
|
248
|
+
dstLocalList[dstStrideFactor * k + 1] = dstLocal[k * r + j * TRANSELEM_021 + blockElem].GetPhyAddr();
|
|
249
|
+
}
|
|
250
|
+
} else {
|
|
251
|
+
for (uint16_t i = 0; i < TRANSELEM_021; i++) {
|
|
252
|
+
dstLocalList[i] = dstLocal[i * r + j * TRANSELEM_021].GetPhyAddr();
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
TransDataTo5HD<T>(dstLocalList, srcLocalList, transDataParams);
|
|
256
|
+
}
|
|
257
|
+
outQueueDst.EnQue<T>(dstLocal);
|
|
258
|
+
inQueueSrc.FreeTensor(srcLocal);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
template <typename T>
|
|
262
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::Compute(uint32_t r, uint32_t c, uint32_t rAlign, uint32_t cAlign)
|
|
263
|
+
{
|
|
264
|
+
if (tiling_->UseRConv) {
|
|
265
|
+
ComputeRConv(r, c, rAlign, cAlign);
|
|
266
|
+
} else {
|
|
267
|
+
ComputeCConv(r, c, rAlign);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
template <typename T>
|
|
272
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::CopyIn(int64_t nIdx, uint32_t ubLoop, uint32_t r, uint32_t c)
|
|
273
|
+
{
|
|
274
|
+
LocalTensor<T> srcLocal = inQueueSrc.AllocTensor<T>();
|
|
275
|
+
|
|
276
|
+
int64_t nSrcOffset = nIdx * tiling_->HLen * tiling_->WLen;
|
|
277
|
+
int64_t ubSrcOffset = 0;
|
|
278
|
+
if (tiling_->UseRConv) {
|
|
279
|
+
ubSrcOffset = ubLoop * tiling_->rUbSplitPara.UbAlignFactor * TRANSELEM_021 * tiling_->WLen;
|
|
280
|
+
} else {
|
|
281
|
+
ubSrcOffset = ubLoop * tiling_->cUbSplitPara.UbAlignFactor * TRANSELEM_021;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
uint32_t actualValidRows;
|
|
285
|
+
if (tiling_->UseRConv) {
|
|
286
|
+
int64_t hOffsetSrc = ubLoop * tiling_->rUbSplitPara.UbAlignFactor * TRANSELEM_021;
|
|
287
|
+
int64_t remainingRows = tiling_->HLen - hOffsetSrc;
|
|
288
|
+
actualValidRows = (remainingRows < r) ? remainingRows : r;
|
|
289
|
+
} else {
|
|
290
|
+
actualValidRows = tiling_->HLen;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
DataCopyPadExtParams<T> copyInPadParams{false, 0, 0, 0};
|
|
294
|
+
int64_t wAlignCheck = (sizeof(T) == 1) ? blockElem : TRANSELEM_021;
|
|
295
|
+
if (tiling_->WLen % wAlignCheck == 0) {
|
|
296
|
+
CopyInWAligned(actualValidRows, r, c);
|
|
297
|
+
} else {
|
|
298
|
+
CopyInWUnaligned(actualValidRows, r, c, ubLoop);
|
|
299
|
+
}
|
|
300
|
+
DataCopyPad(srcLocal, srcGlobal[nSrcOffset + ubSrcOffset], copyInParams_, copyInPadParams);
|
|
301
|
+
inQueueSrc.EnQue(srcLocal);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
template <typename T>
|
|
305
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::CopyInWAligned(uint32_t actualValidRows, uint32_t r, uint32_t c)
|
|
306
|
+
{
|
|
307
|
+
if (tiling_->UseRConv) {
|
|
308
|
+
copyInParams_.blockCount = 1;
|
|
309
|
+
copyInParams_.blockLen = actualValidRows * c * sizeof(T);
|
|
310
|
+
copyInParams_.srcStride = 0;
|
|
311
|
+
copyInParams_.dstStride = 0;
|
|
312
|
+
} else if (c == tiling_->WLen) {
|
|
313
|
+
copyInParams_.blockCount = 1;
|
|
314
|
+
copyInParams_.blockLen = actualValidRows * c * sizeof(T);
|
|
315
|
+
copyInParams_.srcStride = 0;
|
|
316
|
+
copyInParams_.dstStride = 0;
|
|
317
|
+
} else {
|
|
318
|
+
copyInParams_.blockCount = actualValidRows;
|
|
319
|
+
copyInParams_.blockLen = c * sizeof(T);
|
|
320
|
+
copyInParams_.srcStride = (tiling_->WLen - c) * sizeof(T);
|
|
321
|
+
copyInParams_.dstStride = 0;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
template <typename T>
|
|
326
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::CopyInWUnaligned(uint32_t actualValidRows, uint32_t r, uint32_t c,
|
|
327
|
+
uint32_t ubLoop)
|
|
328
|
+
{
|
|
329
|
+
if (tiling_->UseRConv) {
|
|
330
|
+
copyInParams_.blockCount = actualValidRows;
|
|
331
|
+
copyInParams_.blockLen = tiling_->WLen * sizeof(T);
|
|
332
|
+
copyInParams_.srcStride = 0;
|
|
333
|
+
copyInParams_.dstStride = (tiling_->WAlignBlockElem - tiling_->WLen) * sizeof(T) / BLOCK_SIZE_BYTE;
|
|
334
|
+
} else {
|
|
335
|
+
int32_t wOffset = ubLoop * tiling_->cUbSplitPara.UbAlignFactor * TRANSELEM_021;
|
|
336
|
+
int32_t validW = (tiling_->WLen - wOffset < c) ? tiling_->WLen - wOffset : c;
|
|
337
|
+
copyInParams_.blockCount = actualValidRows;
|
|
338
|
+
copyInParams_.blockLen = validW * sizeof(T);
|
|
339
|
+
copyInParams_.srcStride = (tiling_->WLen - validW) * sizeof(T);
|
|
340
|
+
copyInParams_.dstStride = (c - validW) * sizeof(T) / BLOCK_SIZE_BYTE;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
template <typename T>
|
|
345
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::CopyOut(int64_t nIdx, uint32_t ubLoop, uint32_t r, uint32_t c)
|
|
346
|
+
{
|
|
347
|
+
LocalTensor<T> dstLocal = outQueueDst.DeQue<T>();
|
|
348
|
+
|
|
349
|
+
int64_t nDstOffset = nIdx * tiling_->WLen * tiling_->HLen;
|
|
350
|
+
int64_t ubDstOffset = 0;
|
|
351
|
+
if (tiling_->UseRConv) {
|
|
352
|
+
ubDstOffset = ubLoop * tiling_->rUbSplitPara.UbAlignFactor * TRANSELEM_021;
|
|
353
|
+
} else {
|
|
354
|
+
ubDstOffset = ubLoop * tiling_->cUbSplitPara.UbAlignFactor * TRANSELEM_021 * tiling_->HLen;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
int64_t hAlignCheck = (sizeof(T) == 1) ? blockElem : TRANSELEM_021;
|
|
358
|
+
int64_t wAlignCheck = (sizeof(T) == 1) ? blockElem : TRANSELEM_021;
|
|
359
|
+
if (tiling_->HLen % hAlignCheck == 0 && tiling_->WLen % wAlignCheck == 0) {
|
|
360
|
+
CopyOutAligned(r, c);
|
|
361
|
+
} else {
|
|
362
|
+
CopyOutUnaligned(r, c, ubLoop);
|
|
363
|
+
}
|
|
364
|
+
DataCopyPad(dstGlobal[nDstOffset + ubDstOffset], dstLocal, copyOutParams_);
|
|
365
|
+
outQueueDst.FreeTensor(dstLocal);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
template <typename T>
|
|
369
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::CopyOutAligned(uint32_t r, uint32_t c)
|
|
370
|
+
{
|
|
371
|
+
if (tiling_->UseRConv) {
|
|
372
|
+
if (r == tiling_->HLen) {
|
|
373
|
+
copyOutParams_.blockCount = 1;
|
|
374
|
+
copyOutParams_.blockLen = c * r * sizeof(T);
|
|
375
|
+
copyOutParams_.srcStride = 0;
|
|
376
|
+
copyOutParams_.dstStride = 0;
|
|
377
|
+
} else {
|
|
378
|
+
copyOutParams_.blockCount = c;
|
|
379
|
+
copyOutParams_.blockLen = r * sizeof(T);
|
|
380
|
+
copyOutParams_.srcStride = 0;
|
|
381
|
+
copyOutParams_.dstStride = (tiling_->HLen - r) * sizeof(T);
|
|
382
|
+
}
|
|
383
|
+
} else {
|
|
384
|
+
copyOutParams_.blockCount = 1;
|
|
385
|
+
copyOutParams_.blockLen = c * r * sizeof(T);
|
|
386
|
+
copyOutParams_.srcStride = 0;
|
|
387
|
+
copyOutParams_.dstStride = 0;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
template <typename T>
|
|
392
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::CopyOutUnaligned(uint32_t r, uint32_t c, uint32_t ubLoop)
|
|
393
|
+
{
|
|
394
|
+
if (tiling_->UseRConv) {
|
|
395
|
+
int32_t hOffset = ubLoop * tiling_->rUbSplitPara.UbAlignFactor * TRANSELEM_021;
|
|
396
|
+
uint32_t validLen = (tiling_->HLen - hOffset < r) ? tiling_->HLen - hOffset : r;
|
|
397
|
+
uint32_t validCount = (tiling_->WLen < c) ? tiling_->WLen : c;
|
|
398
|
+
copyOutParams_.blockCount = validCount;
|
|
399
|
+
copyOutParams_.blockLen = validLen * sizeof(T);
|
|
400
|
+
copyOutParams_.srcStride = (r - validLen) * sizeof(T) / BLOCK_SIZE_BYTE;
|
|
401
|
+
copyOutParams_.dstStride = (tiling_->HLen - validLen) * sizeof(T);
|
|
402
|
+
} else {
|
|
403
|
+
uint32_t validLen = tiling_->HLen;
|
|
404
|
+
int32_t wOffset = ubLoop * tiling_->cUbSplitPara.UbAlignFactor * TRANSELEM_021;
|
|
405
|
+
uint32_t validCount = (tiling_->WLen - wOffset < c) ? tiling_->WLen - wOffset : c;
|
|
406
|
+
copyOutParams_.blockCount = validCount;
|
|
407
|
+
copyOutParams_.blockLen = validLen * sizeof(T);
|
|
408
|
+
copyOutParams_.srcStride = (r - validLen) * sizeof(T) / BLOCK_SIZE_BYTE;
|
|
409
|
+
copyOutParams_.dstStride = 0;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
template <typename T>
|
|
414
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::ProcessOneBatch(int64_t nIdx)
|
|
415
|
+
{
|
|
416
|
+
for (uint32_t i = 0; i < tiling_->UbLoopCount; i++) {
|
|
417
|
+
uint32_t r, c, rAlign, cAlign;
|
|
418
|
+
if (i < tiling_->UbLoopCount - 1) {
|
|
419
|
+
rAlign = tiling_->rUbSplitPara.UbAlignFactor;
|
|
420
|
+
r = tiling_->rUbSplitPara.UbFactor;
|
|
421
|
+
cAlign = tiling_->cUbSplitPara.UbAlignFactor;
|
|
422
|
+
c = tiling_->cUbSplitPara.UbFactor;
|
|
423
|
+
} else {
|
|
424
|
+
rAlign = tiling_->rUbSplitPara.UbTailAlignFactor;
|
|
425
|
+
r = tiling_->rUbSplitPara.UbTailFactor;
|
|
426
|
+
cAlign = tiling_->cUbSplitPara.UbTailAlignFactor;
|
|
427
|
+
c = tiling_->cUbSplitPara.UbTailFactor;
|
|
428
|
+
}
|
|
429
|
+
CopyIn(nIdx, i, r, c);
|
|
430
|
+
Compute(r, c, rAlign, cAlign);
|
|
431
|
+
CopyOut(nIdx, i, r, c);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
template <typename T>
|
|
436
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::CopyInHSplit(int64_t nIdx, uint32_t ubLoop, uint32_t r, uint32_t c)
|
|
437
|
+
{
|
|
438
|
+
LocalTensor<T> srcLocal = inQueueSrc.AllocTensor<T>();
|
|
439
|
+
|
|
440
|
+
int64_t nSrcOffset = nIdx * tiling_->HLen * tiling_->WLen;
|
|
441
|
+
int64_t hSrcOffset = hStart_ * tiling_->WLen;
|
|
442
|
+
int64_t ubSrcOffset = ubLoop * tiling_->cUbSplitPara.UbFactor;
|
|
443
|
+
|
|
444
|
+
int64_t totalSrcOffset = nSrcOffset + hSrcOffset + ubSrcOffset;
|
|
445
|
+
|
|
446
|
+
DataCopyPadExtParams<T> copyInPadParams{false, 0, 0, 0};
|
|
447
|
+
|
|
448
|
+
int64_t wAlignCheck = blockElem;
|
|
449
|
+
if (tiling_->WLen % wAlignCheck == 0) {
|
|
450
|
+
if (c == tiling_->WAlignBlockElem) {
|
|
451
|
+
copyInParams_.blockCount = 1;
|
|
452
|
+
copyInParams_.blockLen = hCount_ * c * sizeof(T);
|
|
453
|
+
copyInParams_.srcStride = 0;
|
|
454
|
+
copyInParams_.dstStride = 0;
|
|
455
|
+
} else {
|
|
456
|
+
copyInParams_.blockCount = hCount_;
|
|
457
|
+
copyInParams_.blockLen = c * sizeof(T);
|
|
458
|
+
copyInParams_.srcStride = (tiling_->WLen - c) * sizeof(T);
|
|
459
|
+
copyInParams_.dstStride = 0;
|
|
460
|
+
}
|
|
461
|
+
DataCopyPad(srcLocal, srcGlobal[totalSrcOffset], copyInParams_, copyInPadParams);
|
|
462
|
+
} else {
|
|
463
|
+
int32_t wOffset = ubLoop * tiling_->cUbSplitPara.UbFactor;
|
|
464
|
+
int32_t validW = (tiling_->WLen - wOffset < c) ? tiling_->WLen - wOffset : c;
|
|
465
|
+
copyInParams_.blockCount = hCount_;
|
|
466
|
+
copyInParams_.blockLen = validW * sizeof(T);
|
|
467
|
+
copyInParams_.srcStride = (tiling_->WLen - validW) * sizeof(T);
|
|
468
|
+
copyInParams_.dstStride = (c - validW) * sizeof(T) / BLOCK_SIZE_BYTE;
|
|
469
|
+
DataCopyPad(srcLocal, srcGlobal[totalSrcOffset], copyInParams_, copyInPadParams);
|
|
470
|
+
}
|
|
471
|
+
inQueueSrc.EnQue(srcLocal);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
template <typename T>
|
|
475
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::CopyOutHSplit(int64_t nIdx, uint32_t ubLoop, uint32_t r, uint32_t c)
|
|
476
|
+
{
|
|
477
|
+
LocalTensor<T> dstLocal = outQueueDst.DeQue<T>();
|
|
478
|
+
|
|
479
|
+
int64_t nDstOffset = nIdx * tiling_->WLen * tiling_->HLen;
|
|
480
|
+
int64_t ubDstOffset = ubLoop * tiling_->cUbSplitPara.UbFactor * tiling_->HLen;
|
|
481
|
+
int32_t validH = (tiling_->HLen - hStart_ < hCount_) ? tiling_->HLen - hStart_ : hCount_;
|
|
482
|
+
|
|
483
|
+
int32_t validW = (tiling_->WLen - (ubLoop * tiling_->cUbSplitPara.UbFactor) < c) ?
|
|
484
|
+
tiling_->WLen - (ubLoop * tiling_->cUbSplitPara.UbFactor) :
|
|
485
|
+
c;
|
|
486
|
+
copyOutParams_.blockCount = validW;
|
|
487
|
+
copyOutParams_.blockLen = validH * sizeof(T);
|
|
488
|
+
copyOutParams_.srcStride = (r - validH) * sizeof(T) / BLOCK_SIZE_BYTE;
|
|
489
|
+
copyOutParams_.dstStride = (tiling_->HLen - validH) * sizeof(T);
|
|
490
|
+
DataCopyPad(dstGlobal[nDstOffset + hStart_ + ubDstOffset], dstLocal, copyOutParams_);
|
|
491
|
+
outQueueDst.FreeTensor(dstLocal);
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
template <typename T>
|
|
495
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::ProcessOneBatchHSplit(int64_t nIdx)
|
|
496
|
+
{
|
|
497
|
+
int64_t hAlignUnit = (sizeof(T) == 1) ? blockElem : TRANSELEM_021;
|
|
498
|
+
uint32_t rAlign = (hCount_ + hAlignUnit - 1) / hAlignUnit;
|
|
499
|
+
uint32_t r = hCount_;
|
|
500
|
+
for (uint32_t i = 0; i < tiling_->UbLoopCount; i++) {
|
|
501
|
+
uint32_t c, cAlign;
|
|
502
|
+
if (i < tiling_->UbLoopCount - 1) {
|
|
503
|
+
cAlign = tiling_->cUbSplitPara.UbAlignFactor;
|
|
504
|
+
c = tiling_->cUbSplitPara.UbFactor;
|
|
505
|
+
} else {
|
|
506
|
+
cAlign = tiling_->cUbSplitPara.UbTailAlignFactor;
|
|
507
|
+
c = tiling_->cUbSplitPara.UbTailFactor;
|
|
508
|
+
}
|
|
509
|
+
CopyInHSplit(nIdx, i, r, c);
|
|
510
|
+
Compute(r, c, rAlign, cAlign);
|
|
511
|
+
CopyOutHSplit(nIdx, i, r, c);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
template <typename T>
|
|
516
|
+
__aicore__ inline void KernelTransDataTo5HD021<T>::Process()
|
|
517
|
+
{
|
|
518
|
+
if (blockIdx_ >= tiling_->UsedCoreNum) {
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
if (tiling_->UseHSplit) {
|
|
522
|
+
for (int64_t n = 0; n < nCount_; n++) {
|
|
523
|
+
ProcessOneBatchHSplit(n);
|
|
524
|
+
}
|
|
525
|
+
} else {
|
|
526
|
+
for (int64_t n = 0; n < nCount_; n++) {
|
|
527
|
+
ProcessOneBatch(nStart_ + n);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
} // namespace Transpose
|
|
533
|
+
#endif
|