@ttsc/win32-arm64 0.33.1 → 0.34.0

package/bin/go/src/simd/archsimd/_gen/simdgen/gen_utility.go

@@ -16,7 +16,7 @@ import (
 	"slices"
 	"sort"
 	"strings"
-	"text/template"
+	"text/template" // NOLINT
 	"unicode"
 )
 
@@ -88,15 +88,17 @@ const (
 	OneImmIn              // vector and immediate input
 	OneKmaskImmIn         // vector, kmask, and immediate inputs
 	PureKmaskIn           // only mask inputs.
+	VlistIn               // vector list input (e.g. TBL [v1 v2 v3] vidx)
 )
 
 const (
-	InvalidOut     outShape = iota
-	NoOut                   // no output
-	OneVregOut              // (one) vector register output
-	OneGregOut              // (one) general register output
-	OneKmaskOut             // mask output
-	OneVregOutAtIn          // the first input is also the output
+	InvalidOut       outShape = iota
+	NoOut                     // no output
+	OneVregOut                // (one) vector register output
+	OneGregOut                // (one) general register output
+	OneKmaskOut               // mask output
+	OneVregOutAtIn            // the first input is also the output
+	OneVregOutScalar          // (one) vector register output scalar in lane 0, other lanes zeroed
 )
 
 const (
@@ -112,6 +114,7 @@ const (
 	ConstImm             // const only immediate
 	VarImm               // pure imm argument provided by the users
 	ConstVarImm          // a combination of user arg and const
+	VarImmLim            // pure imm argument provided by the users, up to maximum in op.ImmMax
 )
 
 const (
@@ -127,7 +130,7 @@ const (
 //
 // This function does not modify op.
 func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskShape, immType immShape,
-	opNoImm Operation) {
+	opNoImm Operation, immOpIdx string) {
 	if len(op.Out) > 1 {
 		panic(fmt.Errorf("simdgen only supports 1 output: %s", op))
 	}
@@ -136,8 +139,14 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS
 		outputReg = op.Out[0].AsmPos
 		if op.Out[0].Class == "vreg" {
 			shapeOut = OneVregOut
+			if op.Out[0].TreatLikeAScalarOfSize != nil {
+				shapeOut = OneVregOutScalar
+			}
 		} else if op.Out[0].Class == "greg" {
 			shapeOut = OneGregOut
+			if op.Out[0].TreatLikeAScalarOfSize != nil {
+				shapeOut = OneVregOutScalar
+			}
 		} else if op.Out[0].Class == "mask" {
 			shapeOut = OneKmaskOut
 		} else {
@@ -150,14 +159,19 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS
 		panic(fmt.Errorf("simdgen only supports 1 output: %s", op))
 	}
 	hasImm := false
+	immAsmPos := -1
 	maskCount := 0
 	hasVreg := false
+	hasListIn := false
 	for _, in := range op.In {
+		if in.ListNumber != nil {
+			hasListIn = true
+		}
 		if in.AsmPos == outputReg {
 			if shapeOut != OneVregOutAtIn && in.AsmPos == 0 && in.Class == "vreg" {
 				shapeOut = OneVregOutAtIn
-			} else {
-				panic(fmt.Errorf("simdgen only support output and input sharing the same position case of \"the first input is vreg and the only output\": %s", op))
+			} else if in.Class != "immediate" { // on arm64 immediate may be indexing into output register, e.g. INS greg, vreg[1]
+				panic(fmt.Errorf("simdgen only supports output and input sharing the same position case of \"the first input is vreg and the only output\": %s", op))
 			}
 		}
 		if in.Class == "immediate" {
@@ -167,9 +181,16 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS
 				panic(fmt.Errorf("simdgen only supports immediates of 8 bits: %s", op))
 			}
 			hasImm = true
+			immAsmPos = in.AsmPos
+			if immAsmPos == outputReg {
+				immOpIdx += "Out"
+			}
 		} else if in.Class == "mask" {
 			maskCount++
 		} else {
+			if immAsmPos == in.AsmPos {
+				immOpIdx += fmt.Sprintf("In%d", in.AsmPos)
+			}
 			hasVreg = true
 		}
 	}
@@ -177,6 +198,9 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS
 
 	removeImm := func(o *Operation) {
 		o.In = o.In[1:]
+		if o.In[0].Class == "immediate" {
+			o.In = o.In[1:] // e.g. arm64 INS Vn[0], Vd[imm]
+		}
 	}
 	if hasImm {
 		removeImm(&opNoImm)
@@ -187,7 +211,11 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS
 				immType = ConstImm
 			}
 		} else if op.In[0].ImmOffset != nil {
-			immType = VarImm
+			if op.In[0].ImmMax != nil {
+				immType = VarImmLim
+			} else {
+				immType = VarImm
+			}
 		} else {
 			panic(fmt.Errorf("simdgen requires imm to have at least one of ImmOffset or Const set: %s", op))
 		}
@@ -204,12 +232,15 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS
 			panic(fmt.Errorf("simdgen does not support immediates in pure mask operations: %s", op))
 		}
 		if hasVreg {
-			panic(fmt.Errorf("simdgen does not support more than 1 masks in non-pure mask operations: %s", op))
+			panic(fmt.Errorf("simdgen does not support more than 1 mask in non-pure mask operations: %s", op))
 		}
 		return false
 	}
 	if !hasImm && maskCount == 0 {
 		shapeIn = PureVregIn
+		if hasListIn {
+			shapeIn = VlistIn
+		}
 	} else if !hasImm && maskCount > 0 {
 		if maskCount == 1 {
 			shapeIn = OneKmaskIn
@@ -235,7 +266,7 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS
 
 // regShape returns a string representation of the register shape.
 func (op *Operation) regShape(mem memShape) (string, error) {
-	_, _, _, _, gOp := op.shape()
+	_, _, _, _, gOp, _ := op.shape()
 	var regInfo, fixedName string
 	var vRegInCnt, gRegInCnt, kMaskInCnt, vRegOutCnt, gRegOutCnt, kMaskOutCnt, memInCnt, memOutCnt int
 	for i, in := range gOp.In {
@@ -353,7 +384,7 @@ func (op *Operation) adjustAsm() {
 // does not return a vector, i.e., that returns a result in a general
 // register.  Currently there's only one family of Ops in Go's simd library
 // that does this (GetElem), and so this is specialized to work for that,
-// but the problem (mismatch betwen hardware register width and Go type
+// but the problem (mismatch between hardware register width and Go type
 // width) seems likely to recur if there are any other cases.
 func (op Operation) goNormalType() string {
 	if op.Go == "GetElem" {
@@ -392,12 +423,19 @@ func (op Operation) GoType() string {
 }
 
 // ImmName returns the name to use for an operation's immediate operand.
-// This can be overriden in the yaml with "name" on an operand,
+// This can be overridden in the yaml with "name" on an operand,
 // otherwise, for now, "constant"
 func (op Operation) ImmName() string {
 	return op.Op0Name("constant")
 }
 
+func (op Operation) ImmType() string {
+	if strings.Contains(op.Go, "Shift") || strings.Contains(op.Go, "Rotate") {
+		return "uint64"
+	}
+	return "uint8"
+}
+
 func (o Operand) OpName(s string) string {
 	if n := o.Name; n != nil {
 		return *n
@@ -481,26 +519,37 @@ func (op Operation) Op4NameAndType(s string) string {
 	return op.In[4].OpNameAndType(s)
 }
 
-var immClasses []string = []string{"BAD0Imm", "BAD1Imm", "op1Imm8", "op2Imm8", "op3Imm8", "op4Imm8"}
+var immClasses []string = []string{"BAD0Imm", "BAD1Imm", "op1Imm", "op2Imm", "op3Imm", "op4Imm"}
 var classes []string = []string{"BAD0", "op1", "op2", "op3", "op4"}
 
 // classifyOp returns a classification string, modified operation, and perhaps error based
 // on the stub and intrinsic shape for the operation.
-// The classification string is in the regular expression set "op[1234](Imm8)?(_<order>)?"
+// The classification string is in the regular expression set "op[1234](Imm(8)?)?(_<order>)?"
 // where the "<order>" suffix is optionally attached to the Operation in its input yaml.
 // The classification string is used to select a template or a clause of a template
 // for intrinsics declaration and the ssagen intrinisics glue code in the compiler.
 func classifyOp(op Operation) (string, Operation, error) {
-	_, _, _, immType, gOp := op.shape()
+	_, _, _, immType, gOp, _ := op.shape()
 
 	var class string
 
-	if immType == VarImm || immType == ConstVarImm {
+	if immType == VarImm || immType == VarImmLim || immType == ConstVarImm {
 		switch l := len(op.In); l {
 		case 1:
 			return "", op, fmt.Errorf("simdgen does not recognize this operation of only immediate input: %s", op)
 		case 2, 3, 4, 5:
-			class = immClasses[l]
+			if immType == VarImmLim {
+				if len(op.In)-len(gOp.In) == 2 {
+					class = immClasses[l-1] // arm64: do not account const 0 imm in INS Vn[0], Vd[imm]
+				} else {
+					class = immClasses[l] // known immediate maximum value
+				}
+			} else {
+				// No known maximum: default to full uint8 range via the "8"-suffixed
+				// template variants (e.g. "op2Imm"+"8" → "op2Imm8", mapping to
+				// opLen2Imm8 which hardcodes immMax=255).
+				class = immClasses[l] + "8"
+			}
 		default:
 			return "", op, fmt.Errorf("simdgen does not recognize this operation of input length %d: %s", len(op.In), op)
 		}
@@ -530,6 +579,9 @@ func checkVecAsScalar(op Operation) (idx int, err error) {
 			if idx == -1 {
 				idx = i
 				sSize = *o.TreatLikeAScalarOfSize
+				if sSize == 0 && CurrentArch().Arch == "arm64" {
+					sSize = *o.ElemBits // treating lane 0 as element-sized fp scalar, e.g. INS Vn[0], Vd[imm]
+				}
 			} else {
 				err = fmt.Errorf("simdgen only supports one TreatLikeAScalarOfSize in the arg list: %s", op)
 				return
@@ -559,6 +611,8 @@ func rewriteVecAsScalarRegInfo(op Operation, regInfo string) (string, error) {
 			regInfo = "v2fpv"
 		} else if regInfo == "v3kv" {
 			regInfo = "v2fpkv"
+		} else if regInfo == "v21ResultInArg0ImmOutIn1" {
+			regInfo = "vfpvResultInArg0ImmOutIn1"
 		} else {
 			return "", fmt.Errorf("simdgen does not recognize uses of treatLikeAScalarOfSize with op regShape %s in op: %s", regInfo, op)
 		}
@@ -611,6 +665,9 @@ func (op Operation) GenericName() string {
 		}
 	}
 	if op.In[0].Class == "immediate" {
+		if op.In[1].Class == "immediate" {
+			return op.Go + *op.In[2].Go // e.g. arm64 INS Vn[0], Vd[imm]
+		}
 		return op.Go + *op.In[1].Go
 	}
 	return op.Go + *op.In[0].Go
@@ -623,7 +680,7 @@ func (op Operation) GenericName() string {
 func dedupGodef(ops []Operation) ([]Operation, error) {
 	seen := map[string][]Operation{}
 	for _, op := range ops {
-		_, _, _, _, gOp := op.shape()
+		_, _, _, _, gOp, _ := op.shape()
 
 		gN := gOp.GenericName()
 		seen[gN] = append(seen[gN], op)
@@ -689,10 +746,15 @@ func copyConstImm(ops []Operation) error {
 		if op.ConstImm == nil {
 			continue
 		}
-		_, _, _, immType, _ := op.shape()
+		_, _, _, immType, _, _ := op.shape()
 
 		if immType == ConstImm || immType == ConstVarImm {
 			op.In[0].Const = op.ConstImm
+			// If the immediate operand is tagged with name:"@", it is fully constant;
+			// clear ImmOffset to ensure this is treated as ConstImm (no aux field).
+			if op.In[0].Name != nil && *op.In[0].Name == "@" {
+				op.In[0].ImmOffset = nil
+			}
 		}
 		// Otherwise, just not port it - e.g. {VPCMP[BWDQ] imm=0} and {VPCMPEQ[BWDQ]} are
 		// the same operations "Equal", [dedupgodef] should be able to distinguish them.
@@ -727,14 +789,33 @@ func overwrite(ops []Operation) error {
 			*op[idx].Lanes = *op[idx].Bits / *op[idx].ElemBits
 			*op[idx].Go = fmt.Sprintf("%s%dx%d", capitalizeFirst(*op[idx].Base), *op[idx].ElemBits, *op[idx].Lanes)
 		}
-		if op[idx].OverwriteClass != nil {
+		if CurrentArch().Arch == "arm64" && op[idx].OverwriteClass != nil && *op[idx].OverwriteClass == "greg" {
+			if op[idx].OverwriteBase == nil {
+				panic(fmt.Errorf("simdgen: [OverwriteClass] must be set together with [OverwriteBase]: %s", op[idx]))
+			}
+			oBase := *op[idx].OverwriteBase
+			oClass := *op[idx].OverwriteClass
+			if oBase != "float" {
+				panic(fmt.Errorf("simdgen: [Class] overwrite must set [OverwriteBase] to float: %s", op[idx]))
+			}
+			if op[idx].Class != "vreg" {
+				panic(fmt.Errorf("simdgen: [Class] overwrite must be overwriting [Class] from vreg: %s", op[idx]))
+			}
+			// The low lane of vreg (with other lanes zeroed) also represents a regular floating point greg.
+			// This is supposed to be used only by special instructions like float GetElem
+			// and floating point vector reduction across-lanes to lane 0 like FMINNMV.
+			hasClassOverwrite = true
+			*op[idx].Base = oBase
+			op[idx].Class = oClass
+			*op[idx].Go = fmt.Sprintf("float%d", *op[idx].ElemBits)
+		} else if op[idx].OverwriteClass != nil {
 			if op[idx].OverwriteBase == nil {
 				panic(fmt.Errorf("simdgen: [OverwriteClass] must be set together with [OverwriteBase]: %s", op[idx]))
 			}
 			oBase := *op[idx].OverwriteBase
 			oClass := *op[idx].OverwriteClass
 			if oClass != "mask" {
-				panic(fmt.Errorf("simdgen: [Class] overwrite only supports overwritting to mask: %s", op[idx]))
+				panic(fmt.Errorf("simdgen: [Class] overwrite only supports overwriting to mask: %s", op[idx]))
 			}
 			if oBase != "int" {
 				panic(fmt.Errorf("simdgen: [Class] overwrite must set [OverwriteBase] to int: %s", op[idx]))
@@ -753,6 +834,12 @@ func overwrite(ops []Operation) error {
 				*op[idx].Go = strings.ReplaceAll(*op[idx].Go, *op[idx].Base, oBase)
 			}
 			*op[idx].Base = oBase
+		} else if op[idx].OverwriteBits != nil {
+			if op[idx].Class != "greg" {
+				panic(fmt.Errorf("simdgen: [OverwriteBits] is only supported for greg int: %s", op[idx]))
+			}
+			*op[idx].Bits = *op[idx].OverwriteBits
+			*op[idx].Go = fmt.Sprintf("%s%d", *op[idx].Base, *op[idx].Bits)
 		}
 		return nil
 	}
@@ -828,17 +915,27 @@ func reportXEDInconsistency(ops []Operation) error {
 }
 
 func (o *Operation) hasMaskedMerging(maskType maskShape, outType outShape) bool {
+	if o.SpecialLower != nil {
+		// asmRule and argsMatchRule/earlyMatchRule should not affect masked merging
+		ok, _, _ := parseAsmRule(*o.SpecialLower)
+		if !ok {
+			ok, _, _ = parseArgsMatchRule(*o.SpecialLower)
+			if !ok {
+				return false
+			}
+		}
+	}
 	// BLEND and VMOVDQU are not user-facing ops so we should filter them out.
-	return o.OperandOrder == nil && o.SpecialLower == nil && maskType == OneMask && outType == OneVregOut &&
+	return o.OperandOrder == nil && maskType == OneMask && outType == OneVregOut &&
 		len(o.InVariant) == 1 && !strings.Contains(o.Asm, "BLEND") && !strings.Contains(o.Asm, "VMOVDQU")
 }
 
-func getVbcstData(s string) (feat1Match, feat2Match string) {
-	_, err := fmt.Sscanf(s, "feat1=%[^;];feat2=%s", &feat1Match, &feat2Match)
-	if err != nil {
-		panic(err)
+func getVbcstData(s string) (string, string) {
+	feat1, feat2, found := strings.Cut(s, ";")
+	if !found || !strings.HasPrefix(feat1, "feat1=") || !strings.HasPrefix(feat2, "feat2=") {
+		panic(fmt.Sprintf("unexpected format for vbcst data: %s", s))
 	}
-	return
+	return strings.TrimPrefix(feat1, "feat1="), strings.TrimPrefix(feat2, "feat2=")
 }
 
 func (o Operation) String() string {
@@ -848,3 +945,39 @@ func (o Operation) String() string {
 func (op Operand) String() string {
 	return pprints(op)
 }
+
+// hiHalfOpName constructs the SSA machine op name for a hi-half "2" variant.
+// For example: hiHalfAsm="VSHRN2", arrangement="4S" → "VSHRN2_4S".
+func hiHalfOpName(hiHalfAsm string, gOp Operation) string {
+	return hiHalfAsm + "_" + *gOp.Arrangement
+}
+
+// hiHalfRegShape derives the regShape for hi-half "2" variant ops from the base regShape.
+// For narrow "2": the "2" variant has an extra destination input (resultInArg0),
+// so vreg input count increases by 1 (e.g., "v11Imm" → "v21Imm").
+// For long "2": same shape as base.
+func hiHalfRegShape2(baseRegShape string, kind string) string {
+	if kind == "narrow" {
+		if len(baseRegShape) > 2 && baseRegShape[0] == 'v' {
+			inCnt := int(baseRegShape[1] - '0')
+			outCnt := int(baseRegShape[2] - '0')
+			rest := baseRegShape[3:]
+			return fmt.Sprintf("v%d%d%s", inCnt+1, outCnt, rest)
+		}
+		panic(fmt.Sprintf("hiHalfRegShape2: unexpected regShape %q for narrow kind", baseRegShape))
+	}
+	return baseRegShape
+}
+
+// hiHalfLoweringRegShape computes the lowering dispatch regShape for a hi-half operation.
+// Base ops get a suffix like "Narrow" or "Long" appended to their standard regShape.
+// "2" variant ops get a derived regShape + "2" suffix.
+func hiHalfLoweringRegShape(baseRegShape string, kind string, isVariant2 bool) string {
+	suffix := capitalizeFirst(kind)
+	if isVariant2 {
+		suffix += "2"
+		// For narrow "2", the regShape changes (extra vreg input for resultInArg0)
+		baseRegShape = hiHalfRegShape2(baseRegShape, kind)
+	}
+	return baseRegShape + suffix
+}

package/bin/go/src/simd/archsimd/_gen/simdgen/go_amd64.yaml

@@ -0,0 +1 @@
+!import ops/*/go_amd64.yaml

package/bin/go/src/simd/archsimd/_gen/simdgen/go_arm64.yaml

@@ -0,0 +1 @@
+!import ops/*/go_arm64.yaml

package/bin/go/src/simd/archsimd/_gen/simdgen/godefs.go

@@ -7,6 +7,7 @@ package main
 import (
 	"fmt"
 	"log"
+	"math/rand/v2"
 	"regexp"
 	"slices"
 	"strconv"
@@ -47,10 +48,15 @@ type rawOperation struct {
 
 	GoArch       string  // GOARCH for this definition
 	Asm          string  // Assembly mnemonic
+	Arrangement  *string // optional Arrangement for ARM64 SIMD operations (e.g., "4S", "2D")
 	OperandOrder *string // optional Operand order for better Go declarations
 	// Optional tag to indicate this operation is paired with special generic->machine ssa lowering rules.
 	// Should be paired with special templates in gen_simdrules.go
 	SpecialLower *string
+	// HiHalfAsm is the assembly mnemonic for the hi-half "2" variant of this operation,
+	// specified in go_arm64.yaml (e.g., "VSHRN2", "VUMULL2").
+	// When non-nil, simdgen generates the "2" variant machine op and folding rules.
+	HiHalfAsm *string
 
 	In              []Operand // Parameters
 	InVariant       []Operand // Optional parameters
@@ -98,6 +104,38 @@ func (o *Operation) SkipMaskedMethod() bool {
 	return false
 }
 
+// hiHalfKind returns "narrow" or "long" based on whether the operation narrows or widens its elements.
+// Returns "" if HiHalfAsm is nil or classification is ambiguous.
+func (o *Operation) hiHalfKind() string {
+	if o.HiHalfAsm == nil {
+		return ""
+	}
+	// Find the first vreg input and the first vreg output to compare elemBits.
+	var inElemBits, outElemBits *int
+	for i := range o.In {
+		if o.In[i].Class == "vreg" && o.In[i].ElemBits != nil {
+			inElemBits = o.In[i].ElemBits
+			break
+		}
+	}
+	for i := range o.Out {
+		if o.Out[i].Class == "vreg" && o.Out[i].ElemBits != nil {
+			outElemBits = o.Out[i].ElemBits
+			break
+		}
+	}
+	if inElemBits == nil || outElemBits == nil {
+		return ""
+	}
+	if *outElemBits < *inElemBits {
+		return "narrow"
+	}
+	if *outElemBits > *inElemBits {
+		return "long"
+	}
+	return ""
+}
+
 var reForName = regexp.MustCompile(`\bNAME\b`)
 
 func (o *Operation) DecodeUnified(v *unify.Value) error {
@@ -135,13 +173,34 @@ func (o *Operation) DecodeUnified(v *unify.Value) error {
 
 	o.In = append(o.rawOperation.In, o.rawOperation.InVariant...)
 
+	// For operations that read only the lower half of input registers (indicated by hiHalfAsm),
+	// add a doc note showing the compositional pattern for the upper half.
+	if o.rawOperation.HiHalfAsm != nil && o.hiHalfKind() == "long" {
+		// Count vector-register inputs (exclude immediates/scalars).
+		vregIns := 0
+		for _, in := range o.In {
+			if in.Class == "vreg" {
+				vregIns++
+			}
+		}
+		// note this is arm64-specific
+		switch vregIns {
+		case 2:
+			// Binary: MulLong, AddLong, SubLong, etc.
+			o.Documentation += "\n// For the high-indexed elements, use HiToLo:\n//\n//\tx.HiToLo()." + o.Go + "(y.HiToLo())"
+		case 1:
+			// Unary: ShiftLeftLongConst, etc.
+			o.Documentation += "\n// For the high-indexed elements, use HiToLo:\n//\n//\tx.HiToLo()." + o.Go + "(...)"
+		}
+	}
+
 	// For down conversions, the high elements are zeroed if the result has more elements.
 	// TODO: we should encode this logic in the YAML file, instead of hardcoding it here.
 	if len(o.In) > 0 && len(o.Out) > 0 {
 		inLanes := o.In[0].Lanes
 		outLanes := o.Out[0].Lanes
 		if inLanes != nil && outLanes != nil && *inLanes < *outLanes {
-			if (strings.Contains(o.Go, "Saturate") || strings.Contains(o.Go, "Truncate")) &&
+			if (strings.Contains(o.Go, "Saturate") || strings.Contains(o.Go, "TruncTo")) &&
 				!strings.Contains(o.Go, "Concat") {
 				o.Documentation += "\n// Results are packed to low elements in the returned vector, its upper elements are zeroed."
 			}
@@ -189,7 +248,12 @@ func machineOpName(maskType maskShape, gOp Operation) string {
 	if maskType == OneMask {
 		asm += "Masked"
 	}
-	asm = fmt.Sprintf("%s%d", asm, gOp.VectorWidth())
+	// For ARM64, use arrangement to create distinct SSA op names
+	if gOp.Arrangement != nil && *gOp.Arrangement != "" {
+		asm = fmt.Sprintf("%s%s", asm, *gOp.Arrangement)
+	} else {
+		asm = fmt.Sprintf("%s%d", asm, gOp.VectorWidth())
+	}
 	if gOp.SSAVariant != nil {
 		asm += *gOp.SSAVariant
 	}
@@ -276,6 +340,9 @@ func compareOperands(x, y *Operand) int {
 		if c := compareIntPointers(x.Bits, y.Bits); c != 0 {
 			return c
 		}
+		if c := compareIntPointers(x.ListNumber, y.ListNumber); c != 0 {
+			return c
+		}
 		return 0
 	}
 }
@@ -296,6 +363,7 @@ type Operand struct {
 	// The compiler will right-shift the user-passed value by ImmOffset and set it as the AuxInt
 	// field of the operation.
 	ImmOffset *string
+	ImmMax    *int    // optional maximum immediate, also highest case in immediate jump table
 	Name      *string // optional name in the Go intrinsic declaration
 	Lanes     *int    // *Lanes equals Bits/ElemBits except for scalars, when *Lanes == 1
 	// TreatLikeAScalarOfSize means only the lower $TreatLikeAScalarOfSize bits of the vector
@@ -312,8 +380,15 @@ type Operand struct {
 	// because Intel's XED data is inconsistent. e.g. AVX512 VPMADDUBSW marks its operand
 	// elemBits 16, which should be 8.
 	OverwriteElementBits *int
+	// For greg only, specifically VPEXTR[BW], their results are specified by Intel as 32 bits,
+	// but they really are 8/16 bits.
+	OverwriteBits *int
 	// FixedReg is the name of the fixed registers
 	FixedReg *string
+	// If non-nil, marks this vreg as a register list operand (for TBL/TBX).
+	// Currently only list number 0 is supported (we might need to teach regalloc handle register lists
+	// to support more than one register in the list).
+	ListNumber *int
 }
 
 // isDigit returns true if the byte is an ASCII digit.
@@ -357,6 +432,10 @@ func compareNatural(s1, s2 string) int {
 			if num1 > num2 {
 				return 1
 			}
+			// "1" < "01".  Don't expect it in simdgen, but just in case.
+			if ln1, ln2 := i-numStart1, j-numStart2; ln1 != ln2 {
+				return ln1 - ln2
+			}
 			// If numbers are equal, continue to the next segment.
 		} else {
 			// Non-digit comparison.
@@ -375,8 +454,10 @@ func compareNatural(s1, s2 string) int {
 	return strings.Compare(s1, s2)
 }
 
-const generatedHeader = `// Code generated by 'simdgen -o godefs -goroot $GOROOT -xedPath $XED_PATH go.yaml types.yaml categories.yaml'; DO NOT EDIT.
-`
+// generatedHeader returns the architecture-specific header for generated files.
+func generatedHeader() string {
+	return CurrentArch().GeneratedHeader
+}
 
 func writeGoDefs(path string, cl unify.Closure) error {
 	// TODO: Merge operations with the same signature but multiple
@@ -397,6 +478,11 @@ func writeGoDefs(path string, cl unify.Closure) error {
 		op.adjustAsm()
 		ops = append(ops, op)
 	}
+
+	rand.Shuffle(len(ops), func(i, j int) {
+		ops[i], ops[j] = ops[j], ops[i]
+	})
+
 	slices.SortFunc(ops, compareOperations)
 	// The parsed XED data might contain duplicates, like
 	// 512 bits VPADDP.
@@ -404,7 +490,7 @@ func writeGoDefs(path string, cl unify.Closure) error {
 	slices.SortFunc(deduped, compareOperations)
 
 	if *Verbose {
-		log.Printf("dedup len: %d\n", len(ops))
+		log.Printf("dedup len: %d, ops len: %d\n", len(deduped), len(ops))
 	}
 	var err error
 	if err = overwrite(deduped); err != nil {
@@ -432,18 +518,30 @@ func writeGoDefs(path string, cl unify.Closure) error {
 		log.Printf("dedup len: %d\n", len(deduped))
 	}
 	reportXEDInconsistency(deduped)
+
+	// Sorting again, just in case.
+	slices.SortFunc(deduped, compareOperations)
+
 	typeMap := parseSIMDTypes(deduped)
 
-	formatWriteAndClose(writeSIMDTypes(typeMap), path, "src/"+simdPackage+"/types_amd64.go")
-	formatWriteAndClose(writeSIMDFeatures(deduped), path, "src/"+simdPackage+"/cpu.go")
-	f, fI := writeSIMDStubs(deduped, typeMap)
-	formatWriteAndClose(f, path, "src/"+simdPackage+"/ops_amd64.go")
-	formatWriteAndClose(fI, path, "src/"+simdPackage+"/ops_internal_amd64.go")
-	formatWriteAndClose(writeSIMDIntrinsics(deduped, typeMap), path, "src/cmd/compile/internal/ssagen/simdintrinsics.go")
-	formatWriteAndClose(writeSIMDGenericOps(deduped), path, "src/cmd/compile/internal/ssa/_gen/simdgenericOps.go")
-	formatWriteAndClose(writeSIMDMachineOps(deduped), path, "src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go")
-	formatWriteAndClose(writeSIMDSSA(deduped), path, "src/cmd/compile/internal/amd64/simdssa.go")
-	writeAndClose(writeSIMDRules(deduped).Bytes(), path, "src/cmd/compile/internal/ssa/_gen/simdAMD64.rules")
+	archInfo := CurrentArch()
+	archLower := archInfo.Arch
+	archUpper := archInfo.ArchUpper
+
+	formatWriteAndClose(writeSIMDTypes(typeMap), path, "src/"+simdPackage+"/types_"+archLower+".go")
+	// TODO: Enable CPU feature generation for non-x86 architectures.
+	if archLower == "amd64" {
+		formatWriteAndClose(writeSIMDFeatures(deduped), path, "src/"+simdPackage+"/cpu.go")
+	}
+	f, fI := writeSIMDStubs(deduped, typeMap, archLower == "amd64")
+	formatWriteAndClose(f, path, "src/"+simdPackage+"/ops_"+archLower+".go")
+	formatWriteAndClose(fI, path, "src/"+simdPackage+"/ops_internal_"+archLower+".go")
+	formatWriteAndClose(writeSIMDIntrinsics(deduped, typeMap), path, "src/cmd/compile/internal/ssagen/simd"+archUpper+"intrinsics.go")
+	const simdGenericOpsFile = "src/cmd/compile/internal/ssa/_gen/simdgenericOps.go"
+	formatWriteAndClose(writeSIMDGenericOps(deduped, path+"/"+simdGenericOpsFile), path, simdGenericOpsFile)
+	formatWriteAndClose(writeSIMDMachineOps(deduped), path, "src/cmd/compile/internal/ssa/_gen/simd"+archUpper+"ops.go")
+	formatWriteAndClose(writeSIMDSSA(deduped), path, "src/cmd/compile/internal/"+archLower+"/simdssa.go")
+	writeAndClose(writeSIMDRules(deduped).Bytes(), path, "src/cmd/compile/internal/ssa/_gen/simd"+archUpper+".rules")
 
 	return nil
 }