laag-libjpeg-turbo 1.5.3.0

data/vendor/github.com/libjpeg-turbo/libjpeg-turbo/simd/jdsample-altivec.c

@@ -0,0 +1,392 @@
+/*
+ * AltiVec optimizations for libjpeg-turbo
+ *
+ * Copyright (C) 2015, D. R. Commander.  All Rights Reserved.
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty.  In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ *    claim that you wrote the original software. If you use this software
+ *    in a product, an acknowledgment in the product documentation would be
+ *    appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ *    misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+/* CHROMA UPSAMPLING */
+
+#include "jsimd_altivec.h"
+
+
+void
+jsimd_h2v1_fancy_upsample_altivec (int max_v_samp_factor,
+                                   JDIMENSION downsampled_width,
+                                   JSAMPARRAY input_data,
+                                   JSAMPARRAY *output_data_ptr)
+{
+  JSAMPARRAY output_data = *output_data_ptr;
+  JSAMPROW inptr, outptr;
+  int inrow, incol;
+
+  __vector unsigned char this0, last0, p_last0, next0 = {0}, p_next0,
+    out;
+  __vector short this0e, this0o, this0l, this0h, last0l, last0h,
+    next0l, next0h, outle, outhe, outlo, outho;
+
+  /* Constants */
+  __vector unsigned char pb_zero = { __16X(0) }, pb_three = { __16X(3) },
+    last_index_col0 = {0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14},
+    last_index = {15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30},
+    next_index = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16},
+    next_index_lastcol = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,15},
+#if __BIG_ENDIAN__
+    merge_pack_index = {1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31};
+#else
+    merge_pack_index = {0,16,2,18,4,20,6,22,8,24,10,26,12,28,14,30};
+#endif
+  __vector short pw_one = { __8X(1) }, pw_two = { __8X(2) };
+
+  for (inrow = 0; inrow < max_v_samp_factor; inrow++) {
+    inptr = input_data[inrow];
+    outptr = output_data[inrow];
+
+    if (downsampled_width & 15)
+      inptr[downsampled_width] = inptr[downsampled_width - 1];
+
+    this0 = vec_ld(0, inptr);
+    p_last0 = vec_perm(this0, this0, last_index_col0);
+    last0 = this0;
+
+    for (incol = downsampled_width; incol > 0;
+         incol -= 16, inptr += 16, outptr += 32) {
+
+      if (downsampled_width - incol > 0) {
+        p_last0 = vec_perm(last0, this0, last_index);
+        last0 = this0;
+      }
+
+      if (incol <= 16)
+        p_next0 = vec_perm(this0, this0, next_index_lastcol);
+      else {
+        next0 = vec_ld(16, inptr);
+        p_next0 = vec_perm(this0, next0, next_index);
+      }
+
+      this0e = (__vector short)vec_mule(this0, pb_three);
+      this0o = (__vector short)vec_mulo(this0, pb_three);
+      this0l = vec_mergeh(this0e, this0o);
+      this0h = vec_mergel(this0e, this0o);
+
+      last0l = (__vector short)VEC_UNPACKHU(p_last0);
+      last0h = (__vector short)VEC_UNPACKLU(p_last0);
+      last0l = vec_add(last0l, pw_one);
+
+      next0l = (__vector short)VEC_UNPACKHU(p_next0);
+      next0h = (__vector short)VEC_UNPACKLU(p_next0);
+      next0l = vec_add(next0l, pw_two);
+
+      outle = vec_add(this0l, last0l);
+      outlo = vec_add(this0l, next0l);
+      outle = vec_sr(outle, (__vector unsigned short)pw_two);
+      outlo = vec_sr(outlo, (__vector unsigned short)pw_two);
+
+      out = vec_perm((__vector unsigned char)outle,
+                     (__vector unsigned char)outlo, merge_pack_index);
+      vec_st(out, 0, outptr);
+
+      if (incol > 8) {
+        last0h = vec_add(last0h, pw_one);
+        next0h = vec_add(next0h, pw_two);
+
+        outhe = vec_add(this0h, last0h);
+        outho = vec_add(this0h, next0h);
+        outhe = vec_sr(outhe, (__vector unsigned short)pw_two);
+        outho = vec_sr(outho, (__vector unsigned short)pw_two);
+
+        out = vec_perm((__vector unsigned char)outhe,
+                       (__vector unsigned char)outho, merge_pack_index);
+        vec_st(out, 16, outptr);
+      }
+
+      this0 = next0;
+    }
+  }
+}
+
+
+void
+jsimd_h2v2_fancy_upsample_altivec (int max_v_samp_factor,
+                                   JDIMENSION downsampled_width,
+                                   JSAMPARRAY input_data,
+                                   JSAMPARRAY *output_data_ptr)
+{
+  JSAMPARRAY output_data = *output_data_ptr;
+  JSAMPROW inptr_1, inptr0, inptr1, outptr0, outptr1;
+  int inrow, outrow, incol;
+
+  __vector unsigned char this_1, this0, this1, out;
+  __vector short this_1l, this_1h, this0l, this0h, this1l, this1h,
+    lastcolsum_1h, lastcolsum1h,
+    p_lastcolsum_1l, p_lastcolsum_1h, p_lastcolsum1l, p_lastcolsum1h,
+    thiscolsum_1l, thiscolsum_1h, thiscolsum1l, thiscolsum1h,
+    nextcolsum_1l = {0}, nextcolsum_1h = {0},
+    nextcolsum1l = {0}, nextcolsum1h = {0},
+    p_nextcolsum_1l, p_nextcolsum_1h, p_nextcolsum1l, p_nextcolsum1h,
+    tmpl, tmph, outle, outhe, outlo, outho;
+
+  /* Constants */
+  __vector unsigned char pb_zero = { __16X(0) },
+    last_index_col0 = {0,1,0,1,2,3,4,5,6,7,8,9,10,11,12,13},
+    last_index={14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29},
+    next_index = {2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17},
+    next_index_lastcol = {2,3,4,5,6,7,8,9,10,11,12,13,14,15,14,15},
+#if __BIG_ENDIAN__
+    merge_pack_index = {1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31};
+#else
+    merge_pack_index = {0,16,2,18,4,20,6,22,8,24,10,26,12,28,14,30};
+#endif
+  __vector short pw_zero = { __8X(0) }, pw_three = { __8X(3) },
+    pw_seven = { __8X(7) }, pw_eight = { __8X(8) };
+  __vector unsigned short pw_four = { __8X(4) };
+
+  for (inrow = 0, outrow = 0; outrow < max_v_samp_factor; inrow++) {
+
+    inptr_1 = input_data[inrow - 1];
+    inptr0 = input_data[inrow];
+    inptr1 = input_data[inrow + 1];
+    outptr0 = output_data[outrow++];
+    outptr1 = output_data[outrow++];
+
+    if (downsampled_width & 15) {
+      inptr_1[downsampled_width] = inptr_1[downsampled_width - 1];
+      inptr0[downsampled_width] = inptr0[downsampled_width - 1];
+      inptr1[downsampled_width] = inptr1[downsampled_width - 1];
+    }
+
+    this0 = vec_ld(0, inptr0);
+    this0l = (__vector short)VEC_UNPACKHU(this0);
+    this0h = (__vector short)VEC_UNPACKLU(this0);
+    this0l = vec_mladd(this0l, pw_three, pw_zero);
+    this0h = vec_mladd(this0h, pw_three, pw_zero);
+
+    this_1 = vec_ld(0, inptr_1);
+    this_1l = (__vector short)VEC_UNPACKHU(this_1);
+    this_1h = (__vector short)VEC_UNPACKLU(this_1);
+    thiscolsum_1l = vec_add(this0l, this_1l);
+    thiscolsum_1h = vec_add(this0h, this_1h);
+    lastcolsum_1h = thiscolsum_1h;
+    p_lastcolsum_1l = vec_perm(thiscolsum_1l, thiscolsum_1l, last_index_col0);
+    p_lastcolsum_1h = vec_perm(thiscolsum_1l, thiscolsum_1h, last_index);
+
+    this1 = vec_ld(0, inptr1);
+    this1l = (__vector short)VEC_UNPACKHU(this1);
+    this1h = (__vector short)VEC_UNPACKLU(this1);
+    thiscolsum1l = vec_add(this0l, this1l);
+    thiscolsum1h = vec_add(this0h, this1h);
+    lastcolsum1h = thiscolsum1h;
+    p_lastcolsum1l = vec_perm(thiscolsum1l, thiscolsum1l, last_index_col0);
+    p_lastcolsum1h = vec_perm(thiscolsum1l, thiscolsum1h, last_index);
+
+    for (incol = downsampled_width; incol > 0;
+         incol -= 16, inptr_1 += 16, inptr0 += 16, inptr1 += 16,
+         outptr0 += 32, outptr1 += 32) {
+
+      if (downsampled_width - incol > 0) {
+        p_lastcolsum_1l = vec_perm(lastcolsum_1h, thiscolsum_1l, last_index);
+        p_lastcolsum_1h = vec_perm(thiscolsum_1l, thiscolsum_1h, last_index);
+        p_lastcolsum1l = vec_perm(lastcolsum1h, thiscolsum1l, last_index);
+        p_lastcolsum1h = vec_perm(thiscolsum1l, thiscolsum1h, last_index);
+        lastcolsum_1h = thiscolsum_1h;  lastcolsum1h = thiscolsum1h;
+      }
+
+      if (incol <= 16) {
+        p_nextcolsum_1l = vec_perm(thiscolsum_1l, thiscolsum_1h, next_index);
+        p_nextcolsum_1h = vec_perm(thiscolsum_1h, thiscolsum_1h,
+                                   next_index_lastcol);
+        p_nextcolsum1l = vec_perm(thiscolsum1l, thiscolsum1h, next_index);
+        p_nextcolsum1h = vec_perm(thiscolsum1h, thiscolsum1h,
+                                  next_index_lastcol);
+      } else {
+        this0 = vec_ld(16, inptr0);
+        this0l = (__vector short)VEC_UNPACKHU(this0);
+        this0h = (__vector short)VEC_UNPACKLU(this0);
+        this0l = vec_mladd(this0l, pw_three, pw_zero);
+        this0h = vec_mladd(this0h, pw_three, pw_zero);
+
+        this_1 = vec_ld(16, inptr_1);
+        this_1l = (__vector short)VEC_UNPACKHU(this_1);
+        this_1h = (__vector short)VEC_UNPACKLU(this_1);
+        nextcolsum_1l = vec_add(this0l, this_1l);
+        nextcolsum_1h = vec_add(this0h, this_1h);
+        p_nextcolsum_1l = vec_perm(thiscolsum_1l, thiscolsum_1h, next_index);
+        p_nextcolsum_1h = vec_perm(thiscolsum_1h, nextcolsum_1l, next_index);
+
+        this1 = vec_ld(16, inptr1);
+        this1l = (__vector short)VEC_UNPACKHU(this1);
+        this1h = (__vector short)VEC_UNPACKLU(this1);
+        nextcolsum1l = vec_add(this0l, this1l);
+        nextcolsum1h = vec_add(this0h, this1h);
+        p_nextcolsum1l = vec_perm(thiscolsum1l, thiscolsum1h, next_index);
+        p_nextcolsum1h = vec_perm(thiscolsum1h, nextcolsum1l, next_index);
+      }
+
+      /* Process the upper row */
+
+      tmpl = vec_mladd(thiscolsum_1l, pw_three, pw_zero);
+      outle = vec_add(tmpl, p_lastcolsum_1l);
+      outle = vec_add(outle, pw_eight);
+      outle = vec_sr(outle, pw_four);
+
+      outlo = vec_add(tmpl, p_nextcolsum_1l);
+      outlo = vec_add(outlo, pw_seven);
+      outlo = vec_sr(outlo, pw_four);
+
+      out = vec_perm((__vector unsigned char)outle,
+                     (__vector unsigned char)outlo, merge_pack_index);
+      vec_st(out, 0, outptr0);
+
+      if (incol > 8) {
+        tmph = vec_mladd(thiscolsum_1h, pw_three, pw_zero);
+        outhe = vec_add(tmph, p_lastcolsum_1h);
+        outhe = vec_add(outhe, pw_eight);
+        outhe = vec_sr(outhe, pw_four);
+
+        outho = vec_add(tmph, p_nextcolsum_1h);
+        outho = vec_add(outho, pw_seven);
+        outho = vec_sr(outho, pw_four);
+
+        out = vec_perm((__vector unsigned char)outhe,
+                       (__vector unsigned char)outho, merge_pack_index);
+        vec_st(out, 16, outptr0);
+      }
+
+      /* Process the lower row */
+
+      tmpl = vec_mladd(thiscolsum1l, pw_three, pw_zero);
+      outle = vec_add(tmpl, p_lastcolsum1l);
+      outle = vec_add(outle, pw_eight);
+      outle = vec_sr(outle, pw_four);
+
+      outlo = vec_add(tmpl, p_nextcolsum1l);
+      outlo = vec_add(outlo, pw_seven);
+      outlo = vec_sr(outlo, pw_four);
+
+      out = vec_perm((__vector unsigned char)outle,
+                     (__vector unsigned char)outlo, merge_pack_index);
+      vec_st(out, 0, outptr1);
+
+      if (incol > 8) {
+        tmph = vec_mladd(thiscolsum1h, pw_three, pw_zero);
+        outhe = vec_add(tmph, p_lastcolsum1h);
+        outhe = vec_add(outhe, pw_eight);
+        outhe = vec_sr(outhe, pw_four);
+
+        outho = vec_add(tmph, p_nextcolsum1h);
+        outho = vec_add(outho, pw_seven);
+        outho = vec_sr(outho, pw_four);
+
+        out = vec_perm((__vector unsigned char)outhe,
+                       (__vector unsigned char)outho, merge_pack_index);
+        vec_st(out, 16, outptr1);
+      }
+
+      thiscolsum_1l = nextcolsum_1l;  thiscolsum_1h = nextcolsum_1h;
+      thiscolsum1l = nextcolsum1l;  thiscolsum1h = nextcolsum1h;
+    }
+  }
+}
+
+
+/* These are rarely used (mainly just for decompressing YCCK images) */
+
+void
+jsimd_h2v1_upsample_altivec (int max_v_samp_factor,
+                             JDIMENSION output_width,
+                             JSAMPARRAY input_data,
+                             JSAMPARRAY *output_data_ptr)
+{
+  JSAMPARRAY output_data = *output_data_ptr;
+  JSAMPROW inptr, outptr;
+  int inrow, incol;
+
+  __vector unsigned char in, inl, inh;
+
+  for (inrow = 0; inrow < max_v_samp_factor; inrow++) {
+    inptr = input_data[inrow];
+    outptr = output_data[inrow];
+
+    for (incol = (output_width + 31) & (~31); incol > 0;
+         incol -= 64, inptr += 32, outptr += 64) {
+
+      in = vec_ld(0, inptr);
+      inl = vec_mergeh(in, in);
+      inh = vec_mergel(in, in);
+
+      vec_st(inl, 0, outptr);
+      vec_st(inh, 16, outptr);
+
+      if (incol > 32) {
+        in = vec_ld(16, inptr);
+        inl = vec_mergeh(in, in);
+        inh = vec_mergel(in, in);
+
+        vec_st(inl, 32, outptr);
+        vec_st(inh, 48, outptr);
+      }
+    }
+  }
+}
+
+
+void
+jsimd_h2v2_upsample_altivec (int max_v_samp_factor,
+                             JDIMENSION output_width,
+                             JSAMPARRAY input_data,
+                             JSAMPARRAY *output_data_ptr)
+{
+  JSAMPARRAY output_data = *output_data_ptr;
+  JSAMPROW inptr, outptr0, outptr1;
+  int inrow, outrow, incol;
+
+  __vector unsigned char in, inl, inh;
+
+  for (inrow = 0, outrow = 0; outrow < max_v_samp_factor; inrow++) {
+
+    inptr = input_data[inrow];
+    outptr0 = output_data[outrow++];
+    outptr1 = output_data[outrow++];
+
+    for (incol = (output_width + 31) & (~31); incol > 0;
+         incol -= 64, inptr += 32, outptr0 += 64, outptr1 += 64) {
+
+      in = vec_ld(0, inptr);
+      inl = vec_mergeh(in, in);
+      inh = vec_mergel(in, in);
+
+      vec_st(inl, 0, outptr0);
+      vec_st(inl, 0, outptr1);
+
+      vec_st(inh, 16, outptr0);
+      vec_st(inh, 16, outptr1);
+
+      if (incol > 32) {
+        in = vec_ld(16, inptr);
+        inl = vec_mergeh(in, in);
+        inh = vec_mergel(in, in);
+
+        vec_st(inl, 32, outptr0);
+        vec_st(inl, 32, outptr1);
+
+        vec_st(inh, 48, outptr0);
+        vec_st(inh, 48, outptr1);
+      }
+    }
+  }
+}

data/vendor/github.com/libjpeg-turbo/libjpeg-turbo/simd/jdsample-mmx.asm

@@ -0,0 +1,736 @@
+;
+; jdsample.asm - upsampling (MMX)
+;
+; Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
+;
+; Based on the x86 SIMD extension for IJG JPEG library
+; Copyright (C) 1999-2006, MIYASAKA Masaru.
+; For conditions of distribution and use, see copyright notice in jsimdext.inc
+;
+; This file should be assembled with NASM (Netwide Assembler),
+; can *not* be assembled with Microsoft's MASM or any compatible
+; assembler (including Borland's Turbo Assembler).
+; NASM is available from http://nasm.sourceforge.net/ or
+; http://sourceforge.net/project/showfiles.php?group_id=6208
+;
+; [TAB8]
+
+%include "jsimdext.inc"
+
+; --------------------------------------------------------------------------
+        SECTION SEG_CONST
+
+        alignz  16
+        global  EXTN(jconst_fancy_upsample_mmx)
+
+EXTN(jconst_fancy_upsample_mmx):
+
+PW_ONE          times 4 dw  1
+PW_TWO          times 4 dw  2
+PW_THREE        times 4 dw  3
+PW_SEVEN        times 4 dw  7
+PW_EIGHT        times 4 dw  8
+
+        alignz  16
+
+; --------------------------------------------------------------------------
+        SECTION SEG_TEXT
+        BITS    32
+;
+; Fancy processing for the common case of 2:1 horizontal and 1:1 vertical.
+;
+; The upsampling algorithm is linear interpolation between pixel centers,
+; also known as a "triangle filter".  This is a good compromise between
+; speed and visual quality.  The centers of the output pixels are 1/4 and 3/4
+; of the way between input pixel centers.
+;
+; GLOBAL(void)
+; jsimd_h2v1_fancy_upsample_mmx (int max_v_samp_factor,
+;                                JDIMENSION downsampled_width,
+;                                JSAMPARRAY input_data,
+;                                JSAMPARRAY *output_data_ptr);
+;
+
+%define max_v_samp(b)           (b)+8           ; int max_v_samp_factor
+%define downsamp_width(b)       (b)+12          ; JDIMENSION downsampled_width
+%define input_data(b)           (b)+16          ; JSAMPARRAY input_data
+%define output_data_ptr(b)      (b)+20          ; JSAMPARRAY *output_data_ptr
+
+        align   16
+        global  EXTN(jsimd_h2v1_fancy_upsample_mmx)
+
+EXTN(jsimd_h2v1_fancy_upsample_mmx):
+        push    ebp
+        mov     ebp,esp
+        pushpic ebx
+;       push    ecx             ; need not be preserved
+;       push    edx             ; need not be preserved
+        push    esi
+        push    edi
+
+        get_GOT ebx             ; get GOT address
+
+        mov     eax, JDIMENSION [downsamp_width(ebp)]  ; colctr
+        test    eax,eax
+        jz      near .return
+
+        mov     ecx, INT [max_v_samp(ebp)]      ; rowctr
+        test    ecx,ecx
+        jz      near .return
+
+        mov     esi, JSAMPARRAY [input_data(ebp)]       ; input_data
+        mov     edi, POINTER [output_data_ptr(ebp)]
+        mov     edi, JSAMPARRAY [edi]                   ; output_data
+        alignx  16,7
+.rowloop:
+        push    eax                     ; colctr
+        push    edi
+        push    esi
+
+        mov     esi, JSAMPROW [esi]     ; inptr
+        mov     edi, JSAMPROW [edi]     ; outptr
+
+        test    eax, SIZEOF_MMWORD-1
+        jz      short .skip
+        mov     dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
+        mov     JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl    ; insert a dummy sample
+.skip:
+        pxor    mm0,mm0                 ; mm0=(all 0's)
+        pcmpeqb mm7,mm7
+        psrlq   mm7,(SIZEOF_MMWORD-1)*BYTE_BIT
+        pand    mm7, MMWORD [esi+0*SIZEOF_MMWORD]
+
+        add     eax, byte SIZEOF_MMWORD-1
+        and     eax, byte -SIZEOF_MMWORD
+        cmp     eax, byte SIZEOF_MMWORD
+        ja      short .columnloop
+        alignx  16,7
+
+.columnloop_last:
+        pcmpeqb mm6,mm6
+        psllq   mm6,(SIZEOF_MMWORD-1)*BYTE_BIT
+        pand    mm6, MMWORD [esi+0*SIZEOF_MMWORD]
+        jmp     short .upsample
+        alignx  16,7
+
+.columnloop:
+        movq    mm6, MMWORD [esi+1*SIZEOF_MMWORD]
+        psllq   mm6,(SIZEOF_MMWORD-1)*BYTE_BIT
+
+.upsample:
+        movq    mm1, MMWORD [esi+0*SIZEOF_MMWORD]
+        movq    mm2,mm1
+        movq    mm3,mm1                 ; mm1=( 0 1 2 3 4 5 6 7)
+        psllq   mm2,BYTE_BIT            ; mm2=( - 0 1 2 3 4 5 6)
+        psrlq   mm3,BYTE_BIT            ; mm3=( 1 2 3 4 5 6 7 -)
+
+        por     mm2,mm7                 ; mm2=(-1 0 1 2 3 4 5 6)
+        por     mm3,mm6                 ; mm3=( 1 2 3 4 5 6 7 8)
+
+        movq    mm7,mm1
+        psrlq   mm7,(SIZEOF_MMWORD-1)*BYTE_BIT  ; mm7=( 7 - - - - - - -)
+
+        movq      mm4,mm1
+        punpcklbw mm1,mm0               ; mm1=( 0 1 2 3)
+        punpckhbw mm4,mm0               ; mm4=( 4 5 6 7)
+        movq      mm5,mm2
+        punpcklbw mm2,mm0               ; mm2=(-1 0 1 2)
+        punpckhbw mm5,mm0               ; mm5=( 3 4 5 6)
+        movq      mm6,mm3
+        punpcklbw mm3,mm0               ; mm3=( 1 2 3 4)
+        punpckhbw mm6,mm0               ; mm6=( 5 6 7 8)
+
+        pmullw  mm1,[GOTOFF(ebx,PW_THREE)]
+        pmullw  mm4,[GOTOFF(ebx,PW_THREE)]
+        paddw   mm2,[GOTOFF(ebx,PW_ONE)]
+        paddw   mm5,[GOTOFF(ebx,PW_ONE)]
+        paddw   mm3,[GOTOFF(ebx,PW_TWO)]
+        paddw   mm6,[GOTOFF(ebx,PW_TWO)]
+
+        paddw   mm2,mm1
+        paddw   mm5,mm4
+        psrlw   mm2,2                   ; mm2=OutLE=( 0  2  4  6)
+        psrlw   mm5,2                   ; mm5=OutHE=( 8 10 12 14)
+        paddw   mm3,mm1
+        paddw   mm6,mm4
+        psrlw   mm3,2                   ; mm3=OutLO=( 1  3  5  7)
+        psrlw   mm6,2                   ; mm6=OutHO=( 9 11 13 15)
+
+        psllw   mm3,BYTE_BIT
+        psllw   mm6,BYTE_BIT
+        por     mm2,mm3                 ; mm2=OutL=( 0  1  2  3  4  5  6  7)
+        por     mm5,mm6                 ; mm5=OutH=( 8  9 10 11 12 13 14 15)
+
+        movq    MMWORD [edi+0*SIZEOF_MMWORD], mm2
+        movq    MMWORD [edi+1*SIZEOF_MMWORD], mm5
+
+        sub     eax, byte SIZEOF_MMWORD
+        add     esi, byte 1*SIZEOF_MMWORD       ; inptr
+        add     edi, byte 2*SIZEOF_MMWORD       ; outptr
+        cmp     eax, byte SIZEOF_MMWORD
+        ja      near .columnloop
+        test    eax,eax
+        jnz     near .columnloop_last
+
+        pop     esi
+        pop     edi
+        pop     eax
+
+        add     esi, byte SIZEOF_JSAMPROW       ; input_data
+        add     edi, byte SIZEOF_JSAMPROW       ; output_data
+        dec     ecx                             ; rowctr
+        jg      near .rowloop
+
+        emms            ; empty MMX state
+
+.return:
+        pop     edi
+        pop     esi
+;       pop     edx             ; need not be preserved
+;       pop     ecx             ; need not be preserved
+        poppic  ebx
+        pop     ebp
+        ret
+
+; --------------------------------------------------------------------------
+;
+; Fancy processing for the common case of 2:1 horizontal and 2:1 vertical.
+; Again a triangle filter; see comments for h2v1 case, above.
+;
+; GLOBAL(void)
+; jsimd_h2v2_fancy_upsample_mmx (int max_v_samp_factor,
+;                                JDIMENSION downsampled_width,
+;                                JSAMPARRAY input_data,
+;                                JSAMPARRAY *output_data_ptr);
+;
+
+%define max_v_samp(b)           (b)+8           ; int max_v_samp_factor
+%define downsamp_width(b)       (b)+12          ; JDIMENSION downsampled_width
+%define input_data(b)           (b)+16          ; JSAMPARRAY input_data
+%define output_data_ptr(b)      (b)+20          ; JSAMPARRAY *output_data_ptr
+
+%define original_ebp    ebp+0
+%define wk(i)           ebp-(WK_NUM-(i))*SIZEOF_MMWORD  ; mmword wk[WK_NUM]
+%define WK_NUM          4
+%define gotptr          wk(0)-SIZEOF_POINTER    ; void *gotptr
+
+        align   16
+        global  EXTN(jsimd_h2v2_fancy_upsample_mmx)
+
+EXTN(jsimd_h2v2_fancy_upsample_mmx):
+        push    ebp
+        mov     eax,esp                         ; eax = original ebp
+        sub     esp, byte 4
+        and     esp, byte (-SIZEOF_MMWORD)      ; align to 64 bits
+        mov     [esp],eax
+        mov     ebp,esp                         ; ebp = aligned ebp
+        lea     esp, [wk(0)]
+        pushpic eax             ; make a room for GOT address
+        push    ebx
+;       push    ecx             ; need not be preserved
+;       push    edx             ; need not be preserved
+        push    esi
+        push    edi
+
+        get_GOT ebx                     ; get GOT address
+        movpic  POINTER [gotptr], ebx   ; save GOT address
+
+        mov     edx,eax                         ; edx = original ebp
+        mov     eax, JDIMENSION [downsamp_width(edx)]  ; colctr
+        test    eax,eax
+        jz      near .return
+
+        mov     ecx, INT [max_v_samp(edx)]      ; rowctr
+        test    ecx,ecx
+        jz      near .return
+
+        mov     esi, JSAMPARRAY [input_data(edx)]       ; input_data
+        mov     edi, POINTER [output_data_ptr(edx)]
+        mov     edi, JSAMPARRAY [edi]                   ; output_data
+        alignx  16,7
+.rowloop:
+        push    eax                                     ; colctr
+        push    ecx
+        push    edi
+        push    esi
+
+        mov     ecx, JSAMPROW [esi-1*SIZEOF_JSAMPROW]   ; inptr1(above)
+        mov     ebx, JSAMPROW [esi+0*SIZEOF_JSAMPROW]   ; inptr0
+        mov     esi, JSAMPROW [esi+1*SIZEOF_JSAMPROW]   ; inptr1(below)
+        mov     edx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]   ; outptr0
+        mov     edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]   ; outptr1
+
+        test    eax, SIZEOF_MMWORD-1
+        jz      short .skip
+        push    edx
+        mov     dl, JSAMPLE [ecx+(eax-1)*SIZEOF_JSAMPLE]
+        mov     JSAMPLE [ecx+eax*SIZEOF_JSAMPLE], dl
+        mov     dl, JSAMPLE [ebx+(eax-1)*SIZEOF_JSAMPLE]
+        mov     JSAMPLE [ebx+eax*SIZEOF_JSAMPLE], dl
+        mov     dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
+        mov     JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl    ; insert a dummy sample
+        pop     edx
+.skip:
+        ; -- process the first column block
+
+        movq    mm0, MMWORD [ebx+0*SIZEOF_MMWORD]       ; mm0=row[ 0][0]
+        movq    mm1, MMWORD [ecx+0*SIZEOF_MMWORD]       ; mm1=row[-1][0]
+        movq    mm2, MMWORD [esi+0*SIZEOF_MMWORD]       ; mm2=row[+1][0]
+
+        pushpic ebx
+        movpic  ebx, POINTER [gotptr]   ; load GOT address
+
+        pxor      mm3,mm3               ; mm3=(all 0's)
+        movq      mm4,mm0
+        punpcklbw mm0,mm3               ; mm0=row[ 0][0]( 0 1 2 3)
+        punpckhbw mm4,mm3               ; mm4=row[ 0][0]( 4 5 6 7)
+        movq      mm5,mm1
+        punpcklbw mm1,mm3               ; mm1=row[-1][0]( 0 1 2 3)
+        punpckhbw mm5,mm3               ; mm5=row[-1][0]( 4 5 6 7)
+        movq      mm6,mm2
+        punpcklbw mm2,mm3               ; mm2=row[+1][0]( 0 1 2 3)
+        punpckhbw mm6,mm3               ; mm6=row[+1][0]( 4 5 6 7)
+
+        pmullw  mm0,[GOTOFF(ebx,PW_THREE)]
+        pmullw  mm4,[GOTOFF(ebx,PW_THREE)]
+
+        pcmpeqb mm7,mm7
+        psrlq   mm7,(SIZEOF_MMWORD-2)*BYTE_BIT
+
+        paddw   mm1,mm0                 ; mm1=Int0L=( 0 1 2 3)
+        paddw   mm5,mm4                 ; mm5=Int0H=( 4 5 6 7)
+        paddw   mm2,mm0                 ; mm2=Int1L=( 0 1 2 3)
+        paddw   mm6,mm4                 ; mm6=Int1H=( 4 5 6 7)
+
+        movq    MMWORD [edx+0*SIZEOF_MMWORD], mm1       ; temporarily save
+        movq    MMWORD [edx+1*SIZEOF_MMWORD], mm5       ; the intermediate data
+        movq    MMWORD [edi+0*SIZEOF_MMWORD], mm2
+        movq    MMWORD [edi+1*SIZEOF_MMWORD], mm6
+
+        pand    mm1,mm7                 ; mm1=( 0 - - -)
+        pand    mm2,mm7                 ; mm2=( 0 - - -)
+
+        movq    MMWORD [wk(0)], mm1
+        movq    MMWORD [wk(1)], mm2
+
+        poppic  ebx
+
+        add     eax, byte SIZEOF_MMWORD-1
+        and     eax, byte -SIZEOF_MMWORD
+        cmp     eax, byte SIZEOF_MMWORD
+        ja      short .columnloop
+        alignx  16,7
+
+.columnloop_last:
+        ; -- process the last column block
+
+        pushpic ebx
+        movpic  ebx, POINTER [gotptr]   ; load GOT address
+
+        pcmpeqb mm1,mm1
+        psllq   mm1,(SIZEOF_MMWORD-2)*BYTE_BIT
+        movq    mm2,mm1
+
+        pand    mm1, MMWORD [edx+1*SIZEOF_MMWORD]       ; mm1=( - - - 7)
+        pand    mm2, MMWORD [edi+1*SIZEOF_MMWORD]       ; mm2=( - - - 7)
+
+        movq    MMWORD [wk(2)], mm1
+        movq    MMWORD [wk(3)], mm2
+
+        jmp     short .upsample
+        alignx  16,7
+
+.columnloop:
+        ; -- process the next column block
+
+        movq    mm0, MMWORD [ebx+1*SIZEOF_MMWORD]       ; mm0=row[ 0][1]
+        movq    mm1, MMWORD [ecx+1*SIZEOF_MMWORD]       ; mm1=row[-1][1]
+        movq    mm2, MMWORD [esi+1*SIZEOF_MMWORD]       ; mm2=row[+1][1]
+
+        pushpic ebx
+        movpic  ebx, POINTER [gotptr]   ; load GOT address
+
+        pxor      mm3,mm3               ; mm3=(all 0's)
+        movq      mm4,mm0
+        punpcklbw mm0,mm3               ; mm0=row[ 0][1]( 0 1 2 3)
+        punpckhbw mm4,mm3               ; mm4=row[ 0][1]( 4 5 6 7)
+        movq      mm5,mm1
+        punpcklbw mm1,mm3               ; mm1=row[-1][1]( 0 1 2 3)
+        punpckhbw mm5,mm3               ; mm5=row[-1][1]( 4 5 6 7)
+        movq      mm6,mm2
+        punpcklbw mm2,mm3               ; mm2=row[+1][1]( 0 1 2 3)
+        punpckhbw mm6,mm3               ; mm6=row[+1][1]( 4 5 6 7)
+
+        pmullw  mm0,[GOTOFF(ebx,PW_THREE)]
+        pmullw  mm4,[GOTOFF(ebx,PW_THREE)]
+
+        paddw   mm1,mm0                 ; mm1=Int0L=( 0 1 2 3)
+        paddw   mm5,mm4                 ; mm5=Int0H=( 4 5 6 7)
+        paddw   mm2,mm0                 ; mm2=Int1L=( 0 1 2 3)
+        paddw   mm6,mm4                 ; mm6=Int1H=( 4 5 6 7)
+
+        movq    MMWORD [edx+2*SIZEOF_MMWORD], mm1       ; temporarily save
+        movq    MMWORD [edx+3*SIZEOF_MMWORD], mm5       ; the intermediate data
+        movq    MMWORD [edi+2*SIZEOF_MMWORD], mm2
+        movq    MMWORD [edi+3*SIZEOF_MMWORD], mm6
+
+        psllq   mm1,(SIZEOF_MMWORD-2)*BYTE_BIT  ; mm1=( - - - 0)
+        psllq   mm2,(SIZEOF_MMWORD-2)*BYTE_BIT  ; mm2=( - - - 0)
+
+        movq    MMWORD [wk(2)], mm1
+        movq    MMWORD [wk(3)], mm2
+
+.upsample:
+        ; -- process the upper row
+
+        movq    mm7, MMWORD [edx+0*SIZEOF_MMWORD]       ; mm7=Int0L=( 0 1 2 3)
+        movq    mm3, MMWORD [edx+1*SIZEOF_MMWORD]       ; mm3=Int0H=( 4 5 6 7)
+
+        movq    mm0,mm7
+        movq    mm4,mm3
+        psrlq   mm0,2*BYTE_BIT                  ; mm0=( 1 2 3 -)
+        psllq   mm4,(SIZEOF_MMWORD-2)*BYTE_BIT  ; mm4=( - - - 4)
+        movq    mm5,mm7
+        movq    mm6,mm3
+        psrlq   mm5,(SIZEOF_MMWORD-2)*BYTE_BIT  ; mm5=( 3 - - -)
+        psllq   mm6,2*BYTE_BIT                  ; mm6=( - 4 5 6)
+
+        por     mm0,mm4                         ; mm0=( 1 2 3 4)
+        por     mm5,mm6                         ; mm5=( 3 4 5 6)
+
+        movq    mm1,mm7
+        movq    mm2,mm3
+        psllq   mm1,2*BYTE_BIT                  ; mm1=( - 0 1 2)
+        psrlq   mm2,2*BYTE_BIT                  ; mm2=( 5 6 7 -)
+        movq    mm4,mm3
+        psrlq   mm4,(SIZEOF_MMWORD-2)*BYTE_BIT  ; mm4=( 7 - - -)
+
+        por     mm1, MMWORD [wk(0)]             ; mm1=(-1 0 1 2)
+        por     mm2, MMWORD [wk(2)]             ; mm2=( 5 6 7 8)
+
+        movq    MMWORD [wk(0)], mm4
+
+        pmullw  mm7,[GOTOFF(ebx,PW_THREE)]
+        pmullw  mm3,[GOTOFF(ebx,PW_THREE)]
+        paddw   mm1,[GOTOFF(ebx,PW_EIGHT)]
+        paddw   mm5,[GOTOFF(ebx,PW_EIGHT)]
+        paddw   mm0,[GOTOFF(ebx,PW_SEVEN)]
+        paddw   mm2,[GOTOFF(ebx,PW_SEVEN)]
+
+        paddw   mm1,mm7
+        paddw   mm5,mm3
+        psrlw   mm1,4                   ; mm1=Out0LE=( 0  2  4  6)
+        psrlw   mm5,4                   ; mm5=Out0HE=( 8 10 12 14)
+        paddw   mm0,mm7
+        paddw   mm2,mm3
+        psrlw   mm0,4                   ; mm0=Out0LO=( 1  3  5  7)
+        psrlw   mm2,4                   ; mm2=Out0HO=( 9 11 13 15)
+
+        psllw   mm0,BYTE_BIT
+        psllw   mm2,BYTE_BIT
+        por     mm1,mm0                 ; mm1=Out0L=( 0  1  2  3  4  5  6  7)
+        por     mm5,mm2                 ; mm5=Out0H=( 8  9 10 11 12 13 14 15)
+
+        movq    MMWORD [edx+0*SIZEOF_MMWORD], mm1
+        movq    MMWORD [edx+1*SIZEOF_MMWORD], mm5
+
+        ; -- process the lower row
+
+        movq    mm6, MMWORD [edi+0*SIZEOF_MMWORD]       ; mm6=Int1L=( 0 1 2 3)
+        movq    mm4, MMWORD [edi+1*SIZEOF_MMWORD]       ; mm4=Int1H=( 4 5 6 7)
+
+        movq    mm7,mm6
+        movq    mm3,mm4
+        psrlq   mm7,2*BYTE_BIT                  ; mm7=( 1 2 3 -)
+        psllq   mm3,(SIZEOF_MMWORD-2)*BYTE_BIT  ; mm3=( - - - 4)
+        movq    mm0,mm6
+        movq    mm2,mm4
+        psrlq   mm0,(SIZEOF_MMWORD-2)*BYTE_BIT  ; mm0=( 3 - - -)
+        psllq   mm2,2*BYTE_BIT                  ; mm2=( - 4 5 6)
+
+        por     mm7,mm3                         ; mm7=( 1 2 3 4)
+        por     mm0,mm2                         ; mm0=( 3 4 5 6)
+
+        movq    mm1,mm6
+        movq    mm5,mm4
+        psllq   mm1,2*BYTE_BIT                  ; mm1=( - 0 1 2)
+        psrlq   mm5,2*BYTE_BIT                  ; mm5=( 5 6 7 -)
+        movq    mm3,mm4
+        psrlq   mm3,(SIZEOF_MMWORD-2)*BYTE_BIT  ; mm3=( 7 - - -)
+
+        por     mm1, MMWORD [wk(1)]             ; mm1=(-1 0 1 2)
+        por     mm5, MMWORD [wk(3)]             ; mm5=( 5 6 7 8)
+
+        movq    MMWORD [wk(1)], mm3
+
+        pmullw  mm6,[GOTOFF(ebx,PW_THREE)]
+        pmullw  mm4,[GOTOFF(ebx,PW_THREE)]
+        paddw   mm1,[GOTOFF(ebx,PW_EIGHT)]
+        paddw   mm0,[GOTOFF(ebx,PW_EIGHT)]
+        paddw   mm7,[GOTOFF(ebx,PW_SEVEN)]
+        paddw   mm5,[GOTOFF(ebx,PW_SEVEN)]
+
+        paddw   mm1,mm6
+        paddw   mm0,mm4
+        psrlw   mm1,4                   ; mm1=Out1LE=( 0  2  4  6)
+        psrlw   mm0,4                   ; mm0=Out1HE=( 8 10 12 14)
+        paddw   mm7,mm6
+        paddw   mm5,mm4
+        psrlw   mm7,4                   ; mm7=Out1LO=( 1  3  5  7)
+        psrlw   mm5,4                   ; mm5=Out1HO=( 9 11 13 15)
+
+        psllw   mm7,BYTE_BIT
+        psllw   mm5,BYTE_BIT
+        por     mm1,mm7                 ; mm1=Out1L=( 0  1  2  3  4  5  6  7)
+        por     mm0,mm5                 ; mm0=Out1H=( 8  9 10 11 12 13 14 15)
+
+        movq    MMWORD [edi+0*SIZEOF_MMWORD], mm1
+        movq    MMWORD [edi+1*SIZEOF_MMWORD], mm0
+
+        poppic  ebx
+
+        sub     eax, byte SIZEOF_MMWORD
+        add     ecx, byte 1*SIZEOF_MMWORD       ; inptr1(above)
+        add     ebx, byte 1*SIZEOF_MMWORD       ; inptr0
+        add     esi, byte 1*SIZEOF_MMWORD       ; inptr1(below)
+        add     edx, byte 2*SIZEOF_MMWORD       ; outptr0
+        add     edi, byte 2*SIZEOF_MMWORD       ; outptr1
+        cmp     eax, byte SIZEOF_MMWORD
+        ja      near .columnloop
+        test    eax,eax
+        jnz     near .columnloop_last
+
+        pop     esi
+        pop     edi
+        pop     ecx
+        pop     eax
+
+        add     esi, byte 1*SIZEOF_JSAMPROW     ; input_data
+        add     edi, byte 2*SIZEOF_JSAMPROW     ; output_data
+        sub     ecx, byte 2                     ; rowctr
+        jg      near .rowloop
+
+        emms            ; empty MMX state
+
+.return:
+        pop     edi
+        pop     esi
+;       pop     edx             ; need not be preserved
+;       pop     ecx             ; need not be preserved
+        pop     ebx
+        mov     esp,ebp         ; esp <- aligned ebp
+        pop     esp             ; esp <- original ebp
+        pop     ebp
+        ret
+
+; --------------------------------------------------------------------------
+;
+; Fast processing for the common case of 2:1 horizontal and 1:1 vertical.
+; It's still a box filter.
+;
+; GLOBAL(void)
+; jsimd_h2v1_upsample_mmx (int max_v_samp_factor,
+;                          JDIMENSION output_width,
+;                          JSAMPARRAY input_data,
+;                          JSAMPARRAY *output_data_ptr);
+;
+
+%define max_v_samp(b)           (b)+8           ; int max_v_samp_factor
+%define output_width(b)         (b)+12          ; JDIMENSION output_width
+%define input_data(b)           (b)+16          ; JSAMPARRAY input_data
+%define output_data_ptr(b)      (b)+20          ; JSAMPARRAY *output_data_ptr
+
+        align   16
+        global  EXTN(jsimd_h2v1_upsample_mmx)
+
+EXTN(jsimd_h2v1_upsample_mmx):
+        push    ebp
+        mov     ebp,esp
+;       push    ebx             ; unused
+;       push    ecx             ; need not be preserved
+;       push    edx             ; need not be preserved
+        push    esi
+        push    edi
+
+        mov     edx, JDIMENSION [output_width(ebp)]
+        add     edx, byte (2*SIZEOF_MMWORD)-1
+        and     edx, byte -(2*SIZEOF_MMWORD)
+        jz      short .return
+
+        mov     ecx, INT [max_v_samp(ebp)]      ; rowctr
+        test    ecx,ecx
+        jz      short .return
+
+        mov     esi, JSAMPARRAY [input_data(ebp)]       ; input_data
+        mov     edi, POINTER [output_data_ptr(ebp)]
+        mov     edi, JSAMPARRAY [edi]                   ; output_data
+        alignx  16,7
+.rowloop:
+        push    edi
+        push    esi
+
+        mov     esi, JSAMPROW [esi]             ; inptr
+        mov     edi, JSAMPROW [edi]             ; outptr
+        mov     eax,edx                         ; colctr
+        alignx  16,7
+.columnloop:
+
+        movq    mm0, MMWORD [esi+0*SIZEOF_MMWORD]
+
+        movq      mm1,mm0
+        punpcklbw mm0,mm0
+        punpckhbw mm1,mm1
+
+        movq    MMWORD [edi+0*SIZEOF_MMWORD], mm0
+        movq    MMWORD [edi+1*SIZEOF_MMWORD], mm1
+
+        sub     eax, byte 2*SIZEOF_MMWORD
+        jz      short .nextrow
+
+        movq    mm2, MMWORD [esi+1*SIZEOF_MMWORD]
+
+        movq      mm3,mm2
+        punpcklbw mm2,mm2
+        punpckhbw mm3,mm3
+
+        movq    MMWORD [edi+2*SIZEOF_MMWORD], mm2
+        movq    MMWORD [edi+3*SIZEOF_MMWORD], mm3
+
+        sub     eax, byte 2*SIZEOF_MMWORD
+        jz      short .nextrow
+
+        add     esi, byte 2*SIZEOF_MMWORD       ; inptr
+        add     edi, byte 4*SIZEOF_MMWORD       ; outptr
+        jmp     short .columnloop
+        alignx  16,7
+
+.nextrow:
+        pop     esi
+        pop     edi
+
+        add     esi, byte SIZEOF_JSAMPROW       ; input_data
+        add     edi, byte SIZEOF_JSAMPROW       ; output_data
+        dec     ecx                             ; rowctr
+        jg      short .rowloop
+
+        emms            ; empty MMX state
+
+.return:
+        pop     edi
+        pop     esi
+;       pop     edx             ; need not be preserved
+;       pop     ecx             ; need not be preserved
+;       pop     ebx             ; unused
+        pop     ebp
+        ret
+
+; --------------------------------------------------------------------------
+;
+; Fast processing for the common case of 2:1 horizontal and 2:1 vertical.
+; It's still a box filter.
+;
+; GLOBAL(void)
+; jsimd_h2v2_upsample_mmx (int max_v_samp_factor,
+;                          JDIMENSION output_width,
+;                          JSAMPARRAY input_data,
+;                          JSAMPARRAY *output_data_ptr);
+;
+
+%define max_v_samp(b)           (b)+8           ; int max_v_samp_factor
+%define output_width(b)         (b)+12          ; JDIMENSION output_width
+%define input_data(b)           (b)+16          ; JSAMPARRAY input_data
+%define output_data_ptr(b)      (b)+20          ; JSAMPARRAY *output_data_ptr
+
+        align   16
+        global  EXTN(jsimd_h2v2_upsample_mmx)
+
+EXTN(jsimd_h2v2_upsample_mmx):
+        push    ebp
+        mov     ebp,esp
+        push    ebx
+;       push    ecx             ; need not be preserved
+;       push    edx             ; need not be preserved
+        push    esi
+        push    edi
+
+        mov     edx, JDIMENSION [output_width(ebp)]
+        add     edx, byte (2*SIZEOF_MMWORD)-1
+        and     edx, byte -(2*SIZEOF_MMWORD)
+        jz      near .return
+
+        mov     ecx, INT [max_v_samp(ebp)]      ; rowctr
+        test    ecx,ecx
+        jz      short .return
+
+        mov     esi, JSAMPARRAY [input_data(ebp)]       ; input_data
+        mov     edi, POINTER [output_data_ptr(ebp)]
+        mov     edi, JSAMPARRAY [edi]                   ; output_data
+        alignx  16,7
+.rowloop:
+        push    edi
+        push    esi
+
+        mov     esi, JSAMPROW [esi]                     ; inptr
+        mov     ebx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]   ; outptr0
+        mov     edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]   ; outptr1
+        mov     eax,edx                                 ; colctr
+        alignx  16,7
+.columnloop:
+
+        movq    mm0, MMWORD [esi+0*SIZEOF_MMWORD]
+
+        movq      mm1,mm0
+        punpcklbw mm0,mm0
+        punpckhbw mm1,mm1
+
+        movq    MMWORD [ebx+0*SIZEOF_MMWORD], mm0
+        movq    MMWORD [ebx+1*SIZEOF_MMWORD], mm1
+        movq    MMWORD [edi+0*SIZEOF_MMWORD], mm0
+        movq    MMWORD [edi+1*SIZEOF_MMWORD], mm1
+
+        sub     eax, byte 2*SIZEOF_MMWORD
+        jz      short .nextrow
+
+        movq    mm2, MMWORD [esi+1*SIZEOF_MMWORD]
+
+        movq      mm3,mm2
+        punpcklbw mm2,mm2
+        punpckhbw mm3,mm3
+
+        movq    MMWORD [ebx+2*SIZEOF_MMWORD], mm2
+        movq    MMWORD [ebx+3*SIZEOF_MMWORD], mm3
+        movq    MMWORD [edi+2*SIZEOF_MMWORD], mm2
+        movq    MMWORD [edi+3*SIZEOF_MMWORD], mm3
+
+        sub     eax, byte 2*SIZEOF_MMWORD
+        jz      short .nextrow
+
+        add     esi, byte 2*SIZEOF_MMWORD       ; inptr
+        add     ebx, byte 4*SIZEOF_MMWORD       ; outptr0
+        add     edi, byte 4*SIZEOF_MMWORD       ; outptr1
+        jmp     short .columnloop
+        alignx  16,7
+
+.nextrow:
+        pop     esi
+        pop     edi
+
+        add     esi, byte 1*SIZEOF_JSAMPROW     ; input_data
+        add     edi, byte 2*SIZEOF_JSAMPROW     ; output_data
+        sub     ecx, byte 2                     ; rowctr
+        jg      short .rowloop
+
+        emms            ; empty MMX state
+
+.return:
+        pop     edi
+        pop     esi
+;       pop     edx             ; need not be preserved
+;       pop     ecx             ; need not be preserved
+        pop     ebx
+        pop     ebp
+        ret
+
+; For some reason, the OS X linker does not honor the request to align the
+; segment unless we do this.
+        align   16