rfreeimage 0.1.0

data/vendor/FreeImage/Source/LibWebP/src/dsp/dsp.rescaler_mips32.c

@@ -0,0 +1,192 @@
+// Copyright 2014 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// MIPS version of rescaling functions
+//
+// Author(s): Djordje Pesut (djordje.pesut@imgtec.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_MIPS32)
+
+#include "../utils/rescaler.h"
+
+static void ImportRow(WebPRescaler* const wrk,
+                      const uint8_t* const src, int channel) {
+  const int x_stride = wrk->num_channels;
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
+  const int fx_scale = wrk->fx_scale;
+  const int x_add = wrk->x_add;
+  const int x_sub = wrk->x_sub;
+  int* frow = wrk->frow + channel;
+  int* irow = wrk->irow + channel;
+  const uint8_t* src1 = src + channel;
+  int temp1, temp2, temp3;
+  int base, frac, sum;
+  int accum, accum1;
+  const int x_stride1 = x_stride << 2;
+  int loop_c = x_out_max - channel;
+
+  if (!wrk->x_expand) {
+    __asm__ volatile (
+      "li     %[temp1],   0x8000                    \n\t"
+      "li     %[temp2],   0x10000                   \n\t"
+      "li     %[sum],     0                         \n\t"
+      "li     %[accum],   0                         \n\t"
+    "1:                                             \n\t"
+      "addu   %[accum],   %[accum],   %[x_add]      \n\t"
+      "blez   %[accum],   3f                        \n\t"
+    "2:                                             \n\t"
+      "lbu    %[temp3],   0(%[src1])                \n\t"
+      "subu   %[accum],   %[accum],   %[x_sub]      \n\t"
+      "addu   %[src1],    %[src1],    %[x_stride]   \n\t"
+      "addu   %[sum],     %[sum],     %[temp3]      \n\t"
+      "bgtz   %[accum],   2b                        \n\t"
+    "3:                                             \n\t"
+      "lbu    %[base],    0(%[src1])                \n\t"
+      "addu   %[src1],    %[src1],    %[x_stride]   \n\t"
+      "negu   %[accum1],  %[accum]                  \n\t"
+      "mul    %[frac],    %[base],    %[accum1]     \n\t"
+      "addu   %[temp3],   %[sum],     %[base]       \n\t"
+      "mul    %[temp3],   %[temp3],   %[x_sub]      \n\t"
+      "lw     %[base],    0(%[irow])                \n\t"
+      "subu   %[loop_c],  %[loop_c],  %[x_stride]   \n\t"
+      "sll    %[accum1],  %[frac],    2             \n\t"
+      "mult   %[temp1],   %[temp2]                  \n\t"
+      "madd   %[accum1],  %[fx_scale]               \n\t"
+      "mfhi   %[sum]                                \n\t"
+      "subu   %[temp3],   %[temp3],   %[frac]       \n\t"
+      "sw     %[temp3],   0(%[frow])                \n\t"
+      "add    %[base],    %[base],    %[temp3]      \n\t"
+      "sw     %[base],    0(%[irow])                \n\t"
+      "addu   %[irow],    %[irow],    %[x_stride1]  \n\t"
+      "addu   %[frow],    %[frow],    %[x_stride1]  \n\t"
+      "bgtz   %[loop_c],  1b                        \n\t"
+
+      : [accum] "=&r" (accum), [src1] "+r" (src1), [temp3] "=&r" (temp3),
+        [sum] "=&r" (sum), [base] "=&r" (base), [frac] "=&r" (frac),
+        [frow] "+r" (frow), [irow] "+r" (irow), [accum1] "=&r" (accum1),
+        [temp2] "=&r" (temp2), [temp1] "=&r" (temp1)
+      : [x_stride] "r" (x_stride), [fx_scale] "r" (fx_scale),
+        [x_sub] "r" (x_sub), [x_add] "r" (x_add),
+        [loop_c] "r" (loop_c), [x_stride1] "r" (x_stride1)
+      : "memory", "hi", "lo"
+    );
+  } else {
+    __asm__ volatile (
+      "lbu    %[temp1],   0(%[src1])                \n\t"
+      "move   %[temp2],   %[temp1]                  \n\t"
+      "li     %[accum],   0                         \n\t"
+    "1:                                             \n\t"
+      "bgez   %[accum],   2f                        \n\t"
+      "move   %[temp2],   %[temp1]                  \n\t"
+      "addu   %[src1],    %[x_stride]               \n\t"
+      "lbu    %[temp1],   0(%[src1])                \n\t"
+      "addu   %[accum],   %[x_add]                  \n\t"
+    "2:                                             \n\t"
+      "subu   %[temp3],   %[temp2],   %[temp1]      \n\t"
+      "mul    %[temp3],   %[temp3],   %[accum]      \n\t"
+      "mul    %[base],    %[temp1],   %[x_add]      \n\t"
+      "subu   %[accum],   %[accum],   %[x_sub]      \n\t"
+      "lw     %[frac],    0(%[irow])                \n\t"
+      "subu   %[loop_c],  %[loop_c],  %[x_stride]   \n\t"
+      "addu   %[temp3],   %[base],    %[temp3]      \n\t"
+      "sw     %[temp3],   0(%[frow])                \n\t"
+      "addu   %[frow],    %[x_stride1]              \n\t"
+      "addu   %[frac],    %[temp3]                  \n\t"
+      "sw     %[frac],    0(%[irow])                \n\t"
+      "addu   %[irow],    %[x_stride1]              \n\t"
+      "bgtz   %[loop_c],  1b                        \n\t"
+
+      : [src1] "+r" (src1), [accum] "=&r" (accum), [temp1] "=&r" (temp1),
+        [temp2] "=&r" (temp2), [temp3] "=&r" (temp3), [base] "=&r" (base),
+        [frac] "=&r" (frac), [frow] "+r" (frow), [irow] "+r" (irow)
+      : [x_stride] "r" (x_stride), [x_add] "r" (x_add), [x_sub] "r" (x_sub),
+        [x_stride1] "r" (x_stride1), [loop_c] "r" (loop_c)
+      : "memory", "hi", "lo"
+    );
+  }
+}
+
+static void ExportRow(WebPRescaler* const wrk, int x_out) {
+  if (wrk->y_accum <= 0) {
+    uint8_t* const dst = wrk->dst;
+    int32_t* const irow = wrk->irow;
+    const int32_t* const frow = wrk->frow;
+    const int yscale = wrk->fy_scale * (-wrk->y_accum);
+    const int x_out_max = wrk->dst_width * wrk->num_channels;
+    // if wrk->fxy_scale can fit into 32 bits use optimized code,
+    // otherwise use C code
+    if ((wrk->fxy_scale >> 32) == 0) {
+      int temp0, temp1, temp3, temp4, temp5, temp6, temp7, loop_end;
+      const int temp2 = (int)(wrk->fxy_scale);
+      const int temp8 = x_out_max << 2;
+      uint8_t* dst_t = (uint8_t*)dst;
+      int32_t* irow_t = (int32_t*)irow;
+      const int32_t* frow_t = (const int32_t*)frow;
+
+      __asm__ volatile(
+        "addiu    %[temp6],    $zero,       -256          \n\t"
+        "addiu    %[temp7],    $zero,       255           \n\t"
+        "li       %[temp3],    0x10000                    \n\t"
+        "li       %[temp4],    0x8000                     \n\t"
+        "addu     %[loop_end], %[frow_t],   %[temp8]      \n\t"
+      "1:                                                 \n\t"
+        "lw       %[temp0],    0(%[frow_t])               \n\t"
+        "mult     %[temp3],    %[temp4]                   \n\t"
+        "addiu    %[frow_t],   %[frow_t],   4             \n\t"
+        "sll      %[temp0],    %[temp0],    2             \n\t"
+        "madd     %[temp0],    %[yscale]                  \n\t"
+        "mfhi     %[temp1]                                \n\t"
+        "lw       %[temp0],    0(%[irow_t])               \n\t"
+        "addiu    %[dst_t],    %[dst_t],    1             \n\t"
+        "addiu    %[irow_t],   %[irow_t],   4             \n\t"
+        "subu     %[temp0],    %[temp0],    %[temp1]      \n\t"
+        "mult     %[temp3],    %[temp4]                   \n\t"
+        "sll      %[temp0],    %[temp0],    2             \n\t"
+        "madd     %[temp0],    %[temp2]                   \n\t"
+        "mfhi     %[temp5]                                \n\t"
+        "sw       %[temp1],    -4(%[irow_t])              \n\t"
+        "and      %[temp0],    %[temp5],    %[temp6]      \n\t"
+        "slti     %[temp1],    %[temp5],    0             \n\t"
+        "beqz     %[temp0],    2f                         \n\t"
+        "xor      %[temp5],    %[temp5],    %[temp5]      \n\t"
+        "movz     %[temp5],    %[temp7],    %[temp1]      \n\t"
+      "2:                                                 \n\t"
+        "sb       %[temp5],    -1(%[dst_t])               \n\t"
+        "bne      %[frow_t],   %[loop_end], 1b            \n\t"
+
+        : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3),
+          [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp6]"=&r"(temp6),
+          [temp7]"=&r"(temp7), [frow_t]"+r"(frow_t), [irow_t]"+r"(irow_t),
+          [dst_t]"+r"(dst_t), [loop_end]"=&r"(loop_end)
+        : [temp2]"r"(temp2), [yscale]"r"(yscale), [temp8]"r"(temp8)
+        : "memory", "hi", "lo"
+      );
+      wrk->y_accum += wrk->y_add;
+      wrk->dst += wrk->dst_stride;
+    } else {
+      WebPRescalerExportRowC(wrk, x_out);
+    }
+  }
+}
+
+#endif  // WEBP_USE_MIPS32
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern void WebPRescalerDspInitMIPS32(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMIPS32(void) {
+#if defined(WEBP_USE_MIPS32)
+  WebPRescalerImportRow = ImportRow;
+  WebPRescalerExportRow = ExportRow;
+#endif  // WEBP_USE_MIPS32
+}

data/vendor/FreeImage/Source/LibWebP/src/dsp/dsp.rescaler_mips_dsp_r2.c

@@ -0,0 +1,210 @@
+// Copyright 2014 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// MIPS version of rescaling functions
+//
+// Author(s): Djordje Pesut (djordje.pesut@imgtec.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_MIPS_DSP_R2)
+
+#include "../utils/rescaler.h"
+
+static void ImportRow(WebPRescaler* const wrk,
+                      const uint8_t* const src, int channel) {
+  const int x_stride = wrk->num_channels;
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
+  const int fx_scale = wrk->fx_scale;
+  const int x_add = wrk->x_add;
+  const int x_sub = wrk->x_sub;
+  int* frow = wrk->frow + channel;
+  int* irow = wrk->irow + channel;
+  const uint8_t* src1 = src + channel;
+  int temp1, temp2, temp3;
+  int base, frac, sum;
+  int accum, accum1;
+  const int x_stride1 = x_stride << 2;
+  int loop_c = x_out_max - channel;
+
+  if (!wrk->x_expand) {
+    __asm__ volatile (
+      "li         %[sum],     0                         \n\t"
+      "li         %[accum],   0                         \n\t"
+    "1:                                                 \n\t"
+      "addu       %[accum],   %[accum],   %[x_add]      \n\t"
+      "blez       %[accum],   3f                        \n\t"
+    "2:                                                 \n\t"
+      "lbu        %[temp3],   0(%[src1])                \n\t"
+      "subu       %[accum],   %[accum],   %[x_sub]      \n\t"
+      "addu       %[src1],    %[src1],    %[x_stride]   \n\t"
+      "addu       %[sum],     %[sum],     %[temp3]      \n\t"
+      "bgtz       %[accum],   2b                        \n\t"
+    "3:                                                 \n\t"
+      "lbu        %[base],    0(%[src1])                \n\t"
+      "addu       %[src1],    %[src1],    %[x_stride]   \n\t"
+      "negu       %[accum1],  %[accum]                  \n\t"
+      "mul        %[frac],    %[base],    %[accum1]     \n\t"
+      "addu       %[temp3],   %[sum],     %[base]       \n\t"
+      "mul        %[temp3],   %[temp3],   %[x_sub]      \n\t"
+      "lw         %[base],    0(%[irow])                \n\t"
+      "sll        %[accum1],  %[frac],    1             \n\t"
+      "subu       %[loop_c],  %[loop_c],  %[x_stride]   \n\t"
+      "mulq_rs.w  %[sum],     %[accum1],  %[fx_scale]   \n\t"
+      "subu       %[temp3],   %[temp3],   %[frac]       \n\t"
+      "sw         %[temp3],   0(%[frow])                \n\t"
+      "add        %[base],    %[base],    %[temp3]      \n\t"
+      "sw         %[base],    0(%[irow])                \n\t"
+      "addu       %[irow],    %[irow],    %[x_stride1]  \n\t"
+      "addu       %[frow],    %[frow],    %[x_stride1]  \n\t"
+      "bgtz       %[loop_c],  1b                        \n\t"
+
+      : [accum]"=&r"(accum), [src1]"+&r"(src1), [temp3]"=&r"(temp3),
+        [sum]"=&r"(sum), [base]"=&r"(base), [frac]"=&r"(frac),
+        [frow]"+&r"(frow), [irow]"+&r"(irow), [accum1]"=&r"(accum1),
+        [loop_c]"+&r"(loop_c)
+      : [x_stride]"r"(x_stride), [fx_scale]"r"(fx_scale), [x_sub]"r"(x_sub),
+        [x_add] "r" (x_add), [x_stride1] "r" (x_stride1)
+      : "memory", "hi", "lo"
+    );
+  } else {
+    __asm__ volatile (
+      "lbu    %[temp1],   0(%[src1])                \n\t"
+      "move   %[temp2],   %[temp1]                  \n\t"
+      "li     %[accum],   0                         \n\t"
+    "1:                                             \n\t"
+      "bgez   %[accum],   2f                        \n\t"
+      "move   %[temp2],   %[temp1]                  \n\t"
+      "addu   %[src1],    %[x_stride]               \n\t"
+      "lbu    %[temp1],   0(%[src1])                \n\t"
+      "addu   %[accum],   %[x_add]                  \n\t"
+    "2:                                             \n\t"
+      "subu   %[temp3],   %[temp2],   %[temp1]      \n\t"
+      "mul    %[temp3],   %[temp3],   %[accum]      \n\t"
+      "mul    %[base],    %[temp1],   %[x_add]      \n\t"
+      "subu   %[accum],   %[accum],   %[x_sub]      \n\t"
+      "lw     %[frac],    0(%[irow])                \n\t"
+      "subu   %[loop_c],  %[loop_c],  %[x_stride]   \n\t"
+      "addu   %[temp3],   %[base],    %[temp3]      \n\t"
+      "sw     %[temp3],   0(%[frow])                \n\t"
+      "addu   %[frow],    %[x_stride1]              \n\t"
+      "addu   %[frac],    %[temp3]                  \n\t"
+      "sw     %[frac],    0(%[irow])                \n\t"
+      "addu   %[irow],    %[x_stride1]              \n\t"
+      "bgtz   %[loop_c],  1b                        \n\t"
+
+      : [src1]"+&r"(src1), [accum]"=&r"(accum), [temp1]"=&r"(temp1),
+        [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [base]"=&r"(base),
+        [frac]"=&r"(frac), [frow]"+&r"(frow), [irow]"+&r"(irow),
+        [loop_c]"+&r"(loop_c)
+      : [x_stride]"r"(x_stride), [x_add]"r"(x_add), [x_sub]"r"(x_sub),
+        [x_stride1]"r"(x_stride1)
+      : "memory", "hi", "lo"
+    );
+  }
+}
+
+static void ExportRow(WebPRescaler* const wrk, int x_out) {
+  if (wrk->y_accum <= 0) {
+    // if wrk->fxy_scale can fit into 32 bits use optimized code,
+    // otherwise use C code
+    if ((wrk->fxy_scale >> 32) == 0) {
+      uint8_t* dst = wrk->dst;
+      int32_t* irow = wrk->irow;
+      const int32_t* frow = wrk->frow;
+      const int yscale = wrk->fy_scale * (-wrk->y_accum);
+      const int x_out_max = wrk->dst_width * wrk->num_channels;
+      int temp0, temp1, temp3, temp4, temp5, temp6, temp7;
+      const int temp2 = (int)wrk->fxy_scale;
+      const int rest = (x_out_max - x_out) & 1;
+      const int32_t* const loop_end = frow + (x_out_max - x_out) - rest;
+
+      __asm__ volatile (
+        ".set             push                                    \n\t"
+        ".set             noreorder                               \n\t"
+        "beq              %[frow],   %[loop_end],   1f            \n\t"
+        " nop                                                     \n\t"
+      "0:                                                         \n\t"
+        "lw               %[temp0],    0(%[frow])                 \n\t"
+        "lw               %[temp1],    0(%[irow])                 \n\t"
+        "lw               %[temp3],    4(%[frow])                 \n\t"
+        "lw               %[temp4],    4(%[irow])                 \n\t"
+        "sll              %[temp0],    %[temp0],      1           \n\t"
+        "sll              %[temp3],    %[temp3],      1           \n\t"
+        "mulq_rs.w        %[temp5],    %[temp0],      %[yscale]   \n\t"
+        "mulq_rs.w        %[temp6],    %[temp3],      %[yscale]   \n\t"
+        "addiu            %[frow],     %[frow],       8           \n\t"
+        "addiu            %[dst],      %[dst],        2           \n\t"
+        "addiu            %[irow],     %[irow],       8           \n\t"
+        "subu             %[temp1],    %[temp1],      %[temp5]    \n\t"
+        "subu             %[temp4],    %[temp4],      %[temp6]    \n\t"
+        "sll              %[temp1],    %[temp1],      1           \n\t"
+        "sll              %[temp4],    %[temp4],      1           \n\t"
+        "mulq_rs.w        %[temp0],    %[temp1],      %[temp2]    \n\t"
+        "mulq_rs.w        %[temp3],    %[temp4],      %[temp2]    \n\t"
+        "sw               %[temp5],    -8(%[irow])                \n\t"
+        "sw               %[temp6],    -4(%[irow])                \n\t"
+        "shll_s.ph        %[temp0],    %[temp0],      7           \n\t"
+        "shll_s.ph        %[temp3],    %[temp3],      7           \n\t"
+        "precrqu_s.qb.ph  %[temp0],    %[temp0],      %[temp3]    \n\t"
+        "sb               %[temp0],    -1(%[dst])                 \n\t"
+        "srl              %[temp0],    %[temp0],      16          \n\t"
+        "bne              %[frow],     %[loop_end],   0b          \n\t"
+        " sb              %[temp0],    -2(%[dst])                 \n\t"
+      "1:                                                         \n\t"
+        "beqz             %[rest],     3f                         \n\t"
+        " nop                                                     \n\t"
+        "addiu            %[temp6],    $zero,         -256        \n\t"
+        "addiu            %[temp7],    $zero,         255         \n\t"
+        "lw               %[temp0],    0(%[frow])                 \n\t"
+        "sll              %[temp0],    %[temp0],      1           \n\t"
+        "mulq_rs.w        %[temp1],    %[temp0],      %[yscale]   \n\t"
+        "lw               %[temp0],    0(%[irow])                 \n\t"
+        "subu             %[temp0],    %[temp0],      %[temp1]    \n\t"
+        "sll              %[temp0],    %[temp0],      1           \n\t"
+        "mulq_rs.w        %[temp5],    %[temp0],      %[temp2]    \n\t"
+        "sw               %[temp1],    0(%[irow])                 \n\t"
+        "and              %[temp0],    %[temp5],      %[temp6]    \n\t"
+        "beqz             %[temp0],    2f                         \n\t"
+        " slti            %[temp1],    %[temp5],      0           \n\t"
+        "xor              %[temp5],    %[temp5],      %[temp5]    \n\t"
+        "movz             %[temp5],    %[temp7],      %[temp1]    \n\t"
+      "2:                                                         \n\t"
+        "sb               %[temp5],    0(%[dst])                  \n\t"
+      "3:                                                         \n\t"
+        ".set             pop                                     \n\t"
+        : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3),
+          [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp6]"=&r"(temp6),
+          [temp7]"=&r"(temp7), [frow]"+&r"(frow), [irow]"+&r"(irow),
+          [dst]"+&r"(dst)
+        : [temp2]"r"(temp2), [yscale]"r"(yscale), [loop_end]"r"(loop_end),
+          [rest]"r"(rest)
+        : "memory", "hi", "lo"
+      );
+      wrk->y_accum += wrk->y_add;
+      wrk->dst += wrk->dst_stride;
+    } else {
+      WebPRescalerExportRowC(wrk, x_out);
+    }
+  }
+}
+
+#endif  // WEBP_USE_MIPS_DSP_R2
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern void WebPRescalerDspInitMIPSdspR2(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMIPSdspR2(void) {
+#if defined(WEBP_USE_MIPS_DSP_R2)
+  WebPRescalerImportRow = ImportRow;
+  WebPRescalerExportRow = ExportRow;
+#endif  // WEBP_USE_MIPS_DSP_R2
+}

data/vendor/FreeImage/Source/LibWebP/src/dsp/dsp.upsampling.c

@@ -0,0 +1,252 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// YUV to RGB upsampling functions.
+//
+// Author: somnath@google.com (Somnath Banerjee)
+
+#include "./dsp.h"
+#include "./yuv.h"
+
+#include <assert.h>
+
+//------------------------------------------------------------------------------
+// Fancy upsampler
+
+#ifdef FANCY_UPSAMPLING
+
+// Fancy upsampling functions to convert YUV to RGB
+WebPUpsampleLinePairFunc WebPUpsamplers[MODE_LAST];
+
+// Given samples laid out in a square as:
+//  [a b]
+//  [c d]
+// we interpolate u/v as:
+//  ([9*a + 3*b + 3*c +   d    3*a + 9*b + 3*c +   d] + [8 8]) / 16
+//  ([3*a +   b + 9*c + 3*d      a + 3*b + 3*c + 9*d]   [8 8]) / 16
+
+// We process u and v together stashed into 32bit (16bit each).
+#define LOAD_UV(u, v) ((u) | ((v) << 16))
+
+#define UPSAMPLE_FUNC(FUNC_NAME, FUNC, XSTEP)                                  \
+static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y,           \
+                      const uint8_t* top_u, const uint8_t* top_v,              \
+                      const uint8_t* cur_u, const uint8_t* cur_v,              \
+                      uint8_t* top_dst, uint8_t* bottom_dst, int len) {        \
+  int x;                                                                       \
+  const int last_pixel_pair = (len - 1) >> 1;                                  \
+  uint32_t tl_uv = LOAD_UV(top_u[0], top_v[0]);   /* top-left sample */        \
+  uint32_t l_uv  = LOAD_UV(cur_u[0], cur_v[0]);   /* left-sample */            \
+  assert(top_y != NULL);                                                       \
+  {                                                                            \
+    const uint32_t uv0 = (3 * tl_uv + l_uv + 0x00020002u) >> 2;                \
+    FUNC(top_y[0], uv0 & 0xff, (uv0 >> 16), top_dst);                          \
+  }                                                                            \
+  if (bottom_y != NULL) {                                                      \
+    const uint32_t uv0 = (3 * l_uv + tl_uv + 0x00020002u) >> 2;                \
+    FUNC(bottom_y[0], uv0 & 0xff, (uv0 >> 16), bottom_dst);                    \
+  }                                                                            \
+  for (x = 1; x <= last_pixel_pair; ++x) {                                     \
+    const uint32_t t_uv = LOAD_UV(top_u[x], top_v[x]);  /* top sample */       \
+    const uint32_t uv   = LOAD_UV(cur_u[x], cur_v[x]);  /* sample */           \
+    /* precompute invariant values associated with first and second diagonals*/\
+    const uint32_t avg = tl_uv + t_uv + l_uv + uv + 0x00080008u;               \
+    const uint32_t diag_12 = (avg + 2 * (t_uv + l_uv)) >> 3;                   \
+    const uint32_t diag_03 = (avg + 2 * (tl_uv + uv)) >> 3;                    \
+    {                                                                          \
+      const uint32_t uv0 = (diag_12 + tl_uv) >> 1;                             \
+      const uint32_t uv1 = (diag_03 + t_uv) >> 1;                              \
+      FUNC(top_y[2 * x - 1], uv0 & 0xff, (uv0 >> 16),                          \
+           top_dst + (2 * x - 1) * XSTEP);                                     \
+      FUNC(top_y[2 * x - 0], uv1 & 0xff, (uv1 >> 16),                          \
+           top_dst + (2 * x - 0) * XSTEP);                                     \
+    }                                                                          \
+    if (bottom_y != NULL) {                                                    \
+      const uint32_t uv0 = (diag_03 + l_uv) >> 1;                              \
+      const uint32_t uv1 = (diag_12 + uv) >> 1;                                \
+      FUNC(bottom_y[2 * x - 1], uv0 & 0xff, (uv0 >> 16),                       \
+           bottom_dst + (2 * x - 1) * XSTEP);                                  \
+      FUNC(bottom_y[2 * x + 0], uv1 & 0xff, (uv1 >> 16),                       \
+           bottom_dst + (2 * x + 0) * XSTEP);                                  \
+    }                                                                          \
+    tl_uv = t_uv;                                                              \
+    l_uv = uv;                                                                 \
+  }                                                                            \
+  if (!(len & 1)) {                                                            \
+    {                                                                          \
+      const uint32_t uv0 = (3 * tl_uv + l_uv + 0x00020002u) >> 2;              \
+      FUNC(top_y[len - 1], uv0 & 0xff, (uv0 >> 16),                            \
+           top_dst + (len - 1) * XSTEP);                                       \
+    }                                                                          \
+    if (bottom_y != NULL) {                                                    \
+      const uint32_t uv0 = (3 * l_uv + tl_uv + 0x00020002u) >> 2;              \
+      FUNC(bottom_y[len - 1], uv0 & 0xff, (uv0 >> 16),                         \
+           bottom_dst + (len - 1) * XSTEP);                                    \
+    }                                                                          \
+  }                                                                            \
+}
+
+// All variants implemented.
+UPSAMPLE_FUNC(UpsampleRgbLinePair,  VP8YuvToRgb,  3)
+UPSAMPLE_FUNC(UpsampleBgrLinePair,  VP8YuvToBgr,  3)
+UPSAMPLE_FUNC(UpsampleRgbaLinePair, VP8YuvToRgba, 4)
+UPSAMPLE_FUNC(UpsampleBgraLinePair, VP8YuvToBgra, 4)
+UPSAMPLE_FUNC(UpsampleArgbLinePair, VP8YuvToArgb, 4)
+UPSAMPLE_FUNC(UpsampleRgba4444LinePair, VP8YuvToRgba4444, 2)
+UPSAMPLE_FUNC(UpsampleRgb565LinePair,  VP8YuvToRgb565,  2)
+
+#undef LOAD_UV
+#undef UPSAMPLE_FUNC
+
+#endif  // FANCY_UPSAMPLING
+
+//------------------------------------------------------------------------------
+
+#if !defined(FANCY_UPSAMPLING)
+#define DUAL_SAMPLE_FUNC(FUNC_NAME, FUNC)                                      \
+static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bot_y,              \
+                      const uint8_t* top_u, const uint8_t* top_v,              \
+                      const uint8_t* bot_u, const uint8_t* bot_v,              \
+                      uint8_t* top_dst, uint8_t* bot_dst, int len) {           \
+  const int half_len = len >> 1;                                               \
+  int x;                                                                       \
+  assert(top_dst != NULL);                                                     \
+  {                                                                            \
+    for (x = 0; x < half_len; ++x) {                                           \
+      FUNC(top_y[2 * x + 0], top_u[x], top_v[x], top_dst + 8 * x + 0);         \
+      FUNC(top_y[2 * x + 1], top_u[x], top_v[x], top_dst + 8 * x + 4);         \
+    }                                                                          \
+    if (len & 1) FUNC(top_y[2 * x + 0], top_u[x], top_v[x], top_dst + 8 * x);  \
+  }                                                                            \
+  if (bot_dst != NULL) {                                                       \
+    for (x = 0; x < half_len; ++x) {                                           \
+      FUNC(bot_y[2 * x + 0], bot_u[x], bot_v[x], bot_dst + 8 * x + 0);         \
+      FUNC(bot_y[2 * x + 1], bot_u[x], bot_v[x], bot_dst + 8 * x + 4);         \
+    }                                                                          \
+    if (len & 1) FUNC(bot_y[2 * x + 0], bot_u[x], bot_v[x], bot_dst + 8 * x);  \
+  }                                                                            \
+}
+
+DUAL_SAMPLE_FUNC(DualLineSamplerBGRA, VP8YuvToBgra)
+DUAL_SAMPLE_FUNC(DualLineSamplerARGB, VP8YuvToArgb)
+#undef DUAL_SAMPLE_FUNC
+
+#endif  // !FANCY_UPSAMPLING
+
+WebPUpsampleLinePairFunc WebPGetLinePairConverter(int alpha_is_last) {
+  WebPInitUpsamplers();
+  VP8YUVInit();
+#ifdef FANCY_UPSAMPLING
+  return WebPUpsamplers[alpha_is_last ? MODE_BGRA : MODE_ARGB];
+#else
+  return (alpha_is_last ? DualLineSamplerBGRA : DualLineSamplerARGB);
+#endif
+}
+
+//------------------------------------------------------------------------------
+// YUV444 converter
+
+#define YUV444_FUNC(FUNC_NAME, FUNC, XSTEP)                                    \
+static void FUNC_NAME(const uint8_t* y, const uint8_t* u, const uint8_t* v,    \
+                      uint8_t* dst, int len) {                                 \
+  int i;                                                                       \
+  for (i = 0; i < len; ++i) FUNC(y[i], u[i], v[i], &dst[i * XSTEP]);           \
+}
+
+YUV444_FUNC(Yuv444ToRgb,      VP8YuvToRgb,  3)
+YUV444_FUNC(Yuv444ToBgr,      VP8YuvToBgr,  3)
+YUV444_FUNC(Yuv444ToRgba,     VP8YuvToRgba, 4)
+YUV444_FUNC(Yuv444ToBgra,     VP8YuvToBgra, 4)
+YUV444_FUNC(Yuv444ToArgb,     VP8YuvToArgb, 4)
+YUV444_FUNC(Yuv444ToRgba4444, VP8YuvToRgba4444, 2)
+YUV444_FUNC(Yuv444ToRgb565,   VP8YuvToRgb565, 2)
+
+#undef YUV444_FUNC
+
+WebPYUV444Converter WebPYUV444Converters[MODE_LAST];
+
+extern void WebPInitYUV444ConvertersMIPSdspR2(void);
+
+static volatile VP8CPUInfo upsampling_last_cpuinfo_used1 =
+    (VP8CPUInfo)&upsampling_last_cpuinfo_used1;
+
+WEBP_TSAN_IGNORE_FUNCTION void WebPInitYUV444Converters(void) {
+  if (upsampling_last_cpuinfo_used1 == VP8GetCPUInfo) return;
+
+  WebPYUV444Converters[MODE_RGB]       = Yuv444ToRgb;
+  WebPYUV444Converters[MODE_RGBA]      = Yuv444ToRgba;
+  WebPYUV444Converters[MODE_BGR]       = Yuv444ToBgr;
+  WebPYUV444Converters[MODE_BGRA]      = Yuv444ToBgra;
+  WebPYUV444Converters[MODE_ARGB]      = Yuv444ToArgb;
+  WebPYUV444Converters[MODE_RGBA_4444] = Yuv444ToRgba4444;
+  WebPYUV444Converters[MODE_RGB_565]   = Yuv444ToRgb565;
+  WebPYUV444Converters[MODE_rgbA]      = Yuv444ToRgba;
+  WebPYUV444Converters[MODE_bgrA]      = Yuv444ToBgra;
+  WebPYUV444Converters[MODE_Argb]      = Yuv444ToArgb;
+  WebPYUV444Converters[MODE_rgbA_4444] = Yuv444ToRgba4444;
+
+  if (VP8GetCPUInfo != NULL) {
+#if defined(WEBP_USE_MIPS_DSP_R2)
+    if (VP8GetCPUInfo(kMIPSdspR2)) {
+      WebPInitYUV444ConvertersMIPSdspR2();
+    }
+#endif
+  }
+  upsampling_last_cpuinfo_used1 = VP8GetCPUInfo;
+}
+
+//------------------------------------------------------------------------------
+// Main calls
+
+extern void WebPInitUpsamplersSSE2(void);
+extern void WebPInitUpsamplersNEON(void);
+extern void WebPInitUpsamplersMIPSdspR2(void);
+
+static volatile VP8CPUInfo upsampling_last_cpuinfo_used2 =
+    (VP8CPUInfo)&upsampling_last_cpuinfo_used2;
+
+WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplers(void) {
+  if (upsampling_last_cpuinfo_used2 == VP8GetCPUInfo) return;
+
+#ifdef FANCY_UPSAMPLING
+  WebPUpsamplers[MODE_RGB]       = UpsampleRgbLinePair;
+  WebPUpsamplers[MODE_RGBA]      = UpsampleRgbaLinePair;
+  WebPUpsamplers[MODE_BGR]       = UpsampleBgrLinePair;
+  WebPUpsamplers[MODE_BGRA]      = UpsampleBgraLinePair;
+  WebPUpsamplers[MODE_ARGB]      = UpsampleArgbLinePair;
+  WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair;
+  WebPUpsamplers[MODE_RGB_565]   = UpsampleRgb565LinePair;
+  WebPUpsamplers[MODE_rgbA]      = UpsampleRgbaLinePair;
+  WebPUpsamplers[MODE_bgrA]      = UpsampleBgraLinePair;
+  WebPUpsamplers[MODE_Argb]      = UpsampleArgbLinePair;
+  WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair;
+
+  // If defined, use CPUInfo() to overwrite some pointers with faster versions.
+  if (VP8GetCPUInfo != NULL) {
+#if defined(WEBP_USE_SSE2)
+    if (VP8GetCPUInfo(kSSE2)) {
+      WebPInitUpsamplersSSE2();
+    }
+#endif
+#if defined(WEBP_USE_NEON)
+    if (VP8GetCPUInfo(kNEON)) {
+      WebPInitUpsamplersNEON();
+    }
+#endif
+#if defined(WEBP_USE_MIPS_DSP_R2)
+    if (VP8GetCPUInfo(kMIPSdspR2)) {
+      WebPInitUpsamplersMIPSdspR2();
+    }
+#endif
+  }
+#endif  // FANCY_UPSAMPLING
+  upsampling_last_cpuinfo_used2 = VP8GetCPUInfo;
+}
+
+//------------------------------------------------------------------------------