@img/sharp-libvips-dev 0.0.1

package/include/aom/aom_external_partition.h

@@ -0,0 +1,452 @@
+/*
+ * Copyright (c) 2021, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+#ifndef AOM_AOM_AOM_EXTERNAL_PARTITION_H_
+#define AOM_AOM_AOM_EXTERNAL_PARTITION_H_
+
+/*!\defgroup aom_encoder AOMedia AOM/AV1 Encoder
+ * \ingroup aom
+ *
+ * @{
+ */
+#include <stdint.h>
+
+/*!\file
+ * \brief Provides function pointer definitions for the external partition.
+ *
+ * \note The external partition API should be considered experimental. Until the
+ * external partition API is declared stable, breaking changes may be made to
+ * this API in a future libaom release.
+ */
+
+/*!\brief Current ABI version number
+ *
+ * \internal
+ * If this file is altered in any way that changes the ABI, this value
+ * must be bumped. Examples include, but are not limited to, changing
+ * types, removing or reassigning enums, adding/removing/rearranging
+ * fields to structures.
+ */
+#define AOM_EXT_PART_ABI_VERSION 8
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*!\brief Abstract external partition model handler
+ */
+typedef void *aom_ext_part_model_t;
+
+/*!\brief Number of features to determine whether to skip partition none and
+ * do partition split directly. The same as "FEATURE_SIZE_SMS_SPLIT".
+ */
+#define AOM_EXT_PART_SIZE_DIRECT_SPLIT 17
+
+/*!\brief Number of features to use simple motion search to prune out
+ * rectangular partition in some direction. The same as
+ * "FEATURE_SIZE_SMS_PRUNE_PART".
+ */
+#define AOM_EXT_PART_SIZE_PRUNE_PART 25
+
+/*!\brief Number of features to prune split and rectangular partition
+ * after PARTITION_NONE.
+ */
+#define AOM_EXT_PART_SIZE_PRUNE_NONE 4
+
+/*!\brief Number of features to terminates partition after partition none using
+ * simple_motion_search features and the rate, distortion, and rdcost of
+ * PARTITION_NONE. The same as "FEATURE_SIZE_SMS_TERM_NONE".
+ */
+#define AOM_EXT_PART_SIZE_TERM_NONE 28
+
+/*!\brief Number of features to terminates partition after partition split.
+ */
+#define AOM_EXT_PART_SIZE_TERM_SPLIT 31
+
+/*!\brief Number of features to prune rectangular partition using stats
+ * collected after partition split.
+ */
+#define AOM_EXT_PART_SIZE_PRUNE_RECT 9
+
+/*!\brief Number of features to prune AB partition using stats
+ * collected after rectangular partition..
+ */
+#define AOM_EXT_PART_SIZE_PRUNE_AB 10
+
+/*!\brief Number of features to prune 4-way partition using stats
+ * collected after AB partition.
+ */
+#define AOM_EXT_PART_SIZE_PRUNE_4_WAY 18
+
+/*!\brief Decision mode of the external partition model.
+ * AOM_EXT_PART_WHOLE_TREE: the external partition model should provide the
+ * whole partition tree for the superblock.
+ *
+ * AOM_EXT_PART_RECURSIVE: the external partition model provides the partition
+ * decision of the current block only. The decision process starts from
+ * the superblock size, down to the smallest block size (4x4) recursively.
+ */
+typedef enum aom_ext_part_decision_mode {
+  AOM_EXT_PART_WHOLE_TREE = 0,
+  AOM_EXT_PART_RECURSIVE = 1,
+} aom_ext_part_decision_mode_t;
+
+/*!\brief Config information sent to the external partition model.
+ *
+ * For example, the maximum superblock size determined by the sequence header.
+ */
+typedef struct aom_ext_part_config {
+  int superblock_size;  ///< super block size (either 64x64 or 128x128)
+} aom_ext_part_config_t;
+
+/*!\brief Features pass to the external model to make partition decisions.
+ * Specifically, features collected before NONE partition.
+ * Features "f" are used to determine:
+ * partition_none_allowed, partition_horz_allowed, partition_vert_allowed,
+ * do_rectangular_split, do_square_split
+ * Features "f_part2" are used to determine:
+ * prune_horz, prune_vert.
+ */
+typedef struct aom_partition_features_before_none {
+  /*! features to determine whether skip partition none and do split directly */
+  float f[AOM_EXT_PART_SIZE_DIRECT_SPLIT];
+  /*! features to determine whether to prune rectangular partition */
+  float f_part2[AOM_EXT_PART_SIZE_PRUNE_PART];
+} aom_partition_features_before_none_t;
+
+/*!\brief Features pass to the external model to make partition decisions.
+ * Specifically, features collected after NONE partition.
+ */
+typedef struct aom_partition_features_none {
+  /*! features to prune split and rectangular partition */
+  float f[AOM_EXT_PART_SIZE_PRUNE_NONE];
+  /*! features to determine termination of partition */
+  float f_terminate[AOM_EXT_PART_SIZE_TERM_NONE];
+} aom_partition_features_none_t;
+
+/*!\brief Features pass to the external model to make partition decisions.
+ * Specifically, features collected after SPLIT partition.
+ */
+typedef struct aom_partition_features_split {
+  /*! features to determine termination of  partition */
+  float f_terminate[AOM_EXT_PART_SIZE_TERM_SPLIT];
+  /*! features to determine pruning rect partition */
+  float f_prune_rect[AOM_EXT_PART_SIZE_PRUNE_RECT];
+} aom_partition_features_split_t;
+
+/*!\brief Features pass to the external model to make partition decisions.
+ * Specifically, features collected after RECTANGULAR partition.
+ */
+typedef struct aom_partition_features_rect {
+  /*! features to determine pruning AB partition */
+  float f[AOM_EXT_PART_SIZE_PRUNE_AB];
+} aom_partition_features_rect_t;
+
+/*!\brief Features pass to the external model to make partition decisions.
+ * Specifically, features collected after AB partition: HORZ_A, HORZ_B, VERT_A,
+ * VERT_B.
+ */
+typedef struct aom_partition_features_ab {
+  /*! features to determine pruning 4-way partition */
+  float f[AOM_EXT_PART_SIZE_PRUNE_4_WAY];
+} aom_partition_features_ab_t;
+
+/*!\brief Feature id to tell the external model the current stage in partition
+ * pruning and what features to use to make decisions accordingly.
+ */
+typedef enum {
+  AOM_EXT_PART_FEATURE_BEFORE_NONE,
+  AOM_EXT_PART_FEATURE_BEFORE_NONE_PART2,
+  AOM_EXT_PART_FEATURE_AFTER_NONE,
+  AOM_EXT_PART_FEATURE_AFTER_NONE_PART2,
+  AOM_EXT_PART_FEATURE_AFTER_SPLIT,
+  AOM_EXT_PART_FEATURE_AFTER_SPLIT_PART2,
+  AOM_EXT_PART_FEATURE_AFTER_RECT,
+  AOM_EXT_PART_FEATURE_AFTER_AB
+} AOM_EXT_PART_FEATURE_ID;
+
+/*!\brief Features collected from the tpl process.
+ *
+ * The tpl process collects information that help measure the inter-frame
+ * dependency.
+ * The tpl process is computed in the unit of tpl_bsize_1d (16x16).
+ * Therefore, the max number of units inside a superblock is
+ * 128x128 / (16x16) = 64. Change it if the tpl process changes.
+ */
+typedef struct aom_sb_tpl_features {
+  int available;        ///< If tpl stats are available
+  int tpl_unit_length;  ///< The block length of tpl process
+  int num_units;        ///< The number of units inside the current superblock
+  int64_t intra_cost[64];   ///< The intra cost of each unit
+  int64_t inter_cost[64];   ///< The inter cost of each unit
+  int64_t mc_dep_cost[64];  ///< The motion compensated dependency cost
+} aom_sb_tpl_features_t;
+
+/*!\brief Features collected from the simple motion process.
+ *
+ * The simple motion process collects information by applying motion compensated
+ * prediction on each block.
+ * The block size is 16x16, which could be changed. If it is changed, update
+ * comments and the array size here.
+ */
+typedef struct aom_sb_simple_motion_features {
+  int unit_length;    ///< The block length of the simple motion process
+  int num_units;      ///< The number of units inside the current superblock
+  int block_sse[64];  ///< Sum of squared error of each unit
+  int block_var[64];  ///< Variance of each unit
+} aom_sb_simple_motion_features_t;
+
+/*!\brief Features of each super block.
+ *
+ * Features collected for each super block before partition search.
+ */
+typedef struct aom_sb_features {
+  /*! Features from motion search */
+  aom_sb_simple_motion_features_t motion_features;
+  /*! Features from tpl process */
+  aom_sb_tpl_features_t tpl_features;
+} aom_sb_features_t;
+
+/*!\brief Features pass to the external model to make partition decisions.
+ *
+ * The encoder sends these features to the external model through
+ * "func()" defined in .....
+ *
+ * NOTE: new member variables may be added to this structure in the future.
+ * Once new features are finalized, bump the major version of libaom.
+ */
+typedef struct aom_partition_features {
+  // Features for the current supervised multi-stage ML model.
+  /*! Feature ID to indicate active features */
+  AOM_EXT_PART_FEATURE_ID id;
+  /*! Features collected before NONE partition */
+  aom_partition_features_before_none_t before_part_none;
+  /*! Features collected after NONE partition */
+  aom_partition_features_none_t after_part_none;
+  /*! Features collected after SPLIT partition */
+  aom_partition_features_split_t after_part_split;
+  /*! Features collected after RECTANGULAR partition */
+  aom_partition_features_rect_t after_part_rect;
+  /*! Features collected after AB partition */
+  aom_partition_features_ab_t after_part_ab;
+
+  // Features for a new ML model.
+  aom_sb_features_t sb_features;  ///< Features collected for the super block
+  int mi_row;                     ///< Mi_row position of the block
+  int mi_col;                     ///< Mi_col position of the block
+  int frame_width;                ///< Frame width
+  int frame_height;               ///< Frame height
+  int block_size;                 ///< As "BLOCK_SIZE" in av1/common/enums.h
+  /*!
+   * Valid partition types. A bitmask is used.  "1" represents the
+   * corresponding type is valid. The bitmask follows the enum order for
+   * PARTITION_TYPE in "enums.h" to represent one partition type at a bit.
+   * For example, 0x01 stands for only PARTITION_NONE is valid,
+   * 0x09 (00...001001) stands for PARTITION_NONE and PARTITION_SPLIT are valid.
+   */
+  int valid_partition_types;
+  int update_type;    ///< Frame update type, defined in ratectrl.h
+  int qindex;         ///< Quantization index, range: [0, 255]
+  int rdmult;         ///< Rate-distortion multiplier
+  int pyramid_level;  ///< The level of this frame in the hierarchical structure
+  int has_above_block;     ///< Has above neighbor block
+  int above_block_width;   ///< Width of the above block, -1 if not exist
+  int above_block_height;  ///< Height of the above block, -1 if not exist
+  int has_left_block;      ///< Has left neighbor block
+  int left_block_width;    ///< Width of the left block, -1 if not exist
+  int left_block_height;   ///< Height of the left block, -1 if not exist
+  /*!
+   * The following parameters are collected from applying simple motion search.
+   * Sum of squared error (SSE) and variance of motion compensated residual
+   * are good indicators of block partitioning.
+   * If a block is a square, we also apply motion search for its 4 sub blocks.
+   * If not a square, their values are -1.
+   * If a block is able to split horizontally, we apply motion search and get
+   * stats for horizontal blocks. If not, their values are -1.
+   * If a block is able to split vertically, we apply motion search and get
+   * stats for vertical blocks. If not, their values are -1.
+   */
+  unsigned int block_sse;          ///< SSE of motion compensated residual
+  unsigned int block_var;          ///< Variance of motion compensated residual
+  unsigned int sub_block_sse[4];   ///< SSE of sub blocks.
+  unsigned int sub_block_var[4];   ///< Variance of sub blocks.
+  unsigned int horz_block_sse[2];  ///< SSE of horz sub blocks
+  unsigned int horz_block_var[2];  ///< Variance of horz sub blocks
+  unsigned int vert_block_sse[2];  ///< SSE of vert sub blocks
+  unsigned int vert_block_var[2];  ///< Variance of vert sub blocks
+  /*!
+   * The following parameters are calculated from tpl model.
+   * If tpl model is not available, their values are -1.
+   */
+  int64_t tpl_intra_cost;   ///< Intra cost, ref to "TplDepStats" in tpl_model.h
+  int64_t tpl_inter_cost;   ///< Inter cost in tpl model
+  int64_t tpl_mc_dep_cost;  ///< Motion compensated dependency cost in tpl model
+} aom_partition_features_t;
+
+/*!\brief Partition decisions received from the external model.
+ *
+ * The encoder receives partition decisions and encodes the superblock
+ * with the given partition type.
+ * The encoder receives it from "func()" define in ....
+ *
+ * NOTE: new member variables may be added to this structure in the future.
+ * Once new features are finalized, bump the major version of libaom.
+ */
+typedef struct aom_partition_decision {
+  // Decisions for directly set partition types
+  int is_final_decision;         ///< The flag whether it's the final decision
+  int num_nodes;                 ///< The number of leaf nodes
+  int partition_decision[2048];  ///< Partition decisions
+  int current_decision;          ///< Partition decision for the current block
+
+  // Decisions for partition type pruning
+  int terminate_partition_search;  ///< Terminate further partition search
+  int partition_none_allowed;      ///< Allow partition none type
+  int partition_rect_allowed[2];   ///< Allow rectangular partitions
+  int do_rectangular_split;        ///< Try rectangular split partition
+  int do_square_split;             ///< Try square split partition
+  int prune_rect_part[2];          ///< Prune rectangular partition
+  int horza_partition_allowed;     ///< Allow HORZ_A partition
+  int horzb_partition_allowed;     ///< Allow HORZ_B partition
+  int verta_partition_allowed;     ///< Allow VERT_A partition
+  int vertb_partition_allowed;     ///< Allow VERT_B partition
+  int partition_horz4_allowed;     ///< Allow HORZ4 partition
+  int partition_vert4_allowed;     ///< Allow VERT4 partition
+} aom_partition_decision_t;
+
+/*!\brief Encoding stats for the given partition decision.
+ *
+ * The encoding stats collected by encoding the superblock with the
+ * given partition types.
+ * The encoder sends the stats to the external model for training
+ * or inference through "func()" defined in ....
+ */
+typedef struct aom_partition_stats {
+  int rate;        ///< Rate cost of the block
+  int64_t dist;    ///< Distortion of the block
+  int64_t rdcost;  ///< Rate-distortion cost of the block
+} aom_partition_stats_t;
+
+/*!\brief Enum for return status.
+ */
+typedef enum aom_ext_part_status {
+  AOM_EXT_PART_OK = 0,     ///< Status of success
+  AOM_EXT_PART_ERROR = 1,  ///< Status of failure
+  AOM_EXT_PART_TEST = 2,   ///< Status used for tests
+} aom_ext_part_status_t;
+
+/*!\brief Callback of creating an external partition model.
+ *
+ * The callback is invoked by the encoder to create an external partition
+ * model.
+ *
+ * \param[in] priv Callback's private data
+ * \param[in] part_config Config information pointer for model creation
+ * \param[out] ext_part_model Pointer to the model
+ */
+typedef aom_ext_part_status_t (*aom_ext_part_create_model_fn_t)(
+    void *priv, const aom_ext_part_config_t *part_config,
+    aom_ext_part_model_t *ext_part_model);
+
+/*!\brief Callback of sending features to the external partition model.
+ *
+ * The callback is invoked by the encoder to send features to the external
+ * partition model.
+ *
+ * \param[in] ext_part_model The external model
+ * \param[in] part_features Pointer to the features
+ */
+typedef aom_ext_part_status_t (*aom_ext_part_send_features_fn_t)(
+    aom_ext_part_model_t ext_part_model,
+    const aom_partition_features_t *part_features);
+
+/*!\brief Callback of receiving partition decisions from the external
+ * partition model.
+ *
+ * The callback is invoked by the encoder to receive partition decisions from
+ * the external partition model.
+ *
+ * \param[in] ext_part_model The external model
+ * \param[in] ext_part_decision Pointer to the partition decisions
+ */
+typedef aom_ext_part_status_t (*aom_ext_part_get_decision_fn_t)(
+    aom_ext_part_model_t ext_part_model,
+    aom_partition_decision_t *ext_part_decision);
+
+/*!\brief Callback of sending stats to the external partition model.
+ *
+ * The callback is invoked by the encoder to send encoding stats to
+ * the external partition model.
+ *
+ * \param[in] ext_part_model The external model
+ * \param[in] ext_part_stats Pointer to the encoding stats
+ */
+typedef aom_ext_part_status_t (*aom_ext_part_send_partition_stats_fn_t)(
+    aom_ext_part_model_t ext_part_model,
+    const aom_partition_stats_t *ext_part_stats);
+
+/*!\brief Callback of deleting the external partition model.
+ *
+ * The callback is invoked by the encoder to delete the external partition
+ * model.
+ *
+ * \param[in] ext_part_model The external model
+ */
+typedef aom_ext_part_status_t (*aom_ext_part_delete_model_fn_t)(
+    aom_ext_part_model_t ext_part_model);
+
+/*!\brief Callback function set for external partition model.
+ *
+ * Uses can enable external partition model by registering a set of
+ * callback functions with the flag: AV1E_SET_EXTERNAL_PARTITION_MODEL
+ */
+typedef struct aom_ext_part_funcs {
+  /*!
+   * Create an external partition model.
+   */
+  aom_ext_part_create_model_fn_t create_model;
+
+  /*!
+   * Send features to the external partition model to make partition decisions.
+   */
+  aom_ext_part_send_features_fn_t send_features;
+
+  /*!
+   * Get partition decisions from the external partition model.
+   */
+  aom_ext_part_get_decision_fn_t get_partition_decision;
+
+  /*!
+   * Send stats of the current partition to the external model.
+   */
+  aom_ext_part_send_partition_stats_fn_t send_partition_stats;
+
+  /*!
+   * Delete the external partition model.
+   */
+  aom_ext_part_delete_model_fn_t delete_model;
+
+  /*!
+   * The decision mode of the model.
+   */
+  aom_ext_part_decision_mode_t decision_mode;
+
+  /*!
+   * Private data for the external partition model.
+   */
+  void *priv;
+} aom_ext_part_funcs_t;
+
+/*!@} - end defgroup aom_encoder*/
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // AOM_AOM_AOM_EXTERNAL_PARTITION_H_

package/include/aom/aom_frame_buffer.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2016, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AOM_AOM_FRAME_BUFFER_H_
+#define AOM_AOM_AOM_FRAME_BUFFER_H_
+
+/*!\file
+ * \brief Describes the decoder external frame buffer interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "aom/aom_integer.h"
+
+/*!\brief The maximum number of work buffers used by libaom.
+ *  Support maximum 4 threads to decode video in parallel.
+ *  Each thread will use one work buffer.
+ * TODO(hkuang): Add support to set number of worker threads dynamically.
+ */
+#define AOM_MAXIMUM_WORK_BUFFERS 8
+
+/*!\brief The maximum number of reference buffers that a AV1 encoder may use.
+ */
+#define AOM_MAXIMUM_REF_BUFFERS 8
+
+/*!\brief External frame buffer
+ *
+ * This structure holds allocated frame buffers used by the decoder.
+ */
+typedef struct aom_codec_frame_buffer {
+  uint8_t *data; /**< Pointer to the data buffer */
+  size_t size;   /**< Size of data in bytes */
+  void *priv;    /**< Frame's private data */
+} aom_codec_frame_buffer_t;
+
+/*!\brief get frame buffer callback prototype
+ *
+ * This callback is invoked by the decoder to retrieve data for the frame
+ * buffer in order for the decode call to complete. The callback must
+ * allocate at least min_size in bytes and assign it to fb->data. The callback
+ * must zero out all the data allocated. Then the callback must set fb->size
+ * to the allocated size. The application does not need to align the allocated
+ * data. The callback is triggered when the decoder needs a frame buffer to
+ * decode a compressed image into. This function may be called more than once
+ * for every call to aom_codec_decode. The application may set fb->priv to
+ * some data which will be passed back in the aom_image_t and the release
+ * function call. |fb| is guaranteed to not be NULL. On success the callback
+ * must return 0. Any failure the callback must return a value less than 0.
+ *
+ * \param[in] priv         Callback's private data
+ * \param[in] min_size     Size in bytes needed by the buffer
+ * \param[in,out] fb       Pointer to aom_codec_frame_buffer_t
+ */
+typedef int (*aom_get_frame_buffer_cb_fn_t)(void *priv, size_t min_size,
+                                            aom_codec_frame_buffer_t *fb);
+
+/*!\brief release frame buffer callback prototype
+ *
+ * This callback is invoked by the decoder when the frame buffer is not
+ * referenced by any other buffers. |fb| is guaranteed to not be NULL. On
+ * success the callback must return 0. Any failure the callback must return
+ * a value less than 0.
+ *
+ * \param[in] priv         Callback's private data
+ * \param[in] fb           Pointer to aom_codec_frame_buffer_t
+ */
+typedef int (*aom_release_frame_buffer_cb_fn_t)(void *priv,
+                                                aom_codec_frame_buffer_t *fb);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // AOM_AOM_AOM_FRAME_BUFFER_H_