electrobun 0.0.1 → 0.0.2

package/src/bsdiff/zstd/contrib/seekable_format/zstd_seekable.h

@@ -1,226 +0,0 @@
-#ifndef SEEKABLE_H
-#define SEEKABLE_H
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-#include <stdio.h>
-#include "zstd.h"   /* ZSTDLIB_API */
-
-
-#define ZSTD_seekTableFooterSize 9
-
-#define ZSTD_SEEKABLE_MAGICNUMBER 0x8F92EAB1
-
-#define ZSTD_SEEKABLE_MAXFRAMES 0x8000000U
-
-/* Limit maximum size to avoid potential issues storing the compressed size */
-#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0x40000000U
-
-/*-****************************************************************************
-*  Seekable Format
-*
-*  The seekable format splits the compressed data into a series of "frames",
-*  each compressed individually so that decompression of a section in the
-*  middle of an archive only requires zstd to decompress at most a frame's
-*  worth of extra data, instead of the entire archive.
-******************************************************************************/
-
-typedef struct ZSTD_seekable_CStream_s ZSTD_seekable_CStream;
-typedef struct ZSTD_seekable_s ZSTD_seekable;
-typedef struct ZSTD_seekTable_s ZSTD_seekTable;
-
-/*-****************************************************************************
-*  Seekable compression - HowTo
-*  A ZSTD_seekable_CStream object is required to tracking streaming operation.
-*  Use ZSTD_seekable_createCStream() and ZSTD_seekable_freeCStream() to create/
-*  release resources.
-*
-*  Streaming objects are reusable to avoid allocation and deallocation,
-*  to start a new compression operation call ZSTD_seekable_initCStream() on the
-*  compressor.
-*
-*  Data streamed to the seekable compressor will automatically be split into
-*  frames of size `maxFrameSize` (provided in ZSTD_seekable_initCStream()),
-*  or if none is provided, will be cut off whenever ZSTD_seekable_endFrame() is
-*  called or when the default maximum frame size (2GB) is reached.
-*
-*  Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object
-*  for a new compression operation.
-*  - `maxFrameSize` indicates the size at which to automatically start a new
-*            seekable frame.
-*            `maxFrameSize == 0` implies the default maximum size.
-*            Smaller frame sizes allow faster decompression of small segments,
-*            since retrieving a single byte requires decompression of
-*            the full frame where the byte belongs.
-*            In general, size the frames to roughly correspond to
-*            the access granularity (when it's known).
-*            But small sizes also reduce compression ratio.
-*            Avoid really tiny frame sizes (< 1 KB),
-*            that would hurt compression ratio considerably.
-*  - `checksumFlag` indicates whether or not the seek table should include frame
-*            checksums on the uncompressed data for verification.
-*  @return : a size hint for input to provide for compression, or an error code
-*            checkable with ZSTD_isError()
-*
-*  Use ZSTD_seekable_compressStream() repetitively to consume input stream.
-*  The function will automatically update both `pos` fields.
-*  Note that it may not consume the entire input, in which case `pos < size`,
-*  and it's up to the caller to present again remaining data.
-*  @return : a size hint, preferred nb of bytes to use as input for next
-*            function call or an error code, which can be tested using
-*            ZSTD_isError().
-*            Note 1 : it's just a hint, to help latency a little, any other
-*                     value will work fine.
-*
-*  At any time, call ZSTD_seekable_endFrame() to end the current frame and
-*  start a new one.
-*
-*  ZSTD_seekable_endStream() will end the current frame, and then write the seek
-*  table so that decompressors can efficiently find compressed frames.
-*  ZSTD_seekable_endStream() may return a number > 0 if it was unable to flush
-*  all the necessary data to `output`.  In this case, it should be called again
-*  until all remaining data is flushed out and 0 is returned.
-******************************************************************************/
-
-/*===== Seekable compressor management =====*/
-ZSTDLIB_API ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void);
-ZSTDLIB_API size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs);
-
-/*===== Seekable compression functions =====*/
-ZSTDLIB_API size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxFrameSize);
-ZSTDLIB_API size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-ZSTDLIB_API size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
-ZSTDLIB_API size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
-
-/*= Raw seek table API
- *  These functions allow for the seek table to be constructed directly.
- *  This table can then be appended to a file of concatenated frames.
- *  This allows the frames to be compressed independently, even in parallel,
- *  and compiled together afterward into a seekable archive.
- *
- *  Use ZSTD_seekable_createFrameLog() to allocate and initialize a tracking
- *  structure.
- *
- *  Call ZSTD_seekable_logFrame() once for each frame in the archive.
- *  checksum is optional, and will not be used if checksumFlag was 0 when the
- *  frame log was created.  If present, it should be the least significant 32
- *  bits of the XXH64 hash of the uncompressed data.
- *
- *  Call ZSTD_seekable_writeSeekTable to serialize the data into a seek table.
- *  If the entire table was written, the return value will be 0.  Otherwise,
- *  it will be equal to the number of bytes left to write. */
-typedef struct ZSTD_frameLog_s ZSTD_frameLog;
-ZSTDLIB_API ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag);
-ZSTDLIB_API size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl);
-ZSTDLIB_API size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, unsigned compressedSize, unsigned decompressedSize, unsigned checksum);
-ZSTDLIB_API size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output);
-
-
-/*-****************************************************************************
-*  Seekable decompression - HowTo
-*  A ZSTD_seekable object is required to tracking the seekTable.
-*
-*  Call ZSTD_seekable_init* to initialize a ZSTD_seekable object with the
-*  the seek table provided in the input.
-*  There are three modes for ZSTD_seekable_init:
-*    - ZSTD_seekable_initBuff() : An in-memory API.  The data contained in
-*      `src` should be the entire seekable file, including the seek table.
-*      `src` should be kept alive and unmodified until the ZSTD_seekable object
-*      is freed or reset.
-*    - ZSTD_seekable_initFile() : A simplified file API using stdio.  fread and
-*      fseek will be used to access the required data for building the seek
-*      table and doing decompression operations.  `src` should not be closed
-*      or modified until the ZSTD_seekable object is freed or reset.
-*    - ZSTD_seekable_initAdvanced() : A general API allowing the client to
-*      provide its own read and seek callbacks.
-*        + ZSTD_seekable_read() : read exactly `n` bytes into `buffer`.
-*                                 Premature EOF should be treated as an error.
-*        + ZSTD_seekable_seek() : seek the read head to `offset` from `origin`,
-*                                 where origin is either SEEK_SET (beginning of
-*                                 file), or SEEK_END (end of file).
-*  Both functions should return a non-negative value in case of success, and a
-*  negative value in case of failure.  If implementing using this API and
-*  stdio, be careful with files larger than 4GB and fseek.  All of these
-*  functions return an error code checkable with ZSTD_isError().
-*
-*  Call ZSTD_seekable_decompress to decompress `dstSize` bytes at decompressed
-*  offset `offset`.  ZSTD_seekable_decompress may have to decompress the entire
-*  prefix of the frame before the desired data if it has not already processed
-*  this section. If ZSTD_seekable_decompress is called multiple times for a
-*  consecutive range of data, it will efficiently retain the decompressor object
-*  and avoid redecompressing frame prefixes.  The return value is the number of
-*  bytes decompressed, or an error code checkable with ZSTD_isError().
-*
-*  The seek table access functions can be used to obtain the data contained
-*  in the seek table.  If frameIndex is larger than the value returned by
-*  ZSTD_seekable_getNumFrames(), they will return error codes checkable with
-*  ZSTD_isError().  Note that since the offset access functions return
-*  unsigned long long instead of size_t, in this case they will instead return
-*  the value ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE.
-******************************************************************************/
-
-/*===== Seekable decompressor management =====*/
-ZSTDLIB_API ZSTD_seekable* ZSTD_seekable_create(void);
-ZSTDLIB_API size_t ZSTD_seekable_free(ZSTD_seekable* zs);
-
-/*===== Seekable decompression functions =====*/
-ZSTDLIB_API size_t ZSTD_seekable_initBuff(ZSTD_seekable* zs, const void* src, size_t srcSize);
-ZSTDLIB_API size_t ZSTD_seekable_initFile(ZSTD_seekable* zs, FILE* src);
-ZSTDLIB_API size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned long long offset);
-ZSTDLIB_API size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex);
-
-#define ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE (0ULL-2)
-/*===== Seekable seek table access functions =====*/
-ZSTDLIB_API unsigned ZSTD_seekable_getNumFrames(const ZSTD_seekable* zs);
-ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameCompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex);
-ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameDecompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex);
-ZSTDLIB_API size_t ZSTD_seekable_getFrameCompressedSize(const ZSTD_seekable* zs, unsigned frameIndex);
-ZSTDLIB_API size_t ZSTD_seekable_getFrameDecompressedSize(const ZSTD_seekable* zs, unsigned frameIndex);
-ZSTDLIB_API unsigned ZSTD_seekable_offsetToFrameIndex(const ZSTD_seekable* zs, unsigned long long offset);
-
-
-/*-****************************************************************************
-*  Direct exploitation of the seekTable
-*
-*  Memory constrained use cases that manage multiple archives
-*  benefit from retaining multiple archive seek tables
-*  without retaining a ZSTD_seekable instance for each.
-*
-*  Below API allow the above-mentioned use cases
-*  to initialize a ZSTD_seekable, extract its (smaller) ZSTD_seekTable,
-*  then throw the ZSTD_seekable away to save memory.
-*
-*  Standard ZSTD operations can then be used
-*  to decompress frames based on seek table offsets.
-******************************************************************************/
-
-/*===== Independent seek table management =====*/
-ZSTDLIB_API ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs);
-ZSTDLIB_API size_t ZSTD_seekTable_free(ZSTD_seekTable* st);
-
-/*===== Direct seek table access functions =====*/
-ZSTDLIB_API unsigned ZSTD_seekTable_getNumFrames(const ZSTD_seekTable* st);
-ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameCompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex);
-ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameDecompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex);
-ZSTDLIB_API size_t ZSTD_seekTable_getFrameCompressedSize(const ZSTD_seekTable* st, unsigned frameIndex);
-ZSTDLIB_API size_t ZSTD_seekTable_getFrameDecompressedSize(const ZSTD_seekTable* st, unsigned frameIndex);
-ZSTDLIB_API unsigned ZSTD_seekTable_offsetToFrameIndex(const ZSTD_seekTable* st, unsigned long long offset);
-
-
-/*===== Seekable advanced I/O API =====*/
-typedef int(ZSTD_seekable_read)(void* opaque, void* buffer, size_t n);
-typedef int(ZSTD_seekable_seek)(void* opaque, long long offset, int origin);
-typedef struct {
-    void* opaque;
-    ZSTD_seekable_read* read;
-    ZSTD_seekable_seek* seek;
-} ZSTD_seekable_customFile;
-ZSTDLIB_API size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile src);
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif

package/src/bsdiff/zstd/contrib/seekable_format/zstd_seekable_compression_format.md

@@ -1,116 +0,0 @@
-# Zstandard Seekable Format
-
-### Notices
-
-Copyright (c) Meta Platforms, Inc. and affiliates.
-
-Permission is granted to copy and distribute this document
-for any purpose and without charge,
-including translations into other languages
-and incorporation into compilations,
-provided that the copyright notice and this notice are preserved,
-and that any substantive changes or deletions from the original
-are clearly marked.
-Distribution of this document is unlimited.
-
-### Version
-0.1.0 (11/04/17)
-
-## Introduction
-This document defines a format for compressed data to be stored so that subranges of the data can be efficiently decompressed without requiring the entire document to be decompressed.
-This is done by splitting up the input data into frames,
-each of which are compressed independently,
-and so can be decompressed independently.
-Decompression then takes advantage of a provided 'seek table', which allows the decompressor to immediately jump to the desired data.  This is done in a way that is compatible with the original Zstandard format by placing the seek table in a Zstandard skippable frame.
-
-### Overall conventions
-In this document:
-- square brackets i.e. `[` and `]` are used to indicate optional fields or parameters.
-- the naming convention for identifiers is `Mixed_Case_With_Underscores`
-- All numeric fields are little-endian unless specified otherwise
-
-## Format
-
-The format consists of a number of frames (Zstandard compressed frames and skippable frames), followed by a final skippable frame at the end containing the seek table.
-
-### Seek Table Format
-The structure of the seek table frame is as follows:
-
-|`Skippable_Magic_Number`|`Frame_Size`|`[Seek_Table_Entries]`|`Seek_Table_Footer`|
-|------------------------|------------|----------------------|-------------------|
-| 4 bytes                | 4 bytes    | 8-12 bytes each      | 9 bytes           |
-
-__`Skippable_Magic_Number`__
-
-Value : 0x184D2A5E.
-This is for compatibility with [Zstandard skippable frames].
-Since it is legal for other Zstandard skippable frames to use the same
-magic number, it is not recommended for a decoder to recognize frames
-solely on this.
-
-__`Frame_Size`__
-
-The total size of the skippable frame, not including the `Skippable_Magic_Number` or `Frame_Size`.
-This is for compatibility with [Zstandard skippable frames].
-
-[Zstandard skippable frames]: https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#skippable-frames
-
-#### `Seek_Table_Footer`
-The seek table footer format is as follows:
-
-|`Number_Of_Frames`|`Seek_Table_Descriptor`|`Seekable_Magic_Number`|
-|------------------|-----------------------|-----------------------|
-| 4 bytes          | 1 byte                | 4 bytes               |
-
-__`Seekable_Magic_Number`__
-
-Value : 0x8F92EAB1.
-This value must be the last bytes present in the compressed file so that decoders
-can efficiently find it and determine if there is an actual seek table present.
-
-__`Number_Of_Frames`__
-
-The number of stored frames in the data.
-
-__`Seek_Table_Descriptor`__
-
-A bitfield describing the format of the seek table.
-
-| Bit number | Field name                |
-| ---------- | ----------                |
-| 7          | `Checksum_Flag`           |
-| 6-2        | `Reserved_Bits`           |
-| 1-0        | `Unused_Bits`             |
-
-While only `Checksum_Flag` currently exists, there are 7 other bits in this field that can be used for future changes to the format,
-for example the addition of inline dictionaries.
-
-__`Checksum_Flag`__
-
-If the checksum flag is set, each of the seek table entries contains a 4 byte checksum of the uncompressed data contained in its frame.
-
-`Reserved_Bits` are not currently used but may be used in the future for breaking changes, so a compliant decoder should ensure they are set to 0.  `Unused_Bits` may be used in the future for non-breaking changes, so a compliant decoder should not interpret these bits.
-
-#### __`Seek_Table_Entries`__
-
-`Seek_Table_Entries` consists of `Number_Of_Frames` (one for each frame in the data, not including the seek table frame) entries of the following form, in sequence:
-
-|`Compressed_Size`|`Decompressed_Size`|`[Checksum]`|
-|-----------------|-------------------|------------|
-| 4 bytes         | 4 bytes           | 4 bytes    |
-
-__`Compressed_Size`__
-
-The compressed size of the frame.
-The cumulative sum of the `Compressed_Size` fields of frames `0` to `i` gives the offset in the compressed file of frame `i+1`.
-
-__`Decompressed_Size`__
-
-The size of the decompressed data contained in the frame.  For skippable or otherwise empty frames, this value is 0.
-
-__`Checksum`__
-
-Only present if `Checksum_Flag` is set in the `Seek_Table_Descriptor`.  Value : the least significant 32 bits of the XXH64 digest of the uncompressed data, stored in little-endian format.
-
-## Version Changes
-- 0.1.0: initial version