gitx.do 0.0.1 → 0.0.3

package/dist/tiered/parquet-writer.js

@@ -1,49 +1,212 @@
 /**
- * Parquet Writer for Git Analytics
+ * @fileoverview Parquet Writer for Git Analytics
  *
- * Provides functionality to write git analytics data to Parquet format:
- * - Schema definition with various field types
- * - Compression support (SNAPPY, GZIP, ZSTD, LZ4, UNCOMPRESSED)
- * - Row group management
- * - Metadata handling with statistics
+ * @description
+ * Provides functionality to write git analytics data to Parquet format, a
+ * columnar storage format optimized for analytical queries. This module
+ * enables efficient storage and querying of Git repository data.
  *
- * gitdo-6rz: Parquet writer implementation
+ * **Key Features:**
+ * - Schema definition with various field types (STRING, INT32, INT64, etc.)
+ * - Multiple compression algorithms (SNAPPY, GZIP, ZSTD, LZ4, UNCOMPRESSED)
+ * - Row group management for efficient columnar storage
+ * - Automatic and manual row group flushing
+ * - Column-level statistics generation (min, max, null count)
+ * - Custom key-value metadata support
+ * - Memory-efficient streaming writes
+ *
+ * **Parquet Format:**
+ * The generated files follow the Parquet format with:
+ * - Magic bytes "PAR1" at start and end
+ * - Row group data organized by columns
+ * - Footer metadata containing schema and statistics
+ *
+ * @example
+ * ```typescript
+ * // Define schema for commit analytics
+ * const schema = defineSchema([
+ *   { name: 'commit_sha', type: ParquetFieldType.STRING, required: true },
+ *   { name: 'author', type: ParquetFieldType.STRING, required: true },
+ *   { name: 'timestamp', type: ParquetFieldType.TIMESTAMP_MILLIS, required: true },
+ *   { name: 'file_count', type: ParquetFieldType.INT32, required: false }
+ * ])
+ *
+ * // Create writer with options
+ * const writer = createParquetWriter(schema, {
+ *   rowGroupSize: 10000,
+ *   compression: ParquetCompression.SNAPPY,
+ *   enableStatistics: true
+ * })
+ *
+ * // Write data
+ * await writer.writeRows([
+ *   { commit_sha: 'abc123...', author: 'alice', timestamp: Date.now(), file_count: 5 },
+ *   { commit_sha: 'def456...', author: 'bob', timestamp: Date.now(), file_count: 3 }
+ * ])
+ *
+ * // Generate the Parquet file
+ * const buffer = await writer.toBuffer()
+ * ```
+ *
+ * @module tiered/parquet-writer
+ * @see {@link ParquetWriter} - Main writer class
+ * @see {@link defineSchema} - Schema definition helper
  */
 import pako from 'pako';
 // ============================================================================
 // Types and Enums
 // ============================================================================
 /**
- * Supported Parquet field types
+ * Supported Parquet field types.
+ *
+ * @description
+ * Defines the data types that can be used for fields in a Parquet schema.
+ * Each type maps to an appropriate physical and logical Parquet type.
+ *
+ * @example
+ * ```typescript
+ * const field: ParquetField = {
+ *   name: 'count',
+ *   type: ParquetFieldType.INT64,
+ *   required: true
+ * }
+ * ```
+ *
+ * @enum {string}
  */
 export var ParquetFieldType;
 (function (ParquetFieldType) {
+    /**
+     * UTF-8 encoded string.
+     * Maps to Parquet BYTE_ARRAY with UTF8 logical type.
+     */
     ParquetFieldType["STRING"] = "STRING";
+    /**
+     * 32-bit signed integer.
+     * Maps to Parquet INT32 physical type.
+     */
     ParquetFieldType["INT32"] = "INT32";
+    /**
+     * 64-bit signed integer.
+     * Maps to Parquet INT64 physical type.
+     */
     ParquetFieldType["INT64"] = "INT64";
+    /**
+     * Boolean value (true/false).
+     * Maps to Parquet BOOLEAN physical type.
+     */
     ParquetFieldType["BOOLEAN"] = "BOOLEAN";
+    /**
+     * 32-bit IEEE 754 floating point.
+     * Maps to Parquet FLOAT physical type.
+     */
     ParquetFieldType["FLOAT"] = "FLOAT";
+    /**
+     * 64-bit IEEE 754 floating point.
+     * Maps to Parquet DOUBLE physical type.
+     */
     ParquetFieldType["DOUBLE"] = "DOUBLE";
+    /**
+     * Raw binary data.
+     * Maps to Parquet BYTE_ARRAY physical type.
+     */
     ParquetFieldType["BINARY"] = "BINARY";
+    /**
+     * Timestamp with millisecond precision.
+     * Maps to Parquet INT64 with TIMESTAMP_MILLIS logical type.
+     */
     ParquetFieldType["TIMESTAMP_MILLIS"] = "TIMESTAMP_MILLIS";
+    /**
+     * Timestamp with microsecond precision.
+     * Maps to Parquet INT64 with TIMESTAMP_MICROS logical type.
+     */
     ParquetFieldType["TIMESTAMP_MICROS"] = "TIMESTAMP_MICROS";
 })(ParquetFieldType || (ParquetFieldType = {}));
 /**
- * Supported compression types
+ * Supported compression types for Parquet data.
+ *
+ * @description
+ * Different compression algorithms offer trade-offs between compression
+ * ratio, compression speed, and decompression speed.
+ *
+ * **Comparison:**
+ * - SNAPPY: Fast compression/decompression, moderate ratio (default)
+ * - GZIP: Higher ratio, slower compression, fast decompression
+ * - ZSTD: Best ratio, good speed, requires more memory
+ * - LZ4: Fastest, lower ratio
+ * - UNCOMPRESSED: No compression overhead
+ *
+ * @example
+ * ```typescript
+ * const writer = createParquetWriter(schema, {
+ *   compression: ParquetCompression.ZSTD
+ * })
+ * ```
+ *
+ * @enum {string}
  */
 export var ParquetCompression;
 (function (ParquetCompression) {
+    /**
+     * No compression applied.
+     * Fastest writes, largest file size.
+     */
     ParquetCompression["UNCOMPRESSED"] = "UNCOMPRESSED";
+    /**
+     * Snappy compression (default).
+     * Good balance of speed and compression ratio.
+     */
     ParquetCompression["SNAPPY"] = "SNAPPY";
+    /**
+     * GZIP compression.
+     * Higher compression ratio, slower compression.
+     */
     ParquetCompression["GZIP"] = "GZIP";
+    /**
+     * Zstandard compression.
+     * Best compression ratio with good speed.
+     */
     ParquetCompression["ZSTD"] = "ZSTD";
+    /**
+     * LZ4 compression.
+     * Fastest compression, lower ratio.
+     */
     ParquetCompression["LZ4"] = "LZ4";
 })(ParquetCompression || (ParquetCompression = {}));
 /**
- * Error class for Parquet operations
+ * Error class for Parquet-related operations.
+ *
+ * @description
+ * Thrown when Parquet operations fail, such as schema validation errors,
+ * invalid data types, or malformed files.
+ *
+ * @example
+ * ```typescript
+ * try {
+ *   await writer.writeRow({ invalid_field: 'value' })
+ * } catch (error) {
+ *   if (error instanceof ParquetError) {
+ *     console.log(`Parquet error (${error.code}): ${error.message}`)
+ *   }
+ * }
+ * ```
+ *
+ * @class ParquetError
+ * @extends Error
  */
 export class ParquetError extends Error {
     code;
+    /**
+     * Creates a new ParquetError.
+     *
+     * @param message - Human-readable error message
+     * @param code - Error code for programmatic handling
+     *
+     * @example
+     * ```typescript
+     * throw new ParquetError('Field name cannot be empty', 'EMPTY_FIELD_NAME')
+     * ```
+     */
     constructor(message, code) {
         super(message);
         this.code = code;
@@ -54,17 +217,125 @@ export class ParquetError extends Error {
 // ParquetWriter Class
 // ============================================================================
 /**
- * Parquet writer for git analytics data
+ * Parquet writer for git analytics data.
+ *
+ * @description
+ * ParquetWriter provides a streaming interface for writing data to Parquet
+ * format. It handles schema validation, row group management, compression,
+ * and statistics generation.
+ *
+ * **Usage Pattern:**
+ * 1. Create a schema using `defineSchema()`
+ * 2. Create a writer with `createParquetWriter()` or `new ParquetWriter()`
+ * 3. Write rows using `writeRow()` or `writeRows()`
+ * 4. Generate the file with `toBuffer()` or `writeTo()`
+ *
+ * **Row Group Management:**
+ * Rows are buffered in memory until the row group is full (by row count
+ * or memory limit), then flushed. You can also manually flush with
+ * `flushRowGroup()`.
+ *
+ * **Thread Safety:**
+ * Not thread-safe. Use separate writer instances for concurrent writes.
+ *
+ * @example
+ * ```typescript
+ * // Create schema
+ * const schema = defineSchema([
+ *   { name: 'sha', type: ParquetFieldType.STRING, required: true },
+ *   { name: 'type', type: ParquetFieldType.STRING, required: true },
+ *   { name: 'size', type: ParquetFieldType.INT64, required: true },
+ *   { name: 'timestamp', type: ParquetFieldType.TIMESTAMP_MILLIS, required: true }
+ * ])
+ *
+ * // Create writer
+ * const writer = new ParquetWriter(schema, {
+ *   rowGroupSize: 10000,
+ *   compression: ParquetCompression.SNAPPY,
+ *   enableStatistics: true
+ * })
+ *
+ * // Write data
+ * for (const object of gitObjects) {
+ *   await writer.writeRow({
+ *     sha: object.sha,
+ *     type: object.type,
+ *     size: object.size,
+ *     timestamp: Date.now()
+ *   })
+ * }
+ *
+ * // Set custom metadata
+ * writer.setMetadata('git_version', '2.40.0')
+ * writer.setMetadata('repository', 'github.com/org/repo')
+ *
+ * // Generate file
+ * const buffer = await writer.toBuffer()
+ * console.log(`Generated ${buffer.length} bytes`)
+ * console.log(`Rows: ${writer.rowCount}`)
+ * console.log(`Row groups: ${writer.rowGroupCount}`)
+ *
+ * // Reset for reuse
+ * writer.reset()
+ * ```
+ *
+ * @class ParquetWriter
  */
 export class ParquetWriter {
+    /**
+     * The Parquet schema for this writer.
+     * @readonly
+     */
     schema;
+    /**
+     * Resolved options with defaults applied.
+     * @readonly
+     */
     options;
+    /**
+     * Total row count written.
+     * @private
+     */
     _rowCount = 0;
+    /**
+     * Completed row groups.
+     * @private
+     */
     _rowGroups = [];
+    /**
+     * Current row group being built.
+     * @private
+     */
     _currentRowGroup = { rows: [], byteSize: 0 };
+    /**
+     * Whether the writer has been closed.
+     * @private
+     */
     _isClosed = false;
+    /**
+     * Custom key-value metadata.
+     * @private
+     */
     _keyValueMetadata = {};
+    /**
+     * Creation timestamp.
+     * @private
+     */
     _createdAt = Date.now();
+    /**
+     * Creates a new ParquetWriter instance.
+     *
+     * @param schema - The Parquet schema defining columns
+     * @param options - Writer configuration options
+     *
+     * @example
+     * ```typescript
+     * const writer = new ParquetWriter(schema, {
+     *   rowGroupSize: 50000,
+     *   compression: ParquetCompression.GZIP
+     * })
+     * ```
+     */
     constructor(schema, options = {}) {
         this.schema = schema;
         this.options = {
@@ -74,26 +345,83 @@ export class ParquetWriter {
         };
     }
     /**
-     * Get total row count
+     * Gets the total row count written to the writer.
+     *
+     * @description
+     * Returns the total number of rows written, including rows in the
+     * current unflushed row group.
+     *
+     * @returns Total row count
+     *
+     * @example
+     * ```typescript
+     * await writer.writeRows(data)
+     * console.log(`Wrote ${writer.rowCount} rows`)
+     * ```
      */
     get rowCount() {
         return this._rowCount;
     }
     /**
-     * Get number of row groups (including current pending row group if non-empty)
+     * Gets the number of row groups.
+     *
+     * @description
+     * Returns the number of completed row groups plus one if there's
+     * a pending row group with data.
+     *
+     * @returns Number of row groups
+     *
+     * @example
+     * ```typescript
+     * console.log(`Row groups: ${writer.rowGroupCount}`)
+     * ```
      */
     get rowGroupCount() {
         const pendingCount = this._currentRowGroup.rows.length > 0 ? 1 : 0;
         return this._rowGroups.length + pendingCount;
     }
     /**
-     * Check if writer is closed
+     * Checks if the writer has been closed.
+     *
+     * @description
+     * A closed writer cannot accept new rows. Writers are closed
+     * implicitly by `closeWriter()`.
+     *
+     * @returns true if closed
+     *
+     * @example
+     * ```typescript
+     * if (!writer.isClosed) {
+     *   await writer.writeRow(row)
+     * }
+     * ```
      */
     get isClosed() {
         return this._isClosed;
     }
     /**
-     * Write a single row
+     * Writes a single row to the Parquet file.
+     *
+     * @description
+     * Validates the row against the schema and adds it to the current
+     * row group. Automatically flushes the row group when it reaches
+     * the configured size or memory limit.
+     *
+     * @param row - Object with column values keyed by column name
+     * @returns Promise that resolves when the row is written
+     *
+     * @throws {ParquetError} WRITER_CLOSED - If writer is closed
+     * @throws {ParquetError} MISSING_REQUIRED_FIELD - If required field is missing
+     * @throws {ParquetError} INVALID_FIELD_TYPE - If field value type doesn't match schema
+     *
+     * @example
+     * ```typescript
+     * await writer.writeRow({
+     *   id: 123,
+     *   name: 'Alice',
+     *   active: true
+     * })
+     * ```
      */
     async writeRow(row) {
         if (this._isClosed) {
@@ -115,7 +443,25 @@ export class ParquetWriter {
         }
     }
     /**
-     * Write multiple rows at once
+     * Writes multiple rows to the Parquet file.
+     *
+     * @description
+     * Convenience method that writes an array of rows sequentially.
+     * Each row is validated and may trigger row group flushes.
+     *
+     * @param rows - Array of row objects to write
+     * @returns Promise that resolves when all rows are written
+     *
+     * @throws {ParquetError} Any error from writeRow()
+     *
+     * @example
+     * ```typescript
+     * await writer.writeRows([
+     *   { id: 1, name: 'Alice' },
+     *   { id: 2, name: 'Bob' },
+     *   { id: 3, name: 'Carol' }
+     * ])
+     * ```
      */
     async writeRows(rows) {
         for (const row of rows) {
@@ -123,7 +469,26 @@ export class ParquetWriter {
         }
     }
     /**
-     * Manually flush the current row group
+     * Manually flushes the current row group.
+     *
+     * @description
+     * Forces the current row group to be finalized and stored, even if
+     * it hasn't reached the size limit. Has no effect if the current
+     * row group is empty.
+     *
+     * @returns Promise that resolves when flush is complete
+     *
+     * @example
+     * ```typescript
+     * // Write some rows
+     * await writer.writeRows(batch1)
+     *
+     * // Force flush before writing next batch
+     * await writer.flushRowGroup()
+     *
+     * // Continue writing
+     * await writer.writeRows(batch2)
+     * ```
      */
     async flushRowGroup() {
         if (this._currentRowGroup.rows.length === 0) {
@@ -134,25 +499,79 @@ export class ParquetWriter {
         this._currentRowGroup = { rows: [], byteSize: 0 };
     }
     /**
-     * Get the current row group memory size
+     * Gets the current row group's memory size.
+     *
+     * @description
+     * Returns the estimated memory consumption of the unflushed row group.
+     * Useful for monitoring memory usage during streaming writes.
+     *
+     * @returns Memory size in bytes
+     *
+     * @example
+     * ```typescript
+     * if (writer.currentRowGroupMemorySize() > 50 * 1024 * 1024) {
+     *   console.log('Row group using significant memory')
+     *   await writer.flushRowGroup()
+     * }
+     * ```
      */
     currentRowGroupMemorySize() {
         return this._currentRowGroup.byteSize;
     }
     /**
-     * Get completed row groups
+     * Gets the completed row groups.
+     *
+     * @description
+     * Returns a copy of the completed row group metadata array.
+     * Does not include the current unflushed row group.
+     *
+     * @returns Array of row group metadata
+     *
+     * @example
+     * ```typescript
+     * for (const rg of writer.getRowGroups()) {
+     *   console.log(`Row group: ${rg.numRows} rows, ${rg.totalByteSize} bytes`)
+     * }
+     * ```
      */
     getRowGroups() {
         return [...this._rowGroups];
     }
     /**
-     * Set custom key-value metadata
+     * Sets a custom key-value metadata entry.
+     *
+     * @description
+     * Adds custom metadata that will be stored in the Parquet file footer.
+     * Can be used for versioning, provenance, or application-specific data.
+     *
+     * @param key - Metadata key
+     * @param value - Metadata value
+     *
+     * @example
+     * ```typescript
+     * writer.setMetadata('created_by', 'gitdo-analytics')
+     * writer.setMetadata('schema_version', '2.0')
+     * writer.setMetadata('repository', 'github.com/org/repo')
+     * ```
      */
     setMetadata(key, value) {
         this._keyValueMetadata[key] = value;
     }
     /**
-     * Generate the Parquet file as a buffer
+     * Generates the Parquet file as a buffer.
+     *
+     * @description
+     * Finalizes the file by flushing any remaining rows and generating
+     * the complete Parquet file structure including header, row groups,
+     * and footer with metadata.
+     *
+     * @returns Promise resolving to the complete Parquet file as Uint8Array
+     *
+     * @example
+     * ```typescript
+     * const buffer = await writer.toBuffer()
+     * await fs.writeFile('data.parquet', buffer)
+     * ```
      */
     async toBuffer() {
         // Flush any remaining rows
@@ -162,14 +581,45 @@ export class ParquetWriter {
         return this._generateParquetBytes();
     }
     /**
-     * Write to an output stream
+     * Writes the Parquet file to an output stream.
+     *
+     * @description
+     * Generates the file and writes it to the provided output stream.
+     * Useful for streaming to files or network destinations.
+     *
+     * @param output - The output stream to write to
+     * @returns Promise that resolves when writing is complete
+     *
+     * @example
+     * ```typescript
+     * const output = new FileOutputStream('data.parquet')
+     * await writer.writeTo(output)
+     * output.close()
+     * ```
      */
     async writeTo(output) {
         const bytes = await this.toBuffer();
         output.write(bytes);
     }
     /**
-     * Reset the writer state
+     * Resets the writer to its initial state.
+     *
+     * @description
+     * Clears all written data, row groups, and metadata. The schema
+     * and options remain unchanged. Useful for writing multiple files
+     * with the same configuration.
+     *
+     * @example
+     * ```typescript
+     * // Write first file
+     * await writer.writeRows(batch1)
+     * const file1 = await writer.toBuffer()
+     *
+     * // Reset and write second file
+     * writer.reset()
+     * await writer.writeRows(batch2)
+     * const file2 = await writer.toBuffer()
+     * ```
      */
     reset() {
         this._rowCount = 0;
@@ -180,7 +630,11 @@ export class ParquetWriter {
         this._createdAt = Date.now();
     }
     /**
-     * Validate a row against the schema
+     * Validates a row against the schema.
+     *
+     * @param row - The row to validate
+     * @throws {ParquetError} If validation fails
+     * @private
      */
     _validateRow(row) {
         for (const field of this.schema.fields) {
@@ -198,7 +652,12 @@ export class ParquetWriter {
         }
     }
     /**
-     * Validate a value matches the expected type
+     * Validates a value matches the expected Parquet type.
+     *
+     * @param value - The value to validate
+     * @param type - The expected Parquet type
+     * @returns true if valid, false otherwise
+     * @private
      */
     _validateType(value, type) {
         switch (type) {
@@ -220,7 +679,11 @@ export class ParquetWriter {
         }
     }
     /**
-     * Estimate the memory size of a row
+     * Estimates the memory size of a row.
+     *
+     * @param row - The row to estimate
+     * @returns Estimated size in bytes
+     * @private
      */
     _estimateRowSize(row) {
         let size = 0;
@@ -245,7 +708,11 @@ export class ParquetWriter {
         return size;
     }
     /**
-     * Build a row group from internal representation
+     * Builds a row group from internal representation.
+     *
+     * @param internal - The internal row group data
+     * @returns The row group metadata
+     * @private
      */
     _buildRowGroup(internal) {
         const columns = this.schema.fields.map(field => {
@@ -268,7 +735,12 @@ export class ParquetWriter {
         };
     }
     /**
-     * Compute statistics for a column
+     * Computes statistics for a column.
+     *
+     * @param values - The column values
+     * @param type - The column type
+     * @returns Column statistics
+     * @private
      */
     _computeStatistics(values, type) {
         const nonNullValues = values.filter(v => v !== null && v !== undefined);
@@ -309,7 +781,13 @@ export class ParquetWriter {
         }
     }
     /**
-     * Estimate encoded size after compression
+     * Estimates the encoded size after compression.
+     *
+     * @param values - The column values
+     * @param type - The column type
+     * @param compression - The compression type
+     * @returns Estimated compressed size in bytes
+     * @private
      */
     _estimateEncodedSize(values, type, compression) {
         const uncompressedSize = this._estimateUncompressedSize(values, type);
@@ -329,7 +807,12 @@ export class ParquetWriter {
         }
     }
     /**
-     * Estimate uncompressed size
+     * Estimates the uncompressed size of column values.
+     *
+     * @param values - The column values
+     * @param type - The column type
+     * @returns Estimated uncompressed size in bytes
+     * @private
      */
     _estimateUncompressedSize(values, type) {
         let size = 0;
@@ -364,7 +847,10 @@ export class ParquetWriter {
         return size;
     }
     /**
-     * Generate the complete Parquet file bytes
+     * Generates the complete Parquet file bytes.
+     *
+     * @returns The complete Parquet file as Uint8Array
+     * @private
      */
     _generateParquetBytes() {
         // Build all row data - will be populated from row groups in full implementation
@@ -414,7 +900,12 @@ export class ParquetWriter {
         return result;
     }
     /**
-     * Simple compression simulation for non-gzip formats
+     * Simple compression simulation for non-gzip formats.
+     *
+     * @param data - Data to compress
+     * @param compression - Compression type
+     * @returns Compressed data
+     * @private
      */
     _simpleCompress(data, compression) {
         if (compression === ParquetCompression.UNCOMPRESSED) {
@@ -434,7 +925,34 @@ export class ParquetWriter {
 // Helper Functions
 // ============================================================================
 /**
- * Define a Parquet schema
+ * Defines a Parquet schema.
+ *
+ * @description
+ * Creates a validated Parquet schema from field definitions. Validates that:
+ * - Schema has at least one field
+ * - All field names are non-empty
+ * - All field names are unique
+ *
+ * @param fields - Array of field definitions
+ * @param metadata - Optional schema-level metadata
+ * @returns Validated Parquet schema
+ *
+ * @throws {ParquetError} EMPTY_SCHEMA - If fields array is empty
+ * @throws {ParquetError} EMPTY_FIELD_NAME - If any field name is empty
+ * @throws {ParquetError} DUPLICATE_FIELD - If field names are not unique
+ *
+ * @example
+ * ```typescript
+ * const schema = defineSchema([
+ *   { name: 'id', type: ParquetFieldType.INT64, required: true },
+ *   { name: 'name', type: ParquetFieldType.STRING, required: true },
+ *   { name: 'age', type: ParquetFieldType.INT32, required: false },
+ *   { name: 'created_at', type: ParquetFieldType.TIMESTAMP_MILLIS, required: true }
+ * ], {
+ *   version: '1.0',
+ *   description: 'User records'
+ * })
+ * ```
  */
 export function defineSchema(fields, metadata) {
     // Validate schema
@@ -462,13 +980,50 @@ export function defineSchema(fields, metadata) {
     };
 }
 /**
- * Create a Parquet writer
+ * Creates a Parquet writer.
+ *
+ * @description
+ * Factory function to create a ParquetWriter with the specified schema
+ * and options. Equivalent to `new ParquetWriter(schema, options)`.
+ *
+ * @param schema - The Parquet schema
+ * @param options - Writer options
+ * @returns A new ParquetWriter instance
+ *
+ * @example
+ * ```typescript
+ * const writer = createParquetWriter(schema, {
+ *   rowGroupSize: 10000,
+ *   compression: ParquetCompression.SNAPPY
+ * })
+ * ```
  */
 export function createParquetWriter(schema, options = {}) {
     return new ParquetWriter(schema, options);
 }
 /**
- * Write data directly to a Parquet file
+ * Writes data directly to a Parquet file buffer.
+ *
+ * @description
+ * Convenience function that creates a writer, writes all rows, and returns
+ * the complete Parquet file. Useful for simple one-shot writes.
+ *
+ * @param schema - The Parquet schema
+ * @param rows - Array of rows to write
+ * @param options - Writer options
+ * @returns Promise resolving to the complete Parquet file as Uint8Array
+ *
+ * @example
+ * ```typescript
+ * const buffer = await writeParquetFile(schema, [
+ *   { id: 1, name: 'Alice' },
+ *   { id: 2, name: 'Bob' }
+ * ], {
+ *   compression: ParquetCompression.GZIP
+ * })
+ *
+ * await fs.writeFile('data.parquet', buffer)
+ * ```
  */
 export async function writeParquetFile(schema, rows, options = {}) {
     const writer = createParquetWriter(schema, options);
@@ -476,7 +1031,21 @@ export async function writeParquetFile(schema, rows, options = {}) {
     return writer.toBuffer();
 }
 /**
- * Close a writer and return the final buffer
+ * Closes a writer and returns the final buffer.
+ *
+ * @description
+ * Generates the final Parquet file buffer and marks the writer as closed.
+ * The writer cannot be used for further writes after calling this function.
+ *
+ * @param writer - The ParquetWriter to close
+ * @returns Promise resolving to the complete Parquet file as Uint8Array
+ *
+ * @example
+ * ```typescript
+ * await writer.writeRows(data)
+ * const buffer = await closeWriter(writer)
+ * console.log(writer.isClosed)  // true
+ * ```
  */
 export async function closeWriter(writer) {
     const bytes = await writer.toBuffer();
@@ -484,14 +1053,52 @@ export async function closeWriter(writer) {
     return bytes;
 }
 /**
- * Add a row group to the writer
+ * Adds a row group to the writer.
+ *
+ * @description
+ * Writes multiple rows and then flushes them as a single row group.
+ * Useful when you want explicit control over row group boundaries.
+ *
+ * @param writer - The ParquetWriter to use
+ * @param rows - Array of rows for this row group
+ * @returns Promise that resolves when the row group is written
+ *
+ * @example
+ * ```typescript
+ * // Add explicit row groups
+ * await addRowGroup(writer, batch1)  // First row group
+ * await addRowGroup(writer, batch2)  // Second row group
+ * ```
  */
 export async function addRowGroup(writer, rows) {
     await writer.writeRows(rows);
     await writer.flushRowGroup();
 }
 /**
- * Get metadata from a Parquet file buffer
+ * Gets metadata from a Parquet file buffer.
+ *
+ * @description
+ * Parses a Parquet file buffer and extracts the metadata including
+ * schema, row groups, compression settings, and custom metadata.
+ *
+ * @param bytes - The Parquet file buffer
+ * @returns The parsed metadata
+ *
+ * @throws {ParquetError} INVALID_MAGIC - If file doesn't have valid Parquet magic bytes
+ *
+ * @example
+ * ```typescript
+ * const buffer = await fs.readFile('data.parquet')
+ * const metadata = getMetadata(buffer)
+ *
+ * console.log(`Rows: ${metadata.numRows}`)
+ * console.log(`Schema: ${metadata.schema.fields.map(f => f.name).join(', ')}`)
+ * console.log(`Row groups: ${metadata.rowGroups.length}`)
+ *
+ * for (const rg of metadata.rowGroups) {
+ *   console.log(`  - ${rg.numRows} rows, ${rg.totalByteSize} bytes`)
+ * }
+ * ```
  */
 export function getMetadata(bytes) {
     // Verify magic bytes
@@ -546,7 +1153,28 @@ export function getMetadata(bytes) {
     };
 }
 /**
- * Set compression for a writer
+ * Sets the compression type for a writer.
+ *
+ * @description
+ * Updates the default compression algorithm for a writer. Affects all
+ * subsequently written data. Columns with explicit compression settings
+ * in columnCompression are not affected.
+ *
+ * @param writer - The ParquetWriter to update
+ * @param compression - The new compression type
+ *
+ * @example
+ * ```typescript
+ * const writer = createParquetWriter(schema)
+ *
+ * // Write some rows with SNAPPY (default)
+ * await writer.writeRows(batch1)
+ * await writer.flushRowGroup()
+ *
+ * // Switch to GZIP for remaining data
+ * setCompression(writer, ParquetCompression.GZIP)
+ * await writer.writeRows(batch2)
+ * ```
  */
 export function setCompression(writer, compression) {
     ;