gitx.do 0.0.1

package/dist/tiered/parquet-writer.d.ts

@@ -0,0 +1,248 @@
+/**
+ * Parquet Writer for Git Analytics
+ *
+ * Provides functionality to write git analytics data to Parquet format:
+ * - Schema definition with various field types
+ * - Compression support (SNAPPY, GZIP, ZSTD, LZ4, UNCOMPRESSED)
+ * - Row group management
+ * - Metadata handling with statistics
+ *
+ * gitdo-6rz: Parquet writer implementation
+ */
+/**
+ * Supported Parquet field types
+ */
+export declare enum ParquetFieldType {
+    STRING = "STRING",
+    INT32 = "INT32",
+    INT64 = "INT64",
+    BOOLEAN = "BOOLEAN",
+    FLOAT = "FLOAT",
+    DOUBLE = "DOUBLE",
+    BINARY = "BINARY",
+    TIMESTAMP_MILLIS = "TIMESTAMP_MILLIS",
+    TIMESTAMP_MICROS = "TIMESTAMP_MICROS"
+}
+/**
+ * Supported compression types
+ */
+export declare enum ParquetCompression {
+    UNCOMPRESSED = "UNCOMPRESSED",
+    SNAPPY = "SNAPPY",
+    GZIP = "GZIP",
+    ZSTD = "ZSTD",
+    LZ4 = "LZ4"
+}
+/**
+ * Field definition for schema
+ */
+export interface ParquetField {
+    name: string;
+    type: ParquetFieldType;
+    required: boolean;
+    metadata?: Record<string, string>;
+}
+/**
+ * Parquet schema definition
+ */
+export interface ParquetSchema {
+    fields: ParquetField[];
+    metadata?: Record<string, string>;
+}
+/**
+ * Options for creating a Parquet writer
+ */
+export interface ParquetWriteOptions {
+    rowGroupSize?: number;
+    rowGroupMemoryLimit?: number;
+    compression?: ParquetCompression;
+    columnCompression?: Record<string, ParquetCompression>;
+    enableStatistics?: boolean;
+    sortBy?: string[];
+    partitionColumns?: string[];
+}
+/**
+ * Column statistics
+ */
+export interface ColumnStatistics {
+    min?: number | string | boolean;
+    max?: number | string | boolean;
+    nullCount?: number;
+    distinctCount?: number;
+}
+/**
+ * Column metadata in a row group
+ */
+export interface ColumnChunkMetadata {
+    column: string;
+    type: ParquetFieldType;
+    compression: ParquetCompression;
+    encodedSize: number;
+    uncompressedSize: number;
+    statistics?: ColumnStatistics;
+}
+/**
+ * Row group representation
+ */
+export interface RowGroup {
+    numRows: number;
+    totalByteSize: number;
+    columns: ColumnChunkMetadata[];
+}
+/**
+ * Parquet file metadata
+ */
+export interface ParquetMetadata {
+    schema: ParquetSchema;
+    numRows: number;
+    rowGroups: RowGroup[];
+    compression: ParquetCompression;
+    columnMetadata?: Record<string, {
+        compression: ParquetCompression;
+    }>;
+    keyValueMetadata?: Record<string, string>;
+    createdAt: number;
+    fileSize: number;
+    sortedBy?: string[];
+    partitionColumns?: string[];
+}
+/**
+ * Mock output stream interface
+ */
+export interface OutputStream {
+    write(data: Uint8Array): void;
+}
+/**
+ * Error class for Parquet operations
+ */
+export declare class ParquetError extends Error {
+    readonly code: string;
+    constructor(message: string, code: string);
+}
+/**
+ * Parquet writer for git analytics data
+ */
+export declare class ParquetWriter {
+    readonly schema: ParquetSchema;
+    readonly options: Required<Pick<ParquetWriteOptions, 'rowGroupSize' | 'compression'>> & ParquetWriteOptions;
+    private _rowCount;
+    private _rowGroups;
+    private _currentRowGroup;
+    private _isClosed;
+    private _keyValueMetadata;
+    private _createdAt;
+    constructor(schema: ParquetSchema, options?: ParquetWriteOptions);
+    /**
+     * Get total row count
+     */
+    get rowCount(): number;
+    /**
+     * Get number of row groups (including current pending row group if non-empty)
+     */
+    get rowGroupCount(): number;
+    /**
+     * Check if writer is closed
+     */
+    get isClosed(): boolean;
+    /**
+     * Write a single row
+     */
+    writeRow(row: Record<string, unknown>): Promise<void>;
+    /**
+     * Write multiple rows at once
+     */
+    writeRows(rows: Record<string, unknown>[]): Promise<void>;
+    /**
+     * Manually flush the current row group
+     */
+    flushRowGroup(): Promise<void>;
+    /**
+     * Get the current row group memory size
+     */
+    currentRowGroupMemorySize(): number;
+    /**
+     * Get completed row groups
+     */
+    getRowGroups(): RowGroup[];
+    /**
+     * Set custom key-value metadata
+     */
+    setMetadata(key: string, value: string): void;
+    /**
+     * Generate the Parquet file as a buffer
+     */
+    toBuffer(): Promise<Uint8Array>;
+    /**
+     * Write to an output stream
+     */
+    writeTo(output: OutputStream): Promise<void>;
+    /**
+     * Reset the writer state
+     */
+    reset(): void;
+    /**
+     * Validate a row against the schema
+     */
+    private _validateRow;
+    /**
+     * Validate a value matches the expected type
+     */
+    private _validateType;
+    /**
+     * Estimate the memory size of a row
+     */
+    private _estimateRowSize;
+    /**
+     * Build a row group from internal representation
+     */
+    private _buildRowGroup;
+    /**
+     * Compute statistics for a column
+     */
+    private _computeStatistics;
+    /**
+     * Estimate encoded size after compression
+     */
+    private _estimateEncodedSize;
+    /**
+     * Estimate uncompressed size
+     */
+    private _estimateUncompressedSize;
+    /**
+     * Generate the complete Parquet file bytes
+     */
+    private _generateParquetBytes;
+    /**
+     * Simple compression simulation for non-gzip formats
+     */
+    private _simpleCompress;
+}
+/**
+ * Define a Parquet schema
+ */
+export declare function defineSchema(fields: ParquetField[], metadata?: Record<string, string>): ParquetSchema;
+/**
+ * Create a Parquet writer
+ */
+export declare function createParquetWriter(schema: ParquetSchema, options?: ParquetWriteOptions): ParquetWriter;
+/**
+ * Write data directly to a Parquet file
+ */
+export declare function writeParquetFile(schema: ParquetSchema, rows: Record<string, unknown>[], options?: ParquetWriteOptions): Promise<Uint8Array>;
+/**
+ * Close a writer and return the final buffer
+ */
+export declare function closeWriter(writer: ParquetWriter): Promise<Uint8Array>;
+/**
+ * Add a row group to the writer
+ */
+export declare function addRowGroup(writer: ParquetWriter, rows: Record<string, unknown>[]): Promise<void>;
+/**
+ * Get metadata from a Parquet file buffer
+ */
+export declare function getMetadata(bytes: Uint8Array): ParquetMetadata;
+/**
+ * Set compression for a writer
+ */
+export declare function setCompression(writer: ParquetWriter, compression: ParquetCompression): void;
+//# sourceMappingURL=parquet-writer.d.ts.map

package/dist/tiered/parquet-writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../../src/tiered/parquet-writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAQH;;GAEG;AACH,oBAAY,gBAAgB;IAC1B,MAAM,WAAW;IACjB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,gBAAgB,qBAAqB;IACrC,gBAAgB,qBAAqB;CACtC;AAED;;GAEG;AACH,oBAAY,kBAAkB;IAC5B,YAAY,iBAAiB;IAC7B,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,GAAG,QAAQ;CACZ;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,gBAAgB,CAAA;IACtB,QAAQ,EAAE,OAAO,CAAA;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,YAAY,EAAE,CAAA;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,WAAW,CAAC,EAAE,kBAAkB,CAAA;IAChC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;IACtD,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAA;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAA;IAC/B,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAA;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,gBAAgB,CAAA;IACtB,WAAW,EAAE,kBAAkB,CAAA;IAC/B,WAAW,EAAE,MAAM,CAAA;IACnB,gBAAgB,EAAE,MAAM,CAAA;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAA;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,mBAAmB,EAAE,CAAA;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,aAAa,CAAA;IACrB,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,QAAQ,EAAE,CAAA;IACrB,WAAW,EAAE,kBAAkB,CAAA;IAC/B,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,WAAW,EAAE,kBAAkB,CAAA;KAAE,CAAC,CAAA;IACpE,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;IACnB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAA;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,CAAA;CAC9B;AAED;;GAEG;AACH,qBAAa,YAAa,SAAQ,KAAK;aAGnB,IAAI,EAAE,MAAM;gBAD5B,OAAO,EAAE,MAAM,EACC,IAAI,EAAE,MAAM;CAK/B;AAeD;;GAEG;AACH,qBAAa,aAAa;IACxB,SAAgB,MAAM,EAAE,aAAa,CAAA;IACrC,SAAgB,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,GAAG,aAAa,CAAC,CAAC,GAAG,mBAAmB,CAAA;IAElH,OAAO,CAAC,SAAS,CAAI;IACrB,OAAO,CAAC,UAAU,CAAiB;IACnC,OAAO,CAAC,gBAAgB,CAA8C;IACtE,OAAO,CAAC,SAAS,CAAQ;IACzB,OAAO,CAAC,iBAAiB,CAA6B;IACtD,OAAO,CAAC,UAAU,CAAqB;gBAE3B,MAAM,EAAE,aAAa,EAAE,OAAO,GAAE,mBAAwB;IASpE;;OAEG;IACH,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED;;OAEG;IACH,IAAI,aAAa,IAAI,MAAM,CAG1B;IAED;;OAEG;IACH,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED;;OAEG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyB3D;;OAEG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/D;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAUpC;;OAEG;IACH,yBAAyB,IAAI,MAAM;IAInC;;OAEG;IACH,YAAY,IAAI,QAAQ,EAAE;IAI1B;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAI7C;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC;IASrC;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlD;;OAEG;IACH,KAAK,IAAI,IAAI;IASb;;OAEG;IACH,OAAO,CAAC,YAAY;IAwBpB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAmBxB;;OAEG;IACH,OAAO,CAAC,cAAc;IAuBtB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAyC1B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAmB5B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAgCjC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAsD7B;;OAEG;IACH,OAAO,CAAC,eAAe;CAaxB;AAsBD;;GAEG;AACH,wBAAgB,YAAY,CAC1B,MAAM,EAAE,YAAY,EAAE,EACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,aAAa,CA0Bf;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,aAAa,EACrB,OAAO,GAAE,mBAAwB,GAChC,aAAa,CAEf;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,UAAU,CAAC,CAIrB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CAI5E;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CAGf;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,UAAU,GAAG,eAAe,CAwD9D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,WAAW,EAAE,kBAAkB,GAAG,IAAI,CAE3F"}