databonk 0.0.1

package/README.md

@@ -0,0 +1,161 @@
+# Databonk.js
+
+A lightweight, fast data frame library for JavaScript and TypeScript with built-in schema validation.
+
+## Features
+
+- **Lightweight**: Minimal dependencies, tree-shakeable modules
+- **Fast**: Columnar storage using TypedArrays for performance
+- **Simple**: Clean API for common data operations
+- **Flexible**: Works with regular arrays, TypedArrays, or Apache Arrow
+- **Schema Validation**: Built-in Zod integration for data validation
+- **Type Safe**: Full TypeScript support with inferred types
+
+## Installation
+
+```bash
+npm install databonk zod
+```
+
+## Quick Start
+
+```javascript
+import { DataFrame, SchemaValidator, CommonSchemas } from 'databonk';
+
+// Create a DataFrame
+const df = DataFrame.from({
+  name: ['Alice', 'Bob', 'Charlie'],
+  age: [25, 30, 35],
+  city: ['NYC', 'LA', 'Chicago']
+});
+
+// Basic operations
+const adults = df.filter(row => row.age >= 30);
+const avgAge = df.column('age').mean();
+const grouped = df.groupBy(['city']).agg({ avgAge: 'mean' });
+
+// Schema validation
+const result = df.validate(CommonSchemas.person);
+console.log(`Valid rows: ${result.validRows}/${result.totalRows}`);
+```
+
+## Core Features
+
+### Data Operations
+- **Filtering & Selection**: Powerful row/column filtering with predicate functions
+- **Joins**: Inner, left, right, and outer joins with multiple keys
+- **Aggregations**: Sum, mean, count, min, max, std, variance with group-by support
+- **Reshaping**: Pivot, melt, transpose operations for data transformation
+- **Sorting**: Multi-column sorting with custom comparators
+
+### Schema Validation
+- **Built-in Schemas**: Common patterns for users, products, transactions, coordinates
+- **Custom Validation**: Define your own schemas with Zod
+- **Data Cleaning**: Filter valid/invalid rows, transform data types
+- **Error Reporting**: Detailed validation errors with row/column information
+
+### I/O Support
+- **CSV**: Read/write CSV files with automatic type inference
+- **Apache Arrow**: Optional integration for columnar data exchange
+- **Streaming**: Memory-efficient processing of large datasets
+
+## Examples
+
+### Schema Validation
+
+```javascript
+import { DataFrame, SchemaValidator } from 'databonk';
+import { z } from 'zod';
+
+// Define a custom schema
+const userSchema = SchemaValidator.define({
+  name: z.string().min(1),
+  age: z.number().int().min(0).max(150),
+  email: z.string().email(),
+  role: z.enum(['admin', 'user', 'guest'])
+});
+
+const userData = [
+  { name: 'Alice', age: 25, email: 'alice@example.com', role: 'admin' },
+  { name: '', age: -5, email: 'invalid', role: 'unknown' } // Invalid
+];
+
+const df = DataFrame.fromRows(userData);
+
+// Validate data
+const validation = df.validate(userSchema);
+console.log(`Errors: ${validation.errors.length}`);
+
+// Filter valid rows
+const validUsers = df.filterValid(userSchema);
+
+// Transform data with type coercion
+const cleanData = df.validateAndTransform(userSchema);
+```
+
+### Advanced Data Operations
+
+```javascript
+// Join operations
+const sales = DataFrame.fromRows([
+  { product_id: 1, quantity: 100, region: 'North' },
+  { product_id: 2, quantity: 150, region: 'South' }
+]);
+
+const products = DataFrame.fromRows([
+  { product_id: 1, name: 'Widget', price: 10.99 },
+  { product_id: 2, name: 'Gadget', price: 15.99 }
+]);
+
+const joined = sales.join(products, 'product_id', 'inner');
+
+// Group by with multiple aggregations
+const summary = joined
+  .groupBy(['region'])
+  .agg({
+    quantity: ['sum', 'mean'],
+    price: 'mean'
+  });
+
+// Add calculated columns
+const withRevenue = joined.withColumn('revenue', 
+  row => row.quantity * row.price
+);
+
+// Pivot tables
+const pivot = sales.pivot(['region'], 'product_id', 'quantity', 'sum');
+```
+
+## Docker Development
+
+```bash
+# Build and start development environment
+make docker-dev
+
+# Run tests in Docker
+make docker-test
+
+# Open shell in container
+make docker-shell
+```
+
+## Development
+
+```bash
+# Local development
+npm install
+npm run build
+npm test
+
+# With Docker
+make setup
+make dev
+```
+
+## Performance
+
+Databonk.js is designed for small to medium datasets (up to ~1M rows) with:
+- **Memory efficient**: Columnar storage with TypedArrays
+- **Fast operations**: Optimized algorithms for joins, aggregations
+- **Minimal overhead**: Zero-copy operations where possible
+- **Tree-shakeable**: Only import what you use

package/dist/core/column.d.ts

@@ -0,0 +1,25 @@
+import { DataType } from '../utils/types.js';
+export declare class Column<T = any> {
+    readonly name: string;
+    readonly dataType: DataType;
+    private data;
+    private nullBitmap;
+    readonly length: number;
+    constructor(name: string, values: T[], dataType?: DataType);
+    private createDataArray;
+    get(index: number): T | null;
+    isNull(index: number): boolean;
+    slice(start?: number, end?: number): Column<T>;
+    filter(predicate: (value: T | null, index: number) => boolean): Column<T>;
+    map<U>(fn: (value: T | null, index: number) => U, newDataType?: DataType): Column<U>;
+    sum(): number;
+    mean(): number;
+    min(): number;
+    max(): number;
+    count(): number;
+    unique(): T[];
+    values(): Iterator<T | null>;
+    toArray(): (T | null)[];
+    static from<T>(name: string, values: T[], dataType?: DataType): Column<T>;
+}
+//# sourceMappingURL=column.d.ts.map

package/dist/core/column.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"column.d.ts","sourceRoot":"","sources":["../../src/core/column.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAA6D,MAAM,mBAAmB,CAAC;AAGxG,qBAAa,MAAM,CAAC,CAAC,GAAG,GAAG;IACzB,SAAgB,IAAI,EAAE,MAAM,CAAC;IAC7B,SAAgB,QAAQ,EAAE,QAAQ,CAAC;IACnC,OAAO,CAAC,IAAI,CAA6B;IACzC,OAAO,CAAC,UAAU,CAAS;IAC3B,SAAgB,MAAM,EAAE,MAAM,CAAC;gBAEnB,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,QAAQ;IAS1D,OAAO,CAAC,eAAe;IAsBvB,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI;IAY5B,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAI9B,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;IAY9C,MAAM,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC;IAazE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,EAAE,WAAW,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC;IAWpF,GAAG,IAAI,MAAM;IAeb,IAAI,IAAI,MAAM;IASd,GAAG,IAAI,MAAM;IAqBb,GAAG,IAAI,MAAM;IAqBb,KAAK,IAAI,MAAM;IAIf,MAAM,IAAI,CAAC,EAAE;IAYZ,MAAM,IAAI,QAAQ,CAAC,CAAC,GAAG,IAAI,CAAC;IAM7B,OAAO,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE;IAQvB,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC;CAG1E"}

package/dist/core/dataframe.d.ts

@@ -0,0 +1,30 @@
+import { Column } from './column';
+export type RowObject = Record<string, any>;
+export declare class DataFrame {
+    private columns;
+    readonly length: number;
+    constructor(data: Record<string, Column> | Column[]);
+    get columnNames(): string[];
+    get columnCount(): number;
+    column(name: string): Column;
+    hasColumn(name: string): boolean;
+    addColumn(column: Column): DataFrame;
+    removeColumn(name: string): DataFrame;
+    select(columns: string[]): DataFrame;
+    filter(predicate: (row: RowObject, index: number) => boolean): DataFrame;
+    slice(start?: number, end?: number): DataFrame;
+    selectRows(indices: number[]): DataFrame;
+    getRow(index: number): RowObject;
+    rows(): IterableIterator<RowObject>;
+    head(n?: number): DataFrame;
+    tail(n?: number): DataFrame;
+    sort(columnName: string, ascending?: boolean): DataFrame;
+    drop(columns: string[]): DataFrame;
+    rename(columnMapping: Record<string, string>): DataFrame;
+    toArray(): RowObject[];
+    toColumns(): Record<string, any[]>;
+    static from(data: RowObject[] | Record<string, any[]>): DataFrame;
+    static fromRows(rows: RowObject[]): DataFrame;
+    static fromColumns(data: Record<string, any[]>): DataFrame;
+}
+//# sourceMappingURL=dataframe.d.ts.map

package/dist/core/dataframe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataframe.d.ts","sourceRoot":"","sources":["../../src/core/dataframe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAGlC,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAE5C,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAkC;IACjD,SAAgB,MAAM,EAAE,MAAM,CAAC;gBAEnB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,EAAE;IAiBnD,IAAI,WAAW,IAAI,MAAM,EAAE,CAE1B;IAED,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAQ5B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIhC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS;IAUpC,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS;IAUrC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,SAAS;IAapC,MAAM,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,GAAG,SAAS;IAaxE,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS;IAa9C,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,SAAS;IAWxC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAa/B,IAAI,IAAI,gBAAgB,CAAC,SAAS,CAAC;IAMpC,IAAI,CAAC,CAAC,GAAE,MAAU,GAAG,SAAS;IAI9B,IAAI,CAAC,CAAC,GAAE,MAAU,GAAG,SAAS;IAI9B,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,GAAE,OAAc,GAAG,SAAS;IA6B9D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,SAAS;IAYlC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS;IAWxD,OAAO,IAAI,SAAS,EAAE;IAQtB,SAAS,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC;IAQlC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,SAAS;IAQjE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,SAAS;IAgB7C,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,SAAS;CAS3D"}

package/dist/index.d.ts

@@ -0,0 +1,20 @@
+export { DataFrame } from './core/dataframe.js';
+export { Column } from './core/column.js';
+export { GroupBy } from './operations/groupby.js';
+export { Joiner, JoinType } from './operations/join.js';
+export { Reshaper } from './operations/reshape.js';
+export { ColumnDeriver, ColumnExpression } from './operations/derive.js';
+export { Aggregator, AggregateFunction, AggregateSpec } from './operations/aggregation.js';
+export { CsvReader, CsvWriter, CsvOptions } from './io/csv.js';
+export { SchemaValidator, SchemaBuilders, CommonSchemas, ValidationResult, ValidationError, ValidationOptions, DataFrameSchema, SchemaDefinition } from './validation/schema.js';
+export { DataType } from './utils/types.js';
+export { BitSet } from './utils/bitset.js';
+export { PerformanceTimer, DataGenerator, MemoryProfiler } from './utils/performance.js';
+import './operations/aggregation.js';
+import './operations/groupby.js';
+import './operations/join.js';
+import './operations/reshape.js';
+import './operations/derive.js';
+import './io/csv.js';
+import './validation/schema.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC3F,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAEzF,OAAO,6BAA6B,CAAC;AACrC,OAAO,yBAAyB,CAAC;AACjC,OAAO,sBAAsB,CAAC;AAC9B,OAAO,yBAAyB,CAAC;AACjC,OAAO,wBAAwB,CAAC;AAChC,OAAO,aAAa,CAAC;AACrB,OAAO,wBAAwB,CAAC"}