@rip-lang/db 0.10.0 → 1.0.1

package/src/duckdb.zig

@@ -0,0 +1,1156 @@
+//! DuckDB Binary Protocol Serializer
+//!
+//! High-performance Zig implementation that serializes DuckDB query results
+//! directly to the binary format expected by the DuckDB UI.
+//!
+//! Key features:
+//! - Zero-copy for numeric columns (direct memory access)
+//! - Single allocation (reusable output buffer)
+//! - Safe memory management (all data copied before result freed)
+//! - Complete DuckDB UI protocol support
+
+const std = @import("std");
+const c = @cImport({
+    @cInclude("duckdb.h");
+});
+
+// ============================================================================
+// LogicalTypeId - DuckDB UI protocol type IDs
+// ============================================================================
+
+pub const LogicalTypeId = enum(u8) {
+    boolean = 10,
+    tinyint = 11,
+    smallint = 12,
+    integer = 13,
+    bigint = 14,
+    date = 15,
+    time = 16,
+    timestamp_sec = 17,
+    timestamp_ms = 18,
+    timestamp = 19,
+    timestamp_ns = 20,
+    decimal = 21,
+    float = 22,
+    double = 23,
+    char = 24,
+    varchar = 25,
+    blob = 26,
+    interval = 27,
+    utinyint = 28,
+    usmallint = 29,
+    uinteger = 30,
+    ubigint = 31,
+    timestamp_tz = 32,
+    time_tz = 34,
+    bit = 36,
+    hugeint = 50,
+    uhugeint = 49,
+    uuid = 54,
+    @"struct" = 100,
+    list = 101,
+    map = 102,
+    @"enum" = 104,
+    @"union" = 107,
+    array = 108,
+    _,
+};
+
+// ============================================================================
+// BinaryWriter - Efficient buffer writer
+// ============================================================================
+
+pub const BinaryWriter = struct {
+    buffer: []u8,
+    pos: usize,
+    overflow: bool,
+
+    inline fn varIntLen(value: u64) usize {
+        var v = value;
+        var n: usize = 1;
+        while (v > 0x7F) : (n += 1) v >>= 7;
+        return n;
+    }
+
+    pub fn init(buffer: []u8) BinaryWriter {
+        return .{ .buffer = buffer, .pos = 0, .overflow = false };
+    }
+
+    pub fn remaining(self: *const BinaryWriter) usize {
+        return if (self.overflow) 0 else self.buffer.len - self.pos;
+    }
+
+    pub fn hasOverflowed(self: *const BinaryWriter) bool {
+        return self.overflow;
+    }
+
+    fn ensureSpace(self: *BinaryWriter, bytes: usize) bool {
+        if (self.overflow) return false;
+        // Avoid `self.pos + bytes` overflow in ReleaseFast.
+        if (bytes > (self.buffer.len - self.pos)) {
+            self.overflow = true;
+            return false;
+        }
+        return true;
+    }
+
+    pub fn writeUint8(self: *BinaryWriter, value: u8) void {
+        if (!self.ensureSpace(1)) return;
+        self.buffer[self.pos] = value;
+        self.pos += 1;
+    }
+
+    pub fn writeUint16LE(self: *BinaryWriter, value: u16) void {
+        if (!self.ensureSpace(2)) return;
+        std.mem.writeInt(u16, self.buffer[self.pos..][0..2], value, .little);
+        self.pos += 2;
+    }
+
+    pub fn writeUint32LE(self: *BinaryWriter, value: u32) void {
+        if (!self.ensureSpace(4)) return;
+        std.mem.writeInt(u32, self.buffer[self.pos..][0..4], value, .little);
+        self.pos += 4;
+    }
+
+    pub fn writeInt32LE(self: *BinaryWriter, value: i32) void {
+        if (!self.ensureSpace(4)) return;
+        std.mem.writeInt(i32, self.buffer[self.pos..][0..4], value, .little);
+        self.pos += 4;
+    }
+
+    pub fn writeUint64LE(self: *BinaryWriter, value: u64) void {
+        if (!self.ensureSpace(8)) return;
+        std.mem.writeInt(u64, self.buffer[self.pos..][0..8], value, .little);
+        self.pos += 8;
+    }
+
+    pub fn writeInt64LE(self: *BinaryWriter, value: i64) void {
+        if (!self.ensureSpace(8)) return;
+        std.mem.writeInt(i64, self.buffer[self.pos..][0..8], value, .little);
+        self.pos += 8;
+    }
+
+    pub fn writeFloat32(self: *BinaryWriter, value: f32) void {
+        if (!self.ensureSpace(4)) return;
+        const bytes = std.mem.toBytes(value);
+        @memcpy(self.buffer[self.pos..][0..4], &bytes);
+        self.pos += 4;
+    }
+
+    pub fn writeFloat64(self: *BinaryWriter, value: f64) void {
+        if (!self.ensureSpace(8)) return;
+        const bytes = std.mem.toBytes(value);
+        @memcpy(self.buffer[self.pos..][0..8], &bytes);
+        self.pos += 8;
+    }
+
+    pub fn writeVarInt(self: *BinaryWriter, value: u64) void {
+        if (!self.ensureSpace(varIntLen(value))) return;
+        var v = value;
+        while (v > 0x7F) {
+            self.buffer[self.pos] = @intCast((v & 0x7F) | 0x80);
+            self.pos += 1;
+            v >>= 7;
+        }
+        self.buffer[self.pos] = @intCast(v);
+        self.pos += 1;
+    }
+
+    pub fn writeFieldId(self: *BinaryWriter, id: u16) void {
+        self.writeUint16LE(id);
+    }
+
+    pub fn writeEndMarker(self: *BinaryWriter) void {
+        self.writeFieldId(0xFFFF);
+    }
+
+    pub fn writeString(self: *BinaryWriter, str: []const u8) void {
+        if (!self.ensureSpace(varIntLen(str.len) + str.len)) return;
+        self.writeVarInt(str.len);
+        if (str.len > 0) {
+            @memcpy(self.buffer[self.pos..][0..str.len], str);
+            self.pos += str.len;
+        }
+    }
+
+    pub fn writeData(self: *BinaryWriter, data: []const u8) void {
+        if (!self.ensureSpace(varIntLen(data.len) + data.len)) return;
+        self.writeVarInt(data.len);
+        if (data.len > 0) {
+            @memcpy(self.buffer[self.pos..][0..data.len], data);
+            self.pos += data.len;
+        }
+    }
+
+    pub fn writeRawBytes(self: *BinaryWriter, data: []const u8) void {
+        if (!self.ensureSpace(data.len)) return;
+        @memcpy(self.buffer[self.pos..][0..data.len], data);
+        self.pos += data.len;
+    }
+};
+
+// ============================================================================
+// Type Mapping
+// ============================================================================
+
+fn mapToLogicalTypeId(duckdb_type: c.duckdb_type) LogicalTypeId {
+    return switch (duckdb_type) {
+        c.DUCKDB_TYPE_BOOLEAN => .boolean,
+        c.DUCKDB_TYPE_TINYINT => .tinyint,
+        c.DUCKDB_TYPE_SMALLINT => .smallint,
+        c.DUCKDB_TYPE_INTEGER => .integer,
+        c.DUCKDB_TYPE_BIGINT => .bigint,
+        c.DUCKDB_TYPE_UTINYINT => .utinyint,
+        c.DUCKDB_TYPE_USMALLINT => .usmallint,
+        c.DUCKDB_TYPE_UINTEGER => .uinteger,
+        c.DUCKDB_TYPE_UBIGINT => .ubigint,
+        c.DUCKDB_TYPE_FLOAT => .float,
+        c.DUCKDB_TYPE_DOUBLE => .double,
+        c.DUCKDB_TYPE_TIMESTAMP => .timestamp,
+        c.DUCKDB_TYPE_DATE => .date,
+        c.DUCKDB_TYPE_TIME => .time,
+        c.DUCKDB_TYPE_INTERVAL => .interval,
+        c.DUCKDB_TYPE_HUGEINT => .hugeint,
+        c.DUCKDB_TYPE_UHUGEINT => .uhugeint,
+        c.DUCKDB_TYPE_VARCHAR => .varchar,
+        c.DUCKDB_TYPE_BLOB => .blob,
+        c.DUCKDB_TYPE_DECIMAL => .decimal,
+        c.DUCKDB_TYPE_TIMESTAMP_S => .timestamp_sec,
+        c.DUCKDB_TYPE_TIMESTAMP_MS => .timestamp_ms,
+        c.DUCKDB_TYPE_TIMESTAMP_NS => .timestamp_ns,
+        c.DUCKDB_TYPE_ENUM => .@"enum",
+        c.DUCKDB_TYPE_LIST => .list,
+        c.DUCKDB_TYPE_STRUCT => .@"struct",
+        c.DUCKDB_TYPE_MAP => .map,
+        c.DUCKDB_TYPE_ARRAY => .array,
+        c.DUCKDB_TYPE_UUID => .uuid,
+        c.DUCKDB_TYPE_UNION => .@"union",
+        c.DUCKDB_TYPE_BIT => .bit,
+        c.DUCKDB_TYPE_TIME_TZ => .time_tz,
+        c.DUCKDB_TYPE_TIMESTAMP_TZ => .timestamp_tz,
+        else => .varchar,
+    };
+}
+
+fn getTypeByteSize(duckdb_type: c.duckdb_type) usize {
+    return switch (duckdb_type) {
+        c.DUCKDB_TYPE_BOOLEAN => 1,
+        c.DUCKDB_TYPE_TINYINT, c.DUCKDB_TYPE_UTINYINT => 1,
+        c.DUCKDB_TYPE_SMALLINT, c.DUCKDB_TYPE_USMALLINT => 2,
+        c.DUCKDB_TYPE_INTEGER, c.DUCKDB_TYPE_UINTEGER => 4,
+        c.DUCKDB_TYPE_BIGINT, c.DUCKDB_TYPE_UBIGINT => 8,
+        c.DUCKDB_TYPE_FLOAT => 4,
+        c.DUCKDB_TYPE_DOUBLE, c.DUCKDB_TYPE_DECIMAL => 8, // DECIMAL serialized as double
+        c.DUCKDB_TYPE_DATE => 4,
+        c.DUCKDB_TYPE_TIME, c.DUCKDB_TYPE_TIME_TZ => 8,
+        c.DUCKDB_TYPE_TIMESTAMP, c.DUCKDB_TYPE_TIMESTAMP_S, c.DUCKDB_TYPE_TIMESTAMP_MS, c.DUCKDB_TYPE_TIMESTAMP_NS, c.DUCKDB_TYPE_TIMESTAMP_TZ => 8,
+        c.DUCKDB_TYPE_HUGEINT, c.DUCKDB_TYPE_UHUGEINT => 16,
+        c.DUCKDB_TYPE_UUID => 16,
+        c.DUCKDB_TYPE_INTERVAL => 16,
+        else => 0, // Variable-length or complex types
+    };
+}
+
+// ============================================================================
+// Validity Bitmap
+// ============================================================================
+
+fn writeValidityBitmap(result: *c.duckdb_result, col: u64, row_count: u64, writer: *BinaryWriter) void {
+    const byte_count = (row_count + 7) / 8;
+    writer.writeVarInt(byte_count);
+
+    var row: u64 = 0;
+    while (row < row_count) {
+        var byte: u8 = 0;
+        var bit: u3 = 0;
+        while (bit < 8 and row < row_count) : ({
+            bit += 1;
+            row += 1;
+        }) {
+            if (!c.duckdb_value_is_null(result, col, row)) {
+                byte |= @as(u8, 1) << bit;
+            }
+        }
+        writer.writeUint8(byte);
+    }
+}
+
+fn hasNulls(result: *c.duckdb_result, col: u64, row_count: u64) bool {
+    for (0..row_count) |row| {
+        if (c.duckdb_value_is_null(result, col, row)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+// ============================================================================
+// Vector Serialization
+// ============================================================================
+
+fn serializeVector(result: *c.duckdb_result, col: u64, row_count: u64, writer: *BinaryWriter) void {
+    const col_type = c.duckdb_column_type(result, col);
+    const has_nulls = hasNulls(result, col, row_count);
+
+    // field_100: allValid flag (0 = all valid, non-zero = has bitmap)
+    writer.writeFieldId(100);
+    if (has_nulls) {
+        writer.writeUint8(1);
+        // field_101: validity bitmap
+        writer.writeFieldId(101);
+        writeValidityBitmap(result, col, row_count, writer);
+    } else {
+        writer.writeUint8(0);
+    }
+
+    // field_102: data
+    writer.writeFieldId(102);
+
+    const byte_size = getTypeByteSize(col_type);
+    if (byte_size > 0 and col_type != c.DUCKDB_TYPE_BOOLEAN) {
+        // Fixed-size numeric type - serialize all at once
+        // Note: We serialize value by value to handle potential nulls correctly
+        // Future optimization: use duckdb_column_data for zero-copy when no nulls
+        writer.writeVarInt(row_count * byte_size);
+
+        switch (col_type) {
+            c.DUCKDB_TYPE_TINYINT => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_int8(result, col, row);
+                    writer.writeUint8(@bitCast(v));
+                }
+            },
+            c.DUCKDB_TYPE_UTINYINT => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_uint8(result, col, row);
+                    writer.writeUint8(v);
+                }
+            },
+            c.DUCKDB_TYPE_SMALLINT => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_int16(result, col, row);
+                    writer.writeUint16LE(@bitCast(v));
+                }
+            },
+            c.DUCKDB_TYPE_USMALLINT => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_uint16(result, col, row);
+                    writer.writeUint16LE(v);
+                }
+            },
+            c.DUCKDB_TYPE_INTEGER => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_int32(result, col, row);
+                    writer.writeInt32LE(v);
+                }
+            },
+            c.DUCKDB_TYPE_UINTEGER => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_uint32(result, col, row);
+                    writer.writeUint32LE(v);
+                }
+            },
+            c.DUCKDB_TYPE_BIGINT => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_int64(result, col, row);
+                    writer.writeInt64LE(v);
+                }
+            },
+            c.DUCKDB_TYPE_UBIGINT => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_uint64(result, col, row);
+                    writer.writeUint64LE(v);
+                }
+            },
+            c.DUCKDB_TYPE_FLOAT => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_float(result, col, row);
+                    writer.writeFloat32(v);
+                }
+            },
+            c.DUCKDB_TYPE_DOUBLE, c.DUCKDB_TYPE_DECIMAL => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_double(result, col, row);
+                    writer.writeFloat64(v);
+                }
+            },
+            c.DUCKDB_TYPE_DATE => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_date(result, col, row);
+                    writer.writeInt32LE(v.days);
+                }
+            },
+            c.DUCKDB_TYPE_TIME, c.DUCKDB_TYPE_TIME_TZ => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_time(result, col, row);
+                    writer.writeInt64LE(v.micros);
+                }
+            },
+            c.DUCKDB_TYPE_TIMESTAMP, c.DUCKDB_TYPE_TIMESTAMP_S, c.DUCKDB_TYPE_TIMESTAMP_MS, c.DUCKDB_TYPE_TIMESTAMP_NS, c.DUCKDB_TYPE_TIMESTAMP_TZ => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_timestamp(result, col, row);
+                    writer.writeInt64LE(v.micros);
+                }
+            },
+            c.DUCKDB_TYPE_HUGEINT => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_hugeint(result, col, row);
+                    writer.writeUint64LE(v.lower);
+                    writer.writeInt64LE(v.upper);
+                }
+            },
+            c.DUCKDB_TYPE_UHUGEINT => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_uhugeint(result, col, row);
+                    writer.writeUint64LE(v.lower);
+                    writer.writeUint64LE(v.upper);
+                }
+            },
+            c.DUCKDB_TYPE_UUID => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_hugeint(result, col, row);
+                    writer.writeUint64LE(v.lower);
+                    writer.writeInt64LE(v.upper);
+                }
+            },
+            c.DUCKDB_TYPE_INTERVAL => {
+                for (0..row_count) |row| {
+                    const v = c.duckdb_value_interval(result, col, row);
+                    writer.writeInt32LE(v.months);
+                    writer.writeInt32LE(v.days);
+                    writer.writeInt64LE(v.micros);
+                }
+            },
+            else => {},
+        }
+    } else if (col_type == c.DUCKDB_TYPE_BOOLEAN) {
+        // Boolean: 1 byte per value
+        writer.writeVarInt(row_count);
+        for (0..row_count) |row| {
+            const v = c.duckdb_value_boolean(result, col, row);
+            writer.writeUint8(if (v) 1 else 0);
+        }
+    } else {
+        // Variable-length types
+        //
+        // - VARCHAR: list<string> using duckdb_value_varchar_internal (no alloc, no auto-cast)
+        // - BLOB:   list<data> using duckdb_value_blob (copies bytes; frees returned data)
+        // - Other:  list<string> using duckdb_value_varchar (auto-cast; alloc per row)
+        writer.writeVarInt(row_count);
+        for (0..row_count) |row| {
+            if (has_nulls and c.duckdb_value_is_null(result, col, row)) {
+                writer.writeString("");
+                continue;
+            }
+
+            switch (col_type) {
+                c.DUCKDB_TYPE_VARCHAR => {
+                    // Use _internal for performance - safe because we copy immediately
+                    const str = c.duckdb_value_varchar_internal(result, col, row);
+                    if (str) |s| writer.writeString(std.mem.span(s)) else writer.writeString("");
+                },
+                c.DUCKDB_TYPE_BLOB => {
+                    const blob = c.duckdb_value_blob(result, col, row);
+                    if (blob.data) |p| {
+                        const bytes: [*]const u8 = @ptrCast(p);
+                        writer.writeData(bytes[0..blob.size]);
+                        c.duckdb_free(p);
+                    } else {
+                        writer.writeData("");
+                    }
+                },
+                // Note: UUID is handled as fixed-size (16 bytes) in getTypeByteSize
+                else => {
+                    const str = c.duckdb_value_varchar(result, col, row);
+                    if (str) |s| {
+                        writer.writeString(std.mem.span(s));
+                        c.duckdb_free(s);
+                    } else {
+                        writer.writeString("");
+                    }
+                },
+            }
+        }
+    }
+
+    writer.writeEndMarker();
+}
+
+// ============================================================================
+// Column Info Serialization
+// ============================================================================
+
+fn serializeType(result: *c.duckdb_result, col: u64, writer: *BinaryWriter) void {
+    const col_type = c.duckdb_column_type(result, col);
+    const logical_type = mapToLogicalTypeId(col_type);
+
+    // field_100: type id
+    writer.writeFieldId(100);
+    writer.writeUint8(@intFromEnum(logical_type));
+
+    // field_101: extra type info (null for basic types)
+    writer.writeFieldId(101);
+    writer.writeUint8(0); // null marker
+
+    writer.writeEndMarker();
+}
+
+fn serializeColumnInfo(result: *c.duckdb_result, col_count: u64, writer: *BinaryWriter) void {
+    // field_100: list<string> (column names)
+    writer.writeFieldId(100);
+    writer.writeVarInt(col_count);
+    for (0..col_count) |col| {
+        const name = c.duckdb_column_name(result, col);
+        if (name) |n| {
+            writer.writeString(std.mem.span(n));
+        } else {
+            writer.writeString("");
+        }
+    }
+
+    // field_101: list<Type> (column types)
+    writer.writeFieldId(101);
+    writer.writeVarInt(col_count);
+    for (0..col_count) |col| {
+        serializeType(result, col, writer);
+    }
+
+    writer.writeEndMarker();
+}
+
+// ============================================================================
+// Result Serialization
+// ============================================================================
+
+fn serializeDataChunk(result: *c.duckdb_result, row_count: u64, col_count: u64, writer: *BinaryWriter) void {
+    // field_100: row count
+    writer.writeFieldId(100);
+    writer.writeVarInt(row_count);
+
+    // field_101: list<Vector>
+    writer.writeFieldId(101);
+    writer.writeVarInt(col_count);
+    for (0..col_count) |col| {
+        serializeVector(result, col, row_count, writer);
+    }
+
+    writer.writeEndMarker();
+}
+
+fn serializeErrorResult(writer: *BinaryWriter, error_msg: ?[*:0]const u8) void {
+    // field_100: success = false
+    writer.writeFieldId(100);
+    writer.writeUint8(0);
+
+    // field_101: error message
+    writer.writeFieldId(101);
+    if (error_msg) |msg| {
+        writer.writeString(std.mem.span(msg));
+    } else {
+        writer.writeString("Unknown error");
+    }
+
+    writer.writeEndMarker();
+}
+
+fn serializeEmptyResult(writer: *BinaryWriter) void {
+    writer.writeEndMarker();
+}
+
+// ============================================================================
+// Public API
+// ============================================================================
+
+/// Open a DuckDB database. Pass null for in-memory database.
+export fn duck_open(path: ?[*:0]const u8) usize {
+    var db: c.duckdb_database = undefined;
+    if (c.duckdb_open(path, &db) == c.DuckDBError) {
+        return 0;
+    }
+    return @intFromPtr(db);
+}
+
+/// Close a database
+export fn duck_close(db: usize) void {
+    if (db == 0) return;
+    var d: c.duckdb_database = @ptrFromInt(db);
+    c.duckdb_close(&d);
+}
+
+/// Connect to a database
+export fn duck_connect(db: usize) usize {
+    if (db == 0) return 0;
+    var conn: c.duckdb_connection = undefined;
+    if (c.duckdb_connect(@ptrFromInt(db), &conn) == c.DuckDBError) {
+        return 0;
+    }
+    return @intFromPtr(conn);
+}
+
+/// Disconnect from database
+export fn duck_disconnect(conn: usize) void {
+    if (conn == 0) return;
+    var co: c.duckdb_connection = @ptrFromInt(conn);
+    c.duckdb_disconnect(&co);
+}
+
+/// Execute SQL and serialize result directly to binary format.
+/// Returns bytes written to buffer, or 0 on connection error.
+/// If buffer overflows, returns an error result instead.
+export fn duck_query_binary(
+    conn: usize,
+    sql_ptr: [*:0]const u8,
+    buffer_ptr: [*]u8,
+    buffer_size: usize,
+    row_limit: u64,
+) usize {
+    if (conn == 0) return 0;
+
+    const buffer = buffer_ptr[0..buffer_size];
+    var writer = BinaryWriter.init(buffer);
+
+    // Execute query
+    var result: c.duckdb_result = undefined;
+    const status = c.duckdb_query(@ptrFromInt(conn), sql_ptr, &result);
+    defer c.duckdb_destroy_result(&result);
+
+    // Check for error
+    if (status == c.DuckDBError) {
+        const error_msg = c.duckdb_result_error(&result);
+        serializeErrorResult(&writer, error_msg);
+        return writer.pos;
+    }
+
+    // Get result dimensions
+    const total_rows = c.duckdb_row_count(&result);
+    const row_count = @min(total_rows, row_limit);
+    const col_count = c.duckdb_column_count(&result);
+
+    if (col_count == 0) {
+        // No columns - return empty result
+        serializeEmptyResult(&writer);
+        return writer.pos;
+    }
+
+    // field_100: success = true
+    writer.writeFieldId(100);
+    writer.writeUint8(1);
+
+    // field_101: ColumnNamesAndTypes
+    writer.writeFieldId(101);
+    serializeColumnInfo(&result, col_count, &writer);
+
+    // field_102: list<DataChunk> (single chunk for now)
+    writer.writeFieldId(102);
+    writer.writeVarInt(1); // 1 chunk
+    serializeDataChunk(&result, row_count, col_count, &writer);
+
+    writer.writeEndMarker();
+
+    // Check if we overflowed the buffer
+    if (writer.hasOverflowed()) {
+        // Reset and return an error result
+        var error_writer = BinaryWriter.init(buffer);
+        serializeErrorResult(&error_writer, "Result too large for buffer. Use LIMIT or COPY TO for large results.");
+        return error_writer.pos;
+    }
+
+    return writer.pos;
+}
+
+/// Serialize an empty result (for /ddb/interrupt)
+export fn duck_empty_result(buffer_ptr: [*]u8, buffer_size: usize) usize {
+    const buffer = buffer_ptr[0..buffer_size];
+    var writer = BinaryWriter.init(buffer);
+    serializeEmptyResult(&writer);
+    return writer.pos;
+}
+
+// ============================================================================
+// Basic Query API (for JSON endpoints)
+// ============================================================================
+
+/// Execute a query and return result handle (0 on allocation failure)
+/// Note: Always check duck_result_error() - a non-zero handle may still contain an error
+export fn duck_query(conn: usize, sql: [*:0]const u8) usize {
+    if (conn == 0) return 0;
+    const result = std.heap.c_allocator.create(c.duckdb_result) catch return 0;
+    // Note: DuckDB stores error in result struct, so we always return the handle
+    // Caller should check duck_result_error() to see if query failed
+    _ = c.duckdb_query(@ptrFromInt(conn), sql, result);
+    return @intFromPtr(result);
+}
+
+/// Free a result
+export fn duck_free_result(result: usize) void {
+    if (result == 0) return;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    c.duckdb_destroy_result(res);
+    std.heap.c_allocator.destroy(res);
+}
+
+/// Get error message from result
+export fn duck_result_error(result: usize) ?[*:0]const u8 {
+    if (result == 0) return null;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    return c.duckdb_result_error(res);
+}
+
+/// Get row count
+export fn duck_row_count(result: usize) u64 {
+    if (result == 0) return 0;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    return c.duckdb_row_count(res);
+}
+
+/// Get column count
+export fn duck_column_count(result: usize) u64 {
+    if (result == 0) return 0;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    return c.duckdb_column_count(res);
+}
+
+/// Get column name
+export fn duck_column_name(result: usize, col: u64) ?[*:0]const u8 {
+    if (result == 0) return null;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    return c.duckdb_column_name(res, col);
+}
+
+/// Get column type
+export fn duck_column_type(result: usize, col: u64) u32 {
+    if (result == 0) return 0;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    return c.duckdb_column_type(res, col);
+}
+
+/// Check if value is null
+export fn duck_value_is_null(result: usize, col: u64, row: u64) bool {
+    if (result == 0) return true;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    return c.duckdb_value_is_null(res, col, row);
+}
+
+/// Get string value (returns pointer that must NOT be freed - copy immediately!)
+export fn duck_value_varchar(result: usize, col: u64, row: u64) ?[*:0]u8 {
+    if (result == 0) return null;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    // Use the allocating version for safety - caller must free with duck_free
+    return c.duckdb_value_varchar(res, col, row);
+}
+
+/// Get int64 value (works for most integer types)
+export fn duck_value_int64(result: usize, col: u64, row: u64) i64 {
+    if (result == 0) return 0;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    return c.duckdb_value_int64(res, col, row);
+}
+
+/// Get double value (works for float/double)
+export fn duck_value_double(result: usize, col: u64, row: u64) f64 {
+    if (result == 0) return 0;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    return c.duckdb_value_double(res, col, row);
+}
+
+/// Get boolean value
+export fn duck_value_boolean(result: usize, col: u64, row: u64) bool {
+    if (result == 0) return false;
+    const res: *c.duckdb_result = @ptrFromInt(result);
+    return c.duckdb_value_boolean(res, col, row);
+}
+
+/// Free a string returned by duck_value_varchar
+export fn duck_free(pointer: usize) void {
+    if (pointer == 0) return;
+    c.duckdb_free(@ptrFromInt(pointer));
+}
+
+/// Serialize an error result
+export fn duck_error_result(
+    error_ptr: [*:0]const u8,
+    buffer_ptr: [*]u8,
+    buffer_size: usize,
+) usize {
+    const buffer = buffer_ptr[0..buffer_size];
+    var writer = BinaryWriter.init(buffer);
+    serializeErrorResult(&writer, error_ptr);
+    return writer.pos;
+}
+
+// ============================================================================
+// SQL Tokenizer (with comptime lookup tables for performance)
+// ============================================================================
+
+const TokenType = enum(u8) {
+    identifier = 0,
+    numeric_constant = 1,
+    string_constant = 2,
+    operator = 3,
+    keyword = 4,
+    comment = 5,
+};
+
+// Character classes for fast lookup
+const CharClass = enum(u8) {
+    other = 0,
+    whitespace = 1,
+    digit = 2,
+    alpha = 3,
+    operator = 4,
+    single_quote = 5,
+    double_quote = 6,
+    minus = 7, // Special: could be operator or comment start
+    slash = 8, // Special: could be operator or comment start
+    dot = 9, // Special: could be operator or number start
+};
+
+// Comptime lookup table - built at compile time, zero runtime cost
+const char_class: [256]CharClass = blk: {
+    var table: [256]CharClass = [_]CharClass{.other} ** 256;
+
+    // Whitespace
+    table[' '] = .whitespace;
+    table['\t'] = .whitespace;
+    table['\n'] = .whitespace;
+    table['\r'] = .whitespace;
+
+    // Digits
+    for ('0'..('9' + 1)) |ch| table[ch] = .digit;
+
+    // Alpha (including underscore)
+    for ('a'..('z' + 1)) |ch| table[ch] = .alpha;
+    for ('A'..('Z' + 1)) |ch| table[ch] = .alpha;
+    table['_'] = .alpha;
+
+    // Operators (excluding special cases)
+    table['+'] = .operator;
+    table['*'] = .operator;
+    table['%'] = .operator;
+    table['='] = .operator;
+    table['<'] = .operator;
+    table['>'] = .operator;
+    table['!'] = .operator;
+    table['&'] = .operator;
+    table['|'] = .operator;
+    table['^'] = .operator;
+    table['~'] = .operator;
+    table['('] = .operator;
+    table[')'] = .operator;
+    table[','] = .operator;
+    table[';'] = .operator;
+    table['['] = .operator;
+    table[']'] = .operator;
+    table['{'] = .operator;
+    table['}'] = .operator;
+    table[':'] = .operator;
+
+    // Special characters (need context to determine meaning)
+    table['\''] = .single_quote;
+    table['"'] = .double_quote;
+    table['-'] = .minus;
+    table['/'] = .slash;
+    table['.'] = .dot;
+
+    break :blk table;
+};
+
+// Comptime uppercase table
+const to_upper: [256]u8 = blk: {
+    var table: [256]u8 = undefined;
+    for (0..256) |i| {
+        table[i] = if (i >= 'a' and i <= 'z') @intCast(i - 32) else @intCast(i);
+    }
+    break :blk table;
+};
+
+// SQL keywords - using StaticStringMap for efficient O(1) average lookup
+const keyword_map = std.StaticStringMap(void).initComptime(.{
+    .{ "ADD", {} },       .{ "ALL", {} },        .{ "ALTER", {} },      .{ "AND", {} },
+    .{ "ANY", {} },       .{ "AS", {} },         .{ "ASC", {} },        .{ "BETWEEN", {} },
+    .{ "BY", {} },        .{ "CASE", {} },       .{ "CAST", {} },       .{ "CONSTRAINT", {} },
+    .{ "CREATE", {} },    .{ "CROSS", {} },      .{ "CURRENT", {} },    .{ "DEFAULT", {} },
+    .{ "DELETE", {} },    .{ "DESC", {} },       .{ "DISTINCT", {} },   .{ "DROP", {} },
+    .{ "ELSE", {} },      .{ "END", {} },        .{ "ESCAPE", {} },     .{ "EXISTS", {} },
+    .{ "FALSE", {} },     .{ "FOLLOWING", {} },  .{ "FOR", {} },        .{ "FOREIGN", {} },
+    .{ "FROM", {} },      .{ "FULL", {} },       .{ "GROUP", {} },      .{ "HAVING", {} },
+    .{ "IF", {} },        .{ "ILIKE", {} },      .{ "IN", {} },         .{ "INDEX", {} },
+    .{ "INNER", {} },     .{ "INSERT", {} },     .{ "INTO", {} },       .{ "IS", {} },
+    .{ "JOIN", {} },      .{ "KEY", {} },        .{ "LEFT", {} },       .{ "LIKE", {} },
+    .{ "LIMIT", {} },     .{ "NOT", {} },        .{ "NULL", {} },       .{ "OFFSET", {} },
+    .{ "ON", {} },        .{ "OR", {} },         .{ "ORDER", {} },      .{ "OUTER", {} },
+    .{ "OVER", {} },      .{ "PARTITION", {} },  .{ "PRECEDING", {} },  .{ "PRIMARY", {} },
+    .{ "RANGE", {} },     .{ "RECURSIVE", {} },  .{ "REFERENCES", {} }, .{ "RIGHT", {} },
+    .{ "ROW", {} },       .{ "ROWS", {} },       .{ "SELECT", {} },     .{ "SET", {} },
+    .{ "SIMILAR", {} },   .{ "SOME", {} },       .{ "TABLE", {} },      .{ "THEN", {} },
+    .{ "TOP", {} },       .{ "TRUE", {} },       .{ "UNBOUNDED", {} },  .{ "UNION", {} },
+    .{ "UPDATE", {} },    .{ "VALUES", {} },     .{ "VIEW", {} },       .{ "WHEN", {} },
+    .{ "WHERE", {} },     .{ "WINDOW", {} },     .{ "WITH", {} },
+});
+
+fn isKeyword(word: []const u8) bool {
+    if (word.len < 2 or word.len > 11) return false;
+
+    // Convert to uppercase using comptime table
+    var upper_buf: [16]u8 = undefined;
+    for (word, 0..) |ch, idx| {
+        upper_buf[idx] = to_upper[ch];
+    }
+    const upper = upper_buf[0..word.len];
+
+    return keyword_map.has(upper);
+}
+
+// Check if character continues an identifier
+inline fn isIdentChar(ch: u8) bool {
+    const class = char_class[ch];
+    return class == .alpha or class == .digit;
+}
+
+// Check for two-character operator
+inline fn isTwoCharOp(ch: u8, next: u8) bool {
+    return (ch == '<' and (next == '=' or next == '>')) or
+        (ch == '>' and next == '=') or
+        (ch == '!' and next == '=') or
+        (ch == '|' and next == '|') or
+        (ch == '&' and next == '&') or
+        (ch == ':' and next == ':') or
+        (ch == '-' and next == '>');
+}
+
+/// Tokenize SQL and serialize to binary format.
+export fn duck_tokenize(
+    sql_ptr: [*]const u8,
+    sql_len: usize,
+    buffer_ptr: [*]u8,
+    buffer_size: usize,
+) usize {
+    const sql = sql_ptr[0..sql_len];
+    const buffer = buffer_ptr[0..buffer_size];
+    var writer = BinaryWriter.init(buffer);
+
+    var offsets: [4096]u64 = undefined;
+    var types: [4096]u64 = undefined;
+    var token_count: usize = 0;
+
+    var i: usize = 0;
+    while (i < sql.len and token_count < 4096) {
+        const start = i;
+        const ch = sql[i];
+        const class = char_class[ch];
+
+        switch (class) {
+            .whitespace => {
+                i += 1;
+                continue;
+            },
+
+            .alpha => {
+                // Identifier or keyword
+                while (i < sql.len and isIdentChar(sql[i])) : (i += 1) {}
+                offsets[token_count] = start;
+                types[token_count] = if (isKeyword(sql[start..i]))
+                    @intFromEnum(TokenType.keyword)
+                else
+                    @intFromEnum(TokenType.identifier);
+                token_count += 1;
+            },
+
+            .digit => {
+                // Numeric literal
+                while (i < sql.len) {
+                    const byte = sql[i];
+                    if (char_class[byte] == .digit or byte == '.' or byte == 'e' or byte == 'E') {
+                        i += 1;
+                    } else if ((byte == '+' or byte == '-') and i > start and (sql[i - 1] == 'e' or sql[i - 1] == 'E')) {
+                        i += 1; // Exponent sign
+                    } else break;
+                }
+                offsets[token_count] = start;
+                types[token_count] = @intFromEnum(TokenType.numeric_constant);
+                token_count += 1;
+            },
+
+            .single_quote => {
+                // String literal
+                i += 1;
+                while (i < sql.len) {
+                    if (sql[i] == '\'') {
+                        if (i + 1 < sql.len and sql[i + 1] == '\'') {
+                            i += 2; // Escaped quote
+                        } else {
+                            i += 1;
+                            break;
+                        }
+                    } else {
+                        i += 1;
+                    }
+                }
+                offsets[token_count] = start;
+                types[token_count] = @intFromEnum(TokenType.string_constant);
+                token_count += 1;
+            },
+
+            .double_quote => {
+                // Quoted identifier
+                i += 1;
+                while (i < sql.len and sql[i] != '"') : (i += 1) {}
+                if (i < sql.len) i += 1;
+                offsets[token_count] = start;
+                types[token_count] = @intFromEnum(TokenType.identifier);
+                token_count += 1;
+            },
+
+            .minus => {
+                // Could be comment or operator
+                if (i + 1 < sql.len and sql[i + 1] == '-') {
+                    // Line comment
+                    while (i < sql.len and sql[i] != '\n') : (i += 1) {}
+                    offsets[token_count] = start;
+                    types[token_count] = @intFromEnum(TokenType.comment);
+                    token_count += 1;
+                } else {
+                    // Operator
+                    i += if (i + 1 < sql.len and sql[i + 1] == '>') @as(usize, 2) else 1;
+                    offsets[token_count] = start;
+                    types[token_count] = @intFromEnum(TokenType.operator);
+                    token_count += 1;
+                }
+            },
+
+            .slash => {
+                // Could be block comment or operator
+                if (i + 1 < sql.len and sql[i + 1] == '*') {
+                    // Block comment
+                    i += 2;
+                    while (i + 1 < sql.len and !(sql[i] == '*' and sql[i + 1] == '/')) : (i += 1) {}
+                    if (i + 1 < sql.len) i += 2;
+                    offsets[token_count] = start;
+                    types[token_count] = @intFromEnum(TokenType.comment);
+                    token_count += 1;
+                } else {
+                    // Division operator
+                    i += 1;
+                    offsets[token_count] = start;
+                    types[token_count] = @intFromEnum(TokenType.operator);
+                    token_count += 1;
+                }
+            },
+
+            .dot => {
+                // Could be number or operator
+                if (i + 1 < sql.len and char_class[sql[i + 1]] == .digit) {
+                    // Fractional number
+                    i += 1;
+                    while (i < sql.len and char_class[sql[i]] == .digit) : (i += 1) {}
+                    offsets[token_count] = start;
+                    types[token_count] = @intFromEnum(TokenType.numeric_constant);
+                    token_count += 1;
+                } else {
+                    // Dot operator
+                    i += 1;
+                    offsets[token_count] = start;
+                    types[token_count] = @intFromEnum(TokenType.operator);
+                    token_count += 1;
+                }
+            },
+
+            .operator => {
+                // Check for two-char operators
+                if (i + 1 < sql.len and isTwoCharOp(ch, sql[i + 1])) {
+                    i += 2;
+                } else {
+                    i += 1;
+                }
+                offsets[token_count] = start;
+                types[token_count] = @intFromEnum(TokenType.operator);
+                token_count += 1;
+            },
+
+            .other => {
+                i += 1; // Skip unknown
+            },
+        }
+    }
+
+    // Serialize result
+    writer.writeFieldId(100);
+    writer.writeVarInt(token_count);
+    for (offsets[0..token_count]) |offset| {
+        writer.writeVarInt(offset);
+    }
+
+    writer.writeFieldId(101);
+    writer.writeVarInt(token_count);
+    for (types[0..token_count]) |t| {
+        writer.writeVarInt(t);
+    }
+
+    writer.writeEndMarker();
+
+    // Return 0 on overflow (tokenize errors are non-fatal)
+    return if (writer.hasOverflowed()) 0 else writer.pos;
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+test "BinaryWriter basic operations" {
+    var buffer: [256]u8 = undefined;
+    var writer = BinaryWriter.init(&buffer);
+
+    writer.writeUint8(42);
+    try std.testing.expectEqual(@as(u8, 42), buffer[0]);
+
+    writer.writeUint16LE(0x1234);
+    try std.testing.expectEqual(@as(u8, 0x34), buffer[1]);
+    try std.testing.expectEqual(@as(u8, 0x12), buffer[2]);
+
+    writer.writeVarInt(127);
+    try std.testing.expectEqual(@as(u8, 127), buffer[3]);
+
+    writer.writeVarInt(128);
+    try std.testing.expectEqual(@as(u8, 0x80), buffer[4]);
+    try std.testing.expectEqual(@as(u8, 0x01), buffer[5]);
+}
+
+test "BinaryWriter string" {
+    var buffer: [256]u8 = undefined;
+    var writer = BinaryWriter.init(&buffer);
+
+    writer.writeString("hello");
+    try std.testing.expectEqual(@as(u8, 5), buffer[0]); // length
+    try std.testing.expectEqualSlices(u8, "hello", buffer[1..6]);
+}
+
+test "open and close database" {
+    const db = duck_open(null);
+    try std.testing.expect(db != 0);
+    duck_close(db);
+}
+
+test "query binary serialization" {
+    const db = duck_open(null);
+    defer duck_close(db);
+
+    const conn = duck_connect(db);
+    defer duck_disconnect(conn);
+
+    var buffer: [4096]u8 = undefined;
+    const bytes = duck_query_binary(conn, "SELECT 42 as num", &buffer, buffer.len, 1000);
+
+    try std.testing.expect(bytes > 0);
+    // First field should be 100 (success flag)
+    try std.testing.expectEqual(@as(u16, 100), std.mem.readInt(u16, buffer[0..2], .little));
+    // Success flag should be 1
+    try std.testing.expectEqual(@as(u8, 1), buffer[2]);
+}
+
+test "tokenize basic SQL" {
+    var buffer: [1024]u8 = undefined;
+    const sql = "SELECT * FROM users WHERE id = 1";
+    const bytes = duck_tokenize(sql.ptr, sql.len, &buffer, buffer.len);
+
+    try std.testing.expect(bytes > 0);
+}