binja 0.4.2 → 0.4.3

package/README.md

@@ -1,20 +1,19 @@
 <h1 align="center">binja</h1>
 
 <p align="center">
-  <strong>High-performance Jinja2/Django template engine for Bun with native Zig acceleration</strong>
+  <strong>High-performance Jinja2/Django template engine for Bun - 2-4x faster than Nunjucks</strong>
 </p>
 
 <p align="center">
   <a href="#installation">Installation</a> •
   <a href="#quick-start">Quick Start</a> •
   <a href="#features">Features</a> •
-  <a href="#native-acceleration">Native Acceleration</a> •
+  <a href="#benchmarks">Benchmarks</a> •
   <a href="#filters">Filters</a>
 </p>
 
 <p align="center">
   <img src="https://img.shields.io/badge/bun-%23000000.svg?style=for-the-badge&logo=bun&logoColor=white" alt="Bun" />
-  <img src="https://img.shields.io/badge/Zig-F7A41D?style=for-the-badge&logo=zig&logoColor=white" alt="Zig Native" />
   <img src="https://img.shields.io/badge/TypeScript-007ACC?style=for-the-badge&logo=typescript&logoColor=white" alt="TypeScript" />
   <img src="https://img.shields.io/badge/Django-092E20?style=for-the-badge&logo=django&logoColor=white" alt="Django Compatible" />
   <img src="https://img.shields.io/badge/license-BSD--3--Clause-blue.svg?style=for-the-badge" alt="BSD-3-Clause License" />
@@ -26,7 +25,7 @@
 
 | Feature | Binja | Other JS engines |
 |---------|-----------|------------------|
-| **Native Zig Lexer** | ✅ 7x faster | ❌ |
+| **Runtime Performance** | ✅ 2-4x faster | ❌ |
 | **AOT Compilation** | ✅ 160x faster | ❌ |
 | Django DTL Compatible | ✅ 100% | ❌ Partial |
 | Jinja2 Compatible | ✅ Full | ⚠️ Limited |
@@ -43,70 +42,34 @@
 
 ## Benchmarks
 
-Tested on Mac Studio M1 Max, Bun 1.3.5, 10,000 iterations.
+Tested on Mac Studio M1 Max, Bun 1.3.5.
 
 ### Two Rendering Modes
 
 | Mode | Function | Best For | vs Nunjucks |
 |------|----------|----------|-------------|
-| **Runtime** | `render()` | Development | **3.7x faster** |
+| **Runtime** | `render()` | Development | **2-4x faster** |
 | **AOT** | `compile()` | Production | **160x faster** |
 
-### Performance Comparison
+### Runtime Performance (vs Nunjucks)
 
-| Benchmark | Nunjucks | binja Runtime | binja AOT |
-|-----------|----------|---------------|-----------|
-| Simple Template | 95K ops/s | 290K ops/s | **14.3M ops/s** |
-| Complex Template | 28K ops/s | 103K ops/s | **1.07M ops/s** |
-| Nested Loops | 27K ops/s | 130K ops/s | **1.75M ops/s** |
-| HTML Escaping | 65K ops/s | 241K ops/s | **2.23M ops/s** |
-| Conditionals | 27K ops/s | 126K ops/s | **22.8M ops/s** |
-| Large Dataset (100 items) | 21K ops/s | 36K ops/s | **202K ops/s** |
+| Benchmark | binja | Nunjucks | Speedup |
+|-----------|-------|----------|---------|
+| Simple Template | 371K ops/s | 96K ops/s | **3.9x** |
+| Complex Template | 44K ops/s | 23K ops/s | **2.0x** |
+| Multiple Filters | 246K ops/s | 63K ops/s | **3.9x** |
+| Nested Loops | 76K ops/s | 26K ops/s | **3.0x** |
+| Conditionals | 84K ops/s | 25K ops/s | **3.4x** |
+| HTML Escaping | 985K ops/s | 242K ops/s | **4.1x** |
+| Large Dataset | 9.6K ops/s | 6.6K ops/s | **1.5x** |
 
-### Run Benchmarks
-
-```bash
-bun run full-benchmark.ts
-```
-
----
-
-## Native Acceleration
-
-Binja includes a **native Zig lexer** that provides **7x faster** tokenization through Bun's FFI. The native library is automatically used when available.
-
-### Supported Platforms
-
-| Platform | Architecture | Status |
-|----------|--------------|--------|
-| macOS | Apple Silicon (arm64) | ✅ |
-| macOS | Intel (x64) | ✅ |
-| Linux | x64 | ✅ |
-| Linux | arm64 | ✅ |
-
-### Check Native Status
-
-```typescript
-import { isNativeAccelerated } from 'binja/lexer'
-
-console.log('Using native Zig:', isNativeAccelerated())
-// Output: Using native Zig: true
-```
-
-### Performance Comparison
-
-| Template Size | TypeScript Lexer | Zig Native | Speedup |
-|--------------|------------------|------------|---------|
-| Small (100B) | 290K ops/s | 1.2M ops/s | **4x** |
-| Medium (1KB) | 85K ops/s | 450K ops/s | **5x** |
-| Large (10KB) | 12K ops/s | 85K ops/s | **7x** |
+### AOT Compilation (Maximum Performance)
 
-The native lexer automatically handles:
-- ✅ All Jinja2/Django delimiters (`{{`, `{%`, `{#`)
-- ✅ Whitespace control (`{%-`, `-%}`)
-- ✅ Raw/verbatim blocks
-- ✅ UTF-8 characters (€, 日本語, emoji)
-- ✅ Error handling with line numbers
+| Benchmark | binja AOT | binja Runtime | Speedup |
+|-----------|-----------|---------------|---------|
+| Simple Template | **14.3M ops/s** | 371K ops/s | 39x |
+| Complex Template | **1.07M ops/s** | 44K ops/s | 24x |
+| Nested Loops | **1.75M ops/s** | 76K ops/s | 23x |
 
 ---
 

package/dist/cli.js

@@ -1,286 +1,5 @@
 #!/usr/bin/env bun
 // @bun
-var __defProp = Object.defineProperty;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __moduleCache = /* @__PURE__ */ new WeakMap;
-var __toCommonJS = (from) => {
-  var entry = __moduleCache.get(from), desc;
-  if (entry)
-    return entry;
-  entry = __defProp({}, "__esModule", { value: true });
-  if (from && typeof from === "object" || typeof from === "function")
-    __getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
-      get: () => from[key],
-      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
-    }));
-  __moduleCache.set(from, entry);
-  return entry;
-};
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, {
-      get: all[name],
-      enumerable: true,
-      configurable: true,
-      set: (newValue) => all[name] = () => newValue
-    });
-};
-var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
-
-// src/native/index.ts
-var exports_native = {};
-__export(exports_native, {
-  tokenizeCount: () => tokenizeCount,
-  tokenize: () => tokenize,
-  nativeVersion: () => nativeVersion,
-  isNativeAvailable: () => isNativeAvailable,
-  TokenType: () => TokenType,
-  NativeLexer: () => NativeLexer
-});
-import { dlopen, FFIType, ptr, CString } from "bun:ffi";
-import { join, basename } from "path";
-import { existsSync } from "fs";
-function getLibraryPath() {
-  const platform = process.platform;
-  const arch = process.arch;
-  const libExt = platform === "darwin" ? "dylib" : platform === "win32" ? "dll" : "so";
-  const libName = `libbinja.${libExt}`;
-  const dirName = basename(import.meta.dir);
-  const projectRoot = dirName === "native" ? join(import.meta.dir, "..", "..") : join(import.meta.dir, "..");
-  const searchPaths = [
-    join(projectRoot, "native", `${platform}-${arch}`, libName),
-    join(projectRoot, "native", libName),
-    join(projectRoot, "zig-native", "zig-out", "lib", libName),
-    join(projectRoot, "zig-native", libName),
-    join(import.meta.dir, libName)
-  ];
-  for (const p of searchPaths) {
-    if (existsSync(p)) {
-      return p;
-    }
-  }
-  return null;
-}
-function loadLibrary() {
-  if (_loadAttempted) {
-    return _lib;
-  }
-  _loadAttempted = true;
-  const libPath = getLibraryPath();
-  if (!libPath) {
-    console.warn("[binja] Native library not found, using pure JS fallback");
-    return null;
-  }
-  try {
-    _lib = dlopen(libPath, symbols);
-    _nativeAvailable = true;
-    return _lib;
-  } catch (e) {
-    console.warn(`[binja] Failed to load native library: ${e}`);
-    return null;
-  }
-}
-function isNativeAvailable() {
-  loadLibrary();
-  return _nativeAvailable;
-}
-function nativeVersion() {
-  const lib = loadLibrary();
-  if (!lib)
-    return null;
-  const versionPtr = lib.symbols.binja_version();
-  if (!versionPtr)
-    return null;
-  return new CString(versionPtr).toString();
-}
-function tokenizeCount(source) {
-  if (source.length === 0) {
-    return 1;
-  }
-  const lib = loadLibrary();
-  if (!lib) {
-    throw new Error("Native library not available");
-  }
-  const bytes = new TextEncoder().encode(source);
-  return Number(lib.symbols.binja_tokenize_count(ptr(bytes), bytes.length));
-}
-function tokenize(source) {
-  const lexer = new NativeLexer(source);
-  try {
-    return lexer.getAllTokens();
-  } finally {
-    lexer.free();
-  }
-}
-var TokenType, symbols, _lib = null, _loadAttempted = false, _nativeAvailable = false, NativeLexer;
-var init_native = __esm(() => {
-  TokenType = {
-    TEXT: 0,
-    VAR_START: 1,
-    VAR_END: 2,
-    BLOCK_START: 3,
-    BLOCK_END: 4,
-    COMMENT_START: 5,
-    COMMENT_END: 6,
-    IDENTIFIER: 7,
-    STRING: 8,
-    NUMBER: 9,
-    OPERATOR: 10,
-    DOT: 11,
-    COMMA: 12,
-    PIPE: 13,
-    COLON: 14,
-    LPAREN: 15,
-    RPAREN: 16,
-    LBRACKET: 17,
-    RBRACKET: 18,
-    LBRACE: 19,
-    RBRACE: 20,
-    ASSIGN: 21,
-    EOF: 22
-  };
-  symbols = {
-    binja_lexer_new: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.ptr
-    },
-    binja_lexer_free: {
-      args: [FFIType.ptr],
-      returns: FFIType.void
-    },
-    binja_lexer_token_count: {
-      args: [FFIType.ptr],
-      returns: FFIType.u64
-    },
-    binja_lexer_token_type: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u8
-    },
-    binja_lexer_token_start: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u32
-    },
-    binja_lexer_token_end: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u32
-    },
-    binja_lexer_has_error: {
-      args: [FFIType.ptr],
-      returns: FFIType.bool
-    },
-    binja_lexer_error_code: {
-      args: [FFIType.ptr],
-      returns: FFIType.u8
-    },
-    binja_lexer_error_line: {
-      args: [FFIType.ptr],
-      returns: FFIType.u32
-    },
-    binja_tokenize_count: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u64
-    },
-    binja_version: {
-      args: [],
-      returns: FFIType.ptr
-    }
-  };
-  NativeLexer = class NativeLexer {
-    lexerPtr = 0;
-    source;
-    sourceBuffer;
-    lib;
-    _tokenCount = 0;
-    _isEmpty = false;
-    constructor(source) {
-      const lib = loadLibrary();
-      if (!lib) {
-        throw new Error("Native library not available. Use isNativeAvailable() to check first.");
-      }
-      this.lib = lib;
-      this.source = source;
-      if (source.length === 0) {
-        this._isEmpty = true;
-        this._tokenCount = 1;
-        this.sourceBuffer = new Uint8Array(0);
-        return;
-      }
-      this.sourceBuffer = new TextEncoder().encode(source);
-      const result = this.lib.symbols.binja_lexer_new(ptr(this.sourceBuffer), this.sourceBuffer.length);
-      if (!result) {
-        throw new Error("Failed to create native lexer");
-      }
-      this.lexerPtr = result;
-      this._tokenCount = Number(this.lib.symbols.binja_lexer_token_count(this.lexerPtr));
-    }
-    get tokenCount() {
-      return this._tokenCount;
-    }
-    getTokenType(index) {
-      if (this._isEmpty)
-        return TokenType.EOF;
-      return Number(this.lib.symbols.binja_lexer_token_type(this.lexerPtr, index));
-    }
-    getTokenStart(index) {
-      if (this._isEmpty)
-        return 0;
-      return Number(this.lib.symbols.binja_lexer_token_start(this.lexerPtr, index));
-    }
-    getTokenEnd(index) {
-      if (this._isEmpty)
-        return 0;
-      return Number(this.lib.symbols.binja_lexer_token_end(this.lexerPtr, index));
-    }
-    hasError() {
-      if (this._isEmpty)
-        return false;
-      return Boolean(this.lib.symbols.binja_lexer_has_error(this.lexerPtr));
-    }
-    getErrorCode() {
-      if (this._isEmpty)
-        return 0;
-      return Number(this.lib.symbols.binja_lexer_error_code(this.lexerPtr));
-    }
-    getErrorLine() {
-      if (this._isEmpty)
-        return 1;
-      return Number(this.lib.symbols.binja_lexer_error_line(this.lexerPtr));
-    }
-    getTokenValue(index) {
-      if (this._isEmpty)
-        return "";
-      const start = this.getTokenStart(index);
-      const end = this.getTokenEnd(index);
-      return new TextDecoder().decode(this.sourceBuffer.slice(start, end));
-    }
-    getToken(index) {
-      return {
-        type: this.getTokenType(index),
-        start: this.getTokenStart(index),
-        end: this.getTokenEnd(index),
-        value: this.getTokenValue(index)
-      };
-    }
-    getAllTokens() {
-      const tokens = [];
-      for (let i = 0;i < this._tokenCount; i++) {
-        tokens.push(this.getToken(i));
-      }
-      return tokens;
-    }
-    free() {
-      if (this.lexerPtr) {
-        this.lib.symbols.binja_lexer_free(this.lexerPtr);
-        this.lexerPtr = null;
-      }
-    }
-    [Symbol.dispose]() {
-      this.free();
-    }
-  };
-});
 
 // src/cli.ts
 import * as fs from "fs";
@@ -302,21 +21,8 @@ var KEYWORDS = {
 };
 
 // src/lexer/hybrid.ts
-var _nativeChecked = false;
-var _nativeAvailable2 = false;
-var NativeLexerClass = null;
+var _tokenizeBatchFn = null;
 function checkNative() {
-  if (_nativeChecked)
-    return _nativeAvailable2;
-  _nativeChecked = true;
-  try {
-    const native = (init_native(), __toCommonJS(exports_native));
-    if (typeof native.isNativeAvailable === "function" && native.isNativeAvailable()) {
-      _nativeAvailable2 = true;
-      NativeLexerClass = native.NativeLexer;
-      return true;
-    }
-  } catch {}
   return false;
 }
 var NATIVE_TO_TS = {
@@ -363,71 +69,52 @@ var KEYWORD_TO_TYPE = {
   or: "OR" /* OR */,
   not: "NOT" /* NOT */
 };
-var ERROR_MESSAGES = {
-  1: "Unterminated string",
-  2: "Unclosed template tag",
-  3: "Invalid operator",
-  4: "Unexpected character"
-};
 function isNativeAccelerated() {
   return checkNative();
 }
 function tokenizeNative(source) {
-  if (!checkNative() || !NativeLexerClass)
+  if (!checkNative() || !_tokenizeBatchFn)
     return null;
   if (source.length === 0) {
     return [{ type: "EOF" /* EOF */, value: "", line: 1, column: 1 }];
   }
-  const lexer = new NativeLexerClass(source);
-  try {
-    if (lexer.hasError()) {
-      const errorCode = lexer.getErrorCode();
-      const errorLine = lexer.getErrorLine();
-      const message = ERROR_MESSAGES[errorCode] ?? "Unknown error";
-      throw new Error(`${message} at line ${errorLine}`);
-    }
-    const tokens = [];
-    const count = lexer.tokenCount;
-    const lineStarts = [0];
-    for (let i = 0;i < source.length; i++) {
-      if (source[i] === `
+  const rawTokens = _tokenizeBatchFn(source);
+  const lineStarts = [0];
+  for (let i = 0;i < source.length; i++) {
+    if (source[i] === `
 `)
-        lineStarts.push(i + 1);
-    }
-    for (let i = 0;i < count; i++) {
-      const nativeType = lexer.getTokenType(i);
-      const value = lexer.getTokenValue(i);
-      const start = lexer.getTokenStart(i);
-      let lo = 0, hi = lineStarts.length - 1;
-      while (lo < hi) {
-        const mid = lo + hi + 1 >> 1;
-        if (lineStarts[mid] <= start)
-          lo = mid;
-        else
-          hi = mid - 1;
-      }
-      const line = lo + 1;
-      const column = start - lineStarts[lo] + 1;
-      let type = NATIVE_TO_TS[nativeType] ?? "NAME" /* NAME */;
-      let finalValue = value;
-      if (nativeType === 10 && OPERATOR_TO_TYPE[value]) {
-        type = OPERATOR_TO_TYPE[value];
-      } else if (type === "NAME" /* NAME */ && KEYWORD_TO_TYPE[value]) {
-        type = KEYWORD_TO_TYPE[value];
-      }
-      if (type === "STRING" /* STRING */ && finalValue.length >= 2) {
-        const first = finalValue[0];
-        const last = finalValue[finalValue.length - 1];
-        if (first === '"' && last === '"' || first === "'" && last === "'") {
-          finalValue = finalValue.slice(1, -1);
-        }
-      }
-      tokens.push({ type, value: finalValue, line, column });
-    }
-    return tokens;
-  } finally {
-    lexer.free();
-  }
+      lineStarts.push(i + 1);
+  }
+  const tokens = new Array(rawTokens.length);
+  for (let i = 0;i < rawTokens.length; i++) {
+    const [nativeType, start, end] = rawTokens[i];
+    let value = source.slice(start, end);
+    let lo = 0, hi = lineStarts.length - 1;
+    while (lo < hi) {
+      const mid = lo + hi + 1 >> 1;
+      if (lineStarts[mid] <= start)
+        lo = mid;
+      else
+        hi = mid - 1;
+    }
+    const line = lo + 1;
+    const column = start - lineStarts[lo] + 1;
+    let type = NATIVE_TO_TS[nativeType] ?? "NAME" /* NAME */;
+    if (nativeType === 10 && OPERATOR_TO_TYPE[value]) {
+      type = OPERATOR_TO_TYPE[value];
+    } else if (type === "NAME" /* NAME */ && KEYWORD_TO_TYPE[value]) {
+      type = KEYWORD_TO_TYPE[value];
+    }
+    if (type === "STRING" /* STRING */ && value.length >= 2) {
+      const first = value[0];
+      const last = value[value.length - 1];
+      if (first === '"' && last === '"' || first === "'" && last === "'") {
+        value = value.slice(1, -1);
+      }
+    }
+    tokens[i] = { type, value, line, column };
+  }
+  return tokens;
 }
 
 // src/lexer/index.ts

package/dist/index.js

@@ -1,287 +1,6 @@
 // @bun
-var __defProp = Object.defineProperty;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __moduleCache = /* @__PURE__ */ new WeakMap;
-var __toCommonJS = (from) => {
-  var entry = __moduleCache.get(from), desc;
-  if (entry)
-    return entry;
-  entry = __defProp({}, "__esModule", { value: true });
-  if (from && typeof from === "object" || typeof from === "function")
-    __getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
-      get: () => from[key],
-      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
-    }));
-  __moduleCache.set(from, entry);
-  return entry;
-};
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, {
-      get: all[name],
-      enumerable: true,
-      configurable: true,
-      set: (newValue) => all[name] = () => newValue
-    });
-};
-var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
 var __require = import.meta.require;
 
-// src/native/index.ts
-var exports_native = {};
-__export(exports_native, {
-  tokenizeCount: () => tokenizeCount,
-  tokenize: () => tokenize,
-  nativeVersion: () => nativeVersion,
-  isNativeAvailable: () => isNativeAvailable,
-  TokenType: () => TokenType2,
-  NativeLexer: () => NativeLexer
-});
-import { dlopen, FFIType, ptr, CString } from "bun:ffi";
-import { join, basename } from "path";
-import { existsSync } from "fs";
-function getLibraryPath() {
-  const platform = process.platform;
-  const arch = process.arch;
-  const libExt = platform === "darwin" ? "dylib" : platform === "win32" ? "dll" : "so";
-  const libName = `libbinja.${libExt}`;
-  const dirName = basename(import.meta.dir);
-  const projectRoot = dirName === "native" ? join(import.meta.dir, "..", "..") : join(import.meta.dir, "..");
-  const searchPaths = [
-    join(projectRoot, "native", `${platform}-${arch}`, libName),
-    join(projectRoot, "native", libName),
-    join(projectRoot, "zig-native", "zig-out", "lib", libName),
-    join(projectRoot, "zig-native", libName),
-    join(import.meta.dir, libName)
-  ];
-  for (const p of searchPaths) {
-    if (existsSync(p)) {
-      return p;
-    }
-  }
-  return null;
-}
-function loadLibrary() {
-  if (_loadAttempted) {
-    return _lib;
-  }
-  _loadAttempted = true;
-  const libPath = getLibraryPath();
-  if (!libPath) {
-    console.warn("[binja] Native library not found, using pure JS fallback");
-    return null;
-  }
-  try {
-    _lib = dlopen(libPath, symbols);
-    _nativeAvailable = true;
-    return _lib;
-  } catch (e) {
-    console.warn(`[binja] Failed to load native library: ${e}`);
-    return null;
-  }
-}
-function isNativeAvailable() {
-  loadLibrary();
-  return _nativeAvailable;
-}
-function nativeVersion() {
-  const lib = loadLibrary();
-  if (!lib)
-    return null;
-  const versionPtr = lib.symbols.binja_version();
-  if (!versionPtr)
-    return null;
-  return new CString(versionPtr).toString();
-}
-function tokenizeCount(source) {
-  if (source.length === 0) {
-    return 1;
-  }
-  const lib = loadLibrary();
-  if (!lib) {
-    throw new Error("Native library not available");
-  }
-  const bytes = new TextEncoder().encode(source);
-  return Number(lib.symbols.binja_tokenize_count(ptr(bytes), bytes.length));
-}
-function tokenize(source) {
-  const lexer = new NativeLexer(source);
-  try {
-    return lexer.getAllTokens();
-  } finally {
-    lexer.free();
-  }
-}
-var TokenType2, symbols, _lib = null, _loadAttempted = false, _nativeAvailable = false, NativeLexer;
-var init_native = __esm(() => {
-  TokenType2 = {
-    TEXT: 0,
-    VAR_START: 1,
-    VAR_END: 2,
-    BLOCK_START: 3,
-    BLOCK_END: 4,
-    COMMENT_START: 5,
-    COMMENT_END: 6,
-    IDENTIFIER: 7,
-    STRING: 8,
-    NUMBER: 9,
-    OPERATOR: 10,
-    DOT: 11,
-    COMMA: 12,
-    PIPE: 13,
-    COLON: 14,
-    LPAREN: 15,
-    RPAREN: 16,
-    LBRACKET: 17,
-    RBRACKET: 18,
-    LBRACE: 19,
-    RBRACE: 20,
-    ASSIGN: 21,
-    EOF: 22
-  };
-  symbols = {
-    binja_lexer_new: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.ptr
-    },
-    binja_lexer_free: {
-      args: [FFIType.ptr],
-      returns: FFIType.void
-    },
-    binja_lexer_token_count: {
-      args: [FFIType.ptr],
-      returns: FFIType.u64
-    },
-    binja_lexer_token_type: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u8
-    },
-    binja_lexer_token_start: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u32
-    },
-    binja_lexer_token_end: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u32
-    },
-    binja_lexer_has_error: {
-      args: [FFIType.ptr],
-      returns: FFIType.bool
-    },
-    binja_lexer_error_code: {
-      args: [FFIType.ptr],
-      returns: FFIType.u8
-    },
-    binja_lexer_error_line: {
-      args: [FFIType.ptr],
-      returns: FFIType.u32
-    },
-    binja_tokenize_count: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u64
-    },
-    binja_version: {
-      args: [],
-      returns: FFIType.ptr
-    }
-  };
-  NativeLexer = class NativeLexer {
-    lexerPtr = 0;
-    source;
-    sourceBuffer;
-    lib;
-    _tokenCount = 0;
-    _isEmpty = false;
-    constructor(source) {
-      const lib = loadLibrary();
-      if (!lib) {
-        throw new Error("Native library not available. Use isNativeAvailable() to check first.");
-      }
-      this.lib = lib;
-      this.source = source;
-      if (source.length === 0) {
-        this._isEmpty = true;
-        this._tokenCount = 1;
-        this.sourceBuffer = new Uint8Array(0);
-        return;
-      }
-      this.sourceBuffer = new TextEncoder().encode(source);
-      const result = this.lib.symbols.binja_lexer_new(ptr(this.sourceBuffer), this.sourceBuffer.length);
-      if (!result) {
-        throw new Error("Failed to create native lexer");
-      }
-      this.lexerPtr = result;
-      this._tokenCount = Number(this.lib.symbols.binja_lexer_token_count(this.lexerPtr));
-    }
-    get tokenCount() {
-      return this._tokenCount;
-    }
-    getTokenType(index) {
-      if (this._isEmpty)
-        return TokenType2.EOF;
-      return Number(this.lib.symbols.binja_lexer_token_type(this.lexerPtr, index));
-    }
-    getTokenStart(index) {
-      if (this._isEmpty)
-        return 0;
-      return Number(this.lib.symbols.binja_lexer_token_start(this.lexerPtr, index));
-    }
-    getTokenEnd(index) {
-      if (this._isEmpty)
-        return 0;
-      return Number(this.lib.symbols.binja_lexer_token_end(this.lexerPtr, index));
-    }
-    hasError() {
-      if (this._isEmpty)
-        return false;
-      return Boolean(this.lib.symbols.binja_lexer_has_error(this.lexerPtr));
-    }
-    getErrorCode() {
-      if (this._isEmpty)
-        return 0;
-      return Number(this.lib.symbols.binja_lexer_error_code(this.lexerPtr));
-    }
-    getErrorLine() {
-      if (this._isEmpty)
-        return 1;
-      return Number(this.lib.symbols.binja_lexer_error_line(this.lexerPtr));
-    }
-    getTokenValue(index) {
-      if (this._isEmpty)
-        return "";
-      const start = this.getTokenStart(index);
-      const end = this.getTokenEnd(index);
-      return new TextDecoder().decode(this.sourceBuffer.slice(start, end));
-    }
-    getToken(index) {
-      return {
-        type: this.getTokenType(index),
-        start: this.getTokenStart(index),
-        end: this.getTokenEnd(index),
-        value: this.getTokenValue(index)
-      };
-    }
-    getAllTokens() {
-      const tokens = [];
-      for (let i = 0;i < this._tokenCount; i++) {
-        tokens.push(this.getToken(i));
-      }
-      return tokens;
-    }
-    free() {
-      if (this.lexerPtr) {
-        this.lib.symbols.binja_lexer_free(this.lexerPtr);
-        this.lexerPtr = null;
-      }
-    }
-    [Symbol.dispose]() {
-      this.free();
-    }
-  };
-});
-
 // src/lexer/tokens.ts
 var TokenType;
 ((TokenType2) => {
@@ -338,21 +57,8 @@ var KEYWORDS = {
 };
 
 // src/lexer/hybrid.ts
-var _nativeChecked = false;
-var _nativeAvailable2 = false;
-var NativeLexerClass = null;
+var _tokenizeBatchFn = null;
 function checkNative() {
-  if (_nativeChecked)
-    return _nativeAvailable2;
-  _nativeChecked = true;
-  try {
-    const native = (init_native(), __toCommonJS(exports_native));
-    if (typeof native.isNativeAvailable === "function" && native.isNativeAvailable()) {
-      _nativeAvailable2 = true;
-      NativeLexerClass = native.NativeLexer;
-      return true;
-    }
-  } catch {}
   return false;
 }
 var NATIVE_TO_TS = {
@@ -399,71 +105,52 @@ var KEYWORD_TO_TYPE = {
   or: "OR" /* OR */,
   not: "NOT" /* NOT */
 };
-var ERROR_MESSAGES = {
-  1: "Unterminated string",
-  2: "Unclosed template tag",
-  3: "Invalid operator",
-  4: "Unexpected character"
-};
 function isNativeAccelerated() {
   return checkNative();
 }
 function tokenizeNative(source) {
-  if (!checkNative() || !NativeLexerClass)
+  if (!checkNative() || !_tokenizeBatchFn)
     return null;
   if (source.length === 0) {
     return [{ type: "EOF" /* EOF */, value: "", line: 1, column: 1 }];
   }
-  const lexer = new NativeLexerClass(source);
-  try {
-    if (lexer.hasError()) {
-      const errorCode = lexer.getErrorCode();
-      const errorLine = lexer.getErrorLine();
-      const message = ERROR_MESSAGES[errorCode] ?? "Unknown error";
-      throw new Error(`${message} at line ${errorLine}`);
-    }
-    const tokens = [];
-    const count = lexer.tokenCount;
-    const lineStarts = [0];
-    for (let i = 0;i < source.length; i++) {
-      if (source[i] === `
+  const rawTokens = _tokenizeBatchFn(source);
+  const lineStarts = [0];
+  for (let i = 0;i < source.length; i++) {
+    if (source[i] === `
 `)
-        lineStarts.push(i + 1);
-    }
-    for (let i = 0;i < count; i++) {
-      const nativeType = lexer.getTokenType(i);
-      const value = lexer.getTokenValue(i);
-      const start = lexer.getTokenStart(i);
-      let lo = 0, hi = lineStarts.length - 1;
-      while (lo < hi) {
-        const mid = lo + hi + 1 >> 1;
-        if (lineStarts[mid] <= start)
-          lo = mid;
-        else
-          hi = mid - 1;
-      }
-      const line = lo + 1;
-      const column = start - lineStarts[lo] + 1;
-      let type = NATIVE_TO_TS[nativeType] ?? "NAME" /* NAME */;
-      let finalValue = value;
-      if (nativeType === 10 && OPERATOR_TO_TYPE[value]) {
-        type = OPERATOR_TO_TYPE[value];
-      } else if (type === "NAME" /* NAME */ && KEYWORD_TO_TYPE[value]) {
-        type = KEYWORD_TO_TYPE[value];
-      }
-      if (type === "STRING" /* STRING */ && finalValue.length >= 2) {
-        const first = finalValue[0];
-        const last = finalValue[finalValue.length - 1];
-        if (first === '"' && last === '"' || first === "'" && last === "'") {
-          finalValue = finalValue.slice(1, -1);
-        }
+      lineStarts.push(i + 1);
+  }
+  const tokens = new Array(rawTokens.length);
+  for (let i = 0;i < rawTokens.length; i++) {
+    const [nativeType, start, end] = rawTokens[i];
+    let value = source.slice(start, end);
+    let lo = 0, hi = lineStarts.length - 1;
+    while (lo < hi) {
+      const mid = lo + hi + 1 >> 1;
+      if (lineStarts[mid] <= start)
+        lo = mid;
+      else
+        hi = mid - 1;
+    }
+    const line = lo + 1;
+    const column = start - lineStarts[lo] + 1;
+    let type = NATIVE_TO_TS[nativeType] ?? "NAME" /* NAME */;
+    if (nativeType === 10 && OPERATOR_TO_TYPE[value]) {
+      type = OPERATOR_TO_TYPE[value];
+    } else if (type === "NAME" /* NAME */ && KEYWORD_TO_TYPE[value]) {
+      type = KEYWORD_TO_TYPE[value];
+    }
+    if (type === "STRING" /* STRING */ && value.length >= 2) {
+      const first = value[0];
+      const last = value[value.length - 1];
+      if (first === '"' && last === '"' || first === "'" && last === "'") {
+        value = value.slice(1, -1);
       }
-      tokens.push({ type, value: finalValue, line, column });
     }
-    return tokens;
-  } finally {
-    lexer.free();
+    tokens[i] = { type, value, line, column };
   }
+  return tokens;
 }
 
 // src/lexer/index.ts
@@ -2007,7 +1694,7 @@ var last = (value) => {
     return value[value.length - 1];
   return value;
 };
-var join2 = (value, separator = "") => {
+var join = (value, separator = "") => {
   if (Array.isArray(value))
     return value.join(separator);
   return String(value);
@@ -2559,7 +2246,7 @@ var builtinFilters = {
   length_is,
   first,
   last,
-  join: join2,
+  join,
   slice,
   reverse,
   sort,

package/dist/lexer/hybrid.d.ts

@@ -7,7 +7,7 @@ import { Token } from './tokens';
  */
 export declare function isNativeAccelerated(): boolean;
 /**
- * Tokenize using native FFI (returns null if not available)
+ * Tokenize using native FFI with batch API (single FFI call)
  */
 export declare function tokenizeNative(source: string): Token[] | null;
 //# sourceMappingURL=hybrid.d.ts.map

package/dist/lexer/hybrid.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../src/lexer/hybrid.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,KAAK,EAAa,MAAM,UAAU,CAAA;AA0E3C;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,CAkE7D"}
+{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../src/lexer/hybrid.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,KAAK,EAAa,MAAM,UAAU,CAAA;AAiE3C;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,CAqD7D"}

package/dist/native/index.d.ts

@@ -66,7 +66,16 @@ export declare class NativeLexer {
  */
 export declare function tokenizeCount(source: string): number;
 /**
- * Tokenize with native lexer, auto-cleanup
+ * Tokenize with native lexer, auto-cleanup (OLD - per-token FFI calls)
  */
 export declare function tokenize(source: string): NativeToken[];
+/**
+ * Batch tokenize - single FFI call for all tokens (FAST)
+ * Returns array of [type, start, end] tuples for maximum performance
+ */
+export declare function tokenizeBatch(source: string): Array<[number, number, number]>;
+/**
+ * Batch tokenize with full token objects (includes value extraction)
+ */
+export declare function tokenizeBatchFull(source: string): NativeToken[];
 //# sourceMappingURL=index.d.ts.map

package/dist/native/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/native/index.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;CACd;AAED,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;CAwBZ,CAAA;AAkIV;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAG3C;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,GAAG,IAAI,CAO7C;AAED;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,YAAY,CAAY;IAChC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAiB;gBAErB,MAAM,EAAE,MAAM;IAgC1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKnC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKpC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKlC,QAAQ,IAAI,OAAO;IAKnB,YAAY,IAAI,MAAM;IAKtB,YAAY,IAAI,MAAM;IAKtB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IASpC,YAAY,IAAI,WAAW,EAAE;IAQ7B,IAAI,IAAI,IAAI;IAOZ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAYpD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAOtD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/native/index.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;CACd;AAED,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;CAwBZ,CAAA;AA+IV;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAG3C;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,GAAG,IAAI,CAO7C;AAED;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,YAAY,CAAY;IAChC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAiB;gBAErB,MAAM,EAAE,MAAM;IAgC1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKnC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKpC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKlC,QAAQ,IAAI,OAAO;IAKnB,YAAY,IAAI,MAAM;IAKtB,YAAY,IAAI,MAAM;IAKtB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IASpC,YAAY,IAAI,WAAW,EAAE;IAQ7B,IAAI,IAAI,IAAI;IAOZ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAYpD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAOtD;AAUD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CA2D7E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAQ/D"}

package/dist/native/index.js

@@ -1,6 +1,6 @@
 // @bun
 // src/native/index.ts
-import { dlopen, FFIType, ptr, CString } from "bun:ffi";
+import { dlopen, FFIType, ptr, CString, toArrayBuffer } from "bun:ffi";
 import { join, basename } from "path";
 import { existsSync } from "fs";
 var TokenType = {
@@ -65,6 +65,18 @@ var symbols = {
     args: [FFIType.ptr],
     returns: FFIType.u32
   },
+  binja_lexer_get_tokens_buffer: {
+    args: [FFIType.ptr],
+    returns: FFIType.ptr
+  },
+  binja_lexer_tokens_buffer_size: {
+    args: [FFIType.ptr],
+    returns: FFIType.u64
+  },
+  binja_free_tokens_buffer: {
+    args: [FFIType.ptr, FFIType.u64],
+    returns: FFIType.void
+  },
   binja_tokenize_count: {
     args: [FFIType.ptr, FFIType.u64],
     returns: FFIType.u64
@@ -243,8 +255,71 @@ function tokenize(source) {
     lexer.free();
   }
 }
+var ERROR_MESSAGES = {
+  1: "Unterminated string",
+  2: "Unclosed template tag",
+  3: "Invalid operator",
+  4: "Unexpected character"
+};
+function tokenizeBatch(source) {
+  if (source.length === 0) {
+    return [[TokenType.EOF, 0, 0]];
+  }
+  const lib = loadLibrary();
+  if (!lib) {
+    throw new Error("Native library not available");
+  }
+  const sourceBuffer = new TextEncoder().encode(source);
+  const lexerPtr = lib.symbols.binja_lexer_new(ptr(sourceBuffer), sourceBuffer.length);
+  if (!lexerPtr) {
+    throw new Error("Failed to create native lexer");
+  }
+  try {
+    if (lib.symbols.binja_lexer_has_error(lexerPtr)) {
+      const errorCode = Number(lib.symbols.binja_lexer_error_code(lexerPtr));
+      const errorLine = Number(lib.symbols.binja_lexer_error_line(lexerPtr));
+      const message = ERROR_MESSAGES[errorCode] ?? "Unknown error";
+      throw new Error(`${message} at line ${errorLine}`);
+    }
+    const bufferSize = Number(lib.symbols.binja_lexer_tokens_buffer_size(lexerPtr));
+    const bufferPtr = lib.symbols.binja_lexer_get_tokens_buffer(lexerPtr);
+    if (!bufferPtr) {
+      throw new Error("Failed to get tokens buffer");
+    }
+    try {
+      const buffer = new Uint8Array(toArrayBuffer(bufferPtr, 0, bufferSize));
+      const view = new DataView(buffer.buffer);
+      const count = view.getUint32(0, true);
+      const tokens = new Array(count);
+      let offset = 4;
+      for (let i = 0;i < count; i++) {
+        const type = buffer[offset];
+        const start = view.getUint32(offset + 1, true);
+        const end = view.getUint32(offset + 5, true);
+        tokens[i] = [type, start, end];
+        offset += 9;
+      }
+      return tokens;
+    } finally {
+      lib.symbols.binja_free_tokens_buffer(bufferPtr, bufferSize);
+    }
+  } finally {
+    lib.symbols.binja_lexer_free(lexerPtr);
+  }
+}
+function tokenizeBatchFull(source) {
+  const tuples = tokenizeBatch(source);
+  return tuples.map(([type, start, end]) => ({
+    type,
+    start,
+    end,
+    value: source.slice(start, end)
+  }));
+}
 export {
   tokenizeCount,
+  tokenizeBatchFull,
+  tokenizeBatch,
   tokenize,
   nativeVersion,
   isNativeAvailable,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "binja",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "description": "High-performance Jinja2/Django template engine for Bun",
   "main": "dist/index.js",
   "module": "dist/index.js",