binja 0.3.4 → 0.4.1

package/README.md

@@ -1,19 +1,20 @@
 <h1 align="center">binja</h1>
 
 <p align="center">
-  <strong>High-performance Jinja2/Django template engine for Bun</strong>
+  <strong>High-performance Jinja2/Django template engine for Bun with native Zig acceleration</strong>
 </p>
 
 <p align="center">
   <a href="#installation">Installation</a> •
   <a href="#quick-start">Quick Start</a> •
   <a href="#features">Features</a> •
-  <a href="#documentation">Documentation</a> •
+  <a href="#native-acceleration">Native Acceleration</a> •
   <a href="#filters">Filters</a>
 </p>
 
 <p align="center">
   <img src="https://img.shields.io/badge/bun-%23000000.svg?style=for-the-badge&logo=bun&logoColor=white" alt="Bun" />
+  <img src="https://img.shields.io/badge/Zig-F7A41D?style=for-the-badge&logo=zig&logoColor=white" alt="Zig Native" />
   <img src="https://img.shields.io/badge/TypeScript-007ACC?style=for-the-badge&logo=typescript&logoColor=white" alt="TypeScript" />
   <img src="https://img.shields.io/badge/Django-092E20?style=for-the-badge&logo=django&logoColor=white" alt="Django Compatible" />
   <img src="https://img.shields.io/badge/license-BSD--3--Clause-blue.svg?style=for-the-badge" alt="BSD-3-Clause License" />
@@ -25,11 +26,13 @@
 
 | Feature | Binja | Other JS engines |
 |---------|-----------|------------------|
+| **Native Zig Lexer** | ✅ 7x faster | ❌ |
 | **AOT Compilation** | ✅ 160x faster | ❌ |
 | Django DTL Compatible | ✅ 100% | ❌ Partial |
 | Jinja2 Compatible | ✅ Full | ⚠️ Limited |
 | Template Inheritance | ✅ | ⚠️ |
-| 50+ Built-in Filters | ✅ | ❌ |
+| 70+ Built-in Filters | ✅ | ❌ |
+| 28 Built-in Tests | ✅ | ❌ |
 | Debug Panel | ✅ | ❌ |
 | CLI Tool | ✅ | ⚠️ |
 | Autoescape by Default | ✅ | ❌ |
@@ -68,6 +71,45 @@ bun run full-benchmark.ts
 
 ---
 
+## Native Acceleration
+
+Binja includes a **native Zig lexer** that provides **7x faster** tokenization through Bun's FFI. The native library is automatically used when available.
+
+### Supported Platforms
+
+| Platform | Architecture | Status |
+|----------|--------------|--------|
+| macOS | Apple Silicon (arm64) | ✅ |
+| macOS | Intel (x64) | ✅ |
+| Linux | x64 | ✅ |
+| Linux | arm64 | ✅ |
+
+### Check Native Status
+
+```typescript
+import { isNativeAccelerated } from 'binja/lexer'
+
+console.log('Using native Zig:', isNativeAccelerated())
+// Output: Using native Zig: true
+```
+
+### Performance Comparison
+
+| Template Size | TypeScript Lexer | Zig Native | Speedup |
+|--------------|------------------|------------|---------|
+| Small (100B) | 290K ops/s | 1.2M ops/s | **4x** |
+| Medium (1KB) | 85K ops/s | 450K ops/s | **5x** |
+| Large (10KB) | 12K ops/s | 85K ops/s | **7x** |
+
+The native lexer automatically handles:
+- ✅ All Jinja2/Django delimiters (`{{`, `{%`, `{#`)
+- ✅ Whitespace control (`{%-`, `-%}`)
+- ✅ Raw/verbatim blocks
+- ✅ UTF-8 characters (€, 日本語, emoji)
+- ✅ Error handling with line numbers
+
+---
+
 ## Installation
 
 ```bash
@@ -362,9 +404,12 @@ Tests check values using the `is` operator (Jinja2 syntax):
 ```typescript
 import { builtinTests } from 'binja'
 
-// All 30+ built-in tests
+// All 28 built-in tests
 console.log(Object.keys(builtinTests))
-// ['divisibleby', 'even', 'odd', 'number', 'integer', ...]
+// ['divisibleby', 'even', 'odd', 'number', 'integer', 'float',
+//  'defined', 'undefined', 'none', 'boolean', 'string', 'mapping',
+//  'iterable', 'sequence', 'callable', 'upper', 'lower', 'empty',
+//  'in', 'eq', 'ne', 'sameas', 'equalto', 'truthy', 'falsy', ...]
 ```
 
 ---
@@ -400,6 +445,10 @@ const env = new Environment({
   // Auto-escape HTML (default: true)
   autoescape: true,
 
+  // Cache settings
+  cache: true,          // Enable template caching (default: true)
+  cacheMaxSize: 100,    // LRU cache limit (default: 100)
+
   // Timezone for date/time operations
   // All date filters and {% now %} tag will use this timezone
   timezone: 'Europe/Rome',  // or 'UTC', 'America/New_York', etc.
@@ -426,6 +475,11 @@ const env = new Environment({
   // Static file resolver for {% static %} tag
   staticResolver: (path: string) => `/static/${path}`
 })
+
+// Cache monitoring
+env.cacheSize()    // Number of cached templates
+env.cacheStats()   // { size, maxSize, hits, misses, hitRate }
+env.clearCache()   // Clear cache and reset stats
 ```
 
 ---
@@ -568,7 +622,8 @@ await render('{{ script }}', {
 1. **Use AOT in Production** - `compile()` is 160x faster than Nunjucks
 2. **Pre-compile at Startup** - Compile templates once, use many times
 3. **Reuse Environment** - For templates with `{% extends %}`, create once
-4. **Enable caching** - Templates are cached automatically
+4. **LRU Cache** - Templates cached with LRU eviction (default: 100, prevents memory leaks)
+5. **Monitor Cache** - Use `env.cacheStats()` to optimize `cacheMaxSize`
 
 ```typescript
 import { compile } from 'binja'
@@ -650,12 +705,19 @@ Create a configured template environment.
 ```typescript
 const env = new Environment(options)
 
-// Methods
+// Rendering
 env.render(name, context)      // Render template file
 env.renderString(str, context) // Render template string
+
+// Configuration
 env.addFilter(name, fn)        // Add custom filter
 env.addGlobal(name, value)     // Add global variable
-env.loadTemplate(name)         // Pre-load template (for cache warming)
+
+// Cache Management (LRU with configurable max size)
+env.loadTemplate(name)         // Pre-load template (cache warming)
+env.cacheSize()                // Get number of cached templates
+env.cacheStats()               // Get { size, maxSize, hits, misses, hitRate }
+env.clearCache()               // Clear all cached templates and reset stats
 ```
 
 ---

package/dist/cli.js

@@ -1,5 +1,285 @@
 #!/usr/bin/env bun
 // @bun
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __moduleCache = /* @__PURE__ */ new WeakMap;
+var __toCommonJS = (from) => {
+  var entry = __moduleCache.get(from), desc;
+  if (entry)
+    return entry;
+  entry = __defProp({}, "__esModule", { value: true });
+  if (from && typeof from === "object" || typeof from === "function")
+    __getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
+      get: () => from[key],
+      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+    }));
+  __moduleCache.set(from, entry);
+  return entry;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: (newValue) => all[name] = () => newValue
+    });
+};
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+
+// src/native/index.ts
+var exports_native = {};
+__export(exports_native, {
+  tokenizeCount: () => tokenizeCount,
+  tokenize: () => tokenize,
+  nativeVersion: () => nativeVersion,
+  isNativeAvailable: () => isNativeAvailable,
+  TokenType: () => TokenType,
+  NativeLexer: () => NativeLexer
+});
+import { dlopen, FFIType, ptr, CString } from "bun:ffi";
+import { join } from "path";
+import { existsSync } from "fs";
+function getLibraryPath() {
+  const platform = process.platform;
+  const arch = process.arch;
+  const libExt = platform === "darwin" ? "dylib" : platform === "win32" ? "dll" : "so";
+  const libName = `libbinja.${libExt}`;
+  const projectRoot = join(import.meta.dir, "..", "..");
+  const searchPaths = [
+    join(projectRoot, "native", `${platform}-${arch}`, libName),
+    join(projectRoot, "native", libName),
+    join(projectRoot, "zig-native", "zig-out", "lib", libName),
+    join(projectRoot, "zig-native", libName),
+    join(import.meta.dir, libName)
+  ];
+  for (const p of searchPaths) {
+    if (existsSync(p)) {
+      return p;
+    }
+  }
+  return null;
+}
+function loadLibrary() {
+  if (_loadAttempted) {
+    return _lib;
+  }
+  _loadAttempted = true;
+  const libPath = getLibraryPath();
+  if (!libPath) {
+    console.warn("[binja] Native library not found, using pure JS fallback");
+    return null;
+  }
+  try {
+    _lib = dlopen(libPath, symbols);
+    _nativeAvailable = true;
+    return _lib;
+  } catch (e) {
+    console.warn(`[binja] Failed to load native library: ${e}`);
+    return null;
+  }
+}
+function isNativeAvailable() {
+  loadLibrary();
+  return _nativeAvailable;
+}
+function nativeVersion() {
+  const lib = loadLibrary();
+  if (!lib)
+    return null;
+  const versionPtr = lib.symbols.binja_version();
+  if (!versionPtr)
+    return null;
+  return new CString(versionPtr).toString();
+}
+function tokenizeCount(source) {
+  if (source.length === 0) {
+    return 1;
+  }
+  const lib = loadLibrary();
+  if (!lib) {
+    throw new Error("Native library not available");
+  }
+  const bytes = new TextEncoder().encode(source);
+  return Number(lib.symbols.binja_tokenize_count(ptr(bytes), bytes.length));
+}
+function tokenize(source) {
+  const lexer = new NativeLexer(source);
+  try {
+    return lexer.getAllTokens();
+  } finally {
+    lexer.free();
+  }
+}
+var TokenType, symbols, _lib = null, _loadAttempted = false, _nativeAvailable = false, NativeLexer;
+var init_native = __esm(() => {
+  TokenType = {
+    TEXT: 0,
+    VAR_START: 1,
+    VAR_END: 2,
+    BLOCK_START: 3,
+    BLOCK_END: 4,
+    COMMENT_START: 5,
+    COMMENT_END: 6,
+    IDENTIFIER: 7,
+    STRING: 8,
+    NUMBER: 9,
+    OPERATOR: 10,
+    DOT: 11,
+    COMMA: 12,
+    PIPE: 13,
+    COLON: 14,
+    LPAREN: 15,
+    RPAREN: 16,
+    LBRACKET: 17,
+    RBRACKET: 18,
+    LBRACE: 19,
+    RBRACE: 20,
+    ASSIGN: 21,
+    EOF: 22
+  };
+  symbols = {
+    binja_lexer_new: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.ptr
+    },
+    binja_lexer_free: {
+      args: [FFIType.ptr],
+      returns: FFIType.void
+    },
+    binja_lexer_token_count: {
+      args: [FFIType.ptr],
+      returns: FFIType.u64
+    },
+    binja_lexer_token_type: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.u8
+    },
+    binja_lexer_token_start: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.u32
+    },
+    binja_lexer_token_end: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.u32
+    },
+    binja_lexer_has_error: {
+      args: [FFIType.ptr],
+      returns: FFIType.bool
+    },
+    binja_lexer_error_code: {
+      args: [FFIType.ptr],
+      returns: FFIType.u8
+    },
+    binja_lexer_error_line: {
+      args: [FFIType.ptr],
+      returns: FFIType.u32
+    },
+    binja_tokenize_count: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.u64
+    },
+    binja_version: {
+      args: [],
+      returns: FFIType.ptr
+    }
+  };
+  NativeLexer = class NativeLexer {
+    lexerPtr = 0;
+    source;
+    sourceBuffer;
+    lib;
+    _tokenCount = 0;
+    _isEmpty = false;
+    constructor(source) {
+      const lib = loadLibrary();
+      if (!lib) {
+        throw new Error("Native library not available. Use isNativeAvailable() to check first.");
+      }
+      this.lib = lib;
+      this.source = source;
+      if (source.length === 0) {
+        this._isEmpty = true;
+        this._tokenCount = 1;
+        this.sourceBuffer = new Uint8Array(0);
+        return;
+      }
+      this.sourceBuffer = new TextEncoder().encode(source);
+      const result = this.lib.symbols.binja_lexer_new(ptr(this.sourceBuffer), this.sourceBuffer.length);
+      if (!result) {
+        throw new Error("Failed to create native lexer");
+      }
+      this.lexerPtr = result;
+      this._tokenCount = Number(this.lib.symbols.binja_lexer_token_count(this.lexerPtr));
+    }
+    get tokenCount() {
+      return this._tokenCount;
+    }
+    getTokenType(index) {
+      if (this._isEmpty)
+        return TokenType.EOF;
+      return Number(this.lib.symbols.binja_lexer_token_type(this.lexerPtr, index));
+    }
+    getTokenStart(index) {
+      if (this._isEmpty)
+        return 0;
+      return Number(this.lib.symbols.binja_lexer_token_start(this.lexerPtr, index));
+    }
+    getTokenEnd(index) {
+      if (this._isEmpty)
+        return 0;
+      return Number(this.lib.symbols.binja_lexer_token_end(this.lexerPtr, index));
+    }
+    hasError() {
+      if (this._isEmpty)
+        return false;
+      return Boolean(this.lib.symbols.binja_lexer_has_error(this.lexerPtr));
+    }
+    getErrorCode() {
+      if (this._isEmpty)
+        return 0;
+      return Number(this.lib.symbols.binja_lexer_error_code(this.lexerPtr));
+    }
+    getErrorLine() {
+      if (this._isEmpty)
+        return 1;
+      return Number(this.lib.symbols.binja_lexer_error_line(this.lexerPtr));
+    }
+    getTokenValue(index) {
+      if (this._isEmpty)
+        return "";
+      const start = this.getTokenStart(index);
+      const end = this.getTokenEnd(index);
+      return new TextDecoder().decode(this.sourceBuffer.slice(start, end));
+    }
+    getToken(index) {
+      return {
+        type: this.getTokenType(index),
+        start: this.getTokenStart(index),
+        end: this.getTokenEnd(index),
+        value: this.getTokenValue(index)
+      };
+    }
+    getAllTokens() {
+      const tokens = [];
+      for (let i = 0;i < this._tokenCount; i++) {
+        tokens.push(this.getToken(i));
+      }
+      return tokens;
+    }
+    free() {
+      if (this.lexerPtr) {
+        this.lib.symbols.binja_lexer_free(this.lexerPtr);
+        this.lexerPtr = null;
+      }
+    }
+    [Symbol.dispose]() {
+      this.free();
+    }
+  };
+});
 
 // src/cli.ts
 import * as fs from "fs";
@@ -20,6 +300,135 @@ var KEYWORDS = {
   in: "NAME" /* NAME */
 };
 
+// src/lexer/hybrid.ts
+var _nativeChecked = false;
+var _nativeAvailable2 = false;
+var NativeLexerClass = null;
+function checkNative() {
+  if (_nativeChecked)
+    return _nativeAvailable2;
+  _nativeChecked = true;
+  try {
+    const native = (init_native(), __toCommonJS(exports_native));
+    if (typeof native.isNativeAvailable === "function" && native.isNativeAvailable()) {
+      _nativeAvailable2 = true;
+      NativeLexerClass = native.NativeLexer;
+      return true;
+    }
+  } catch {}
+  return false;
+}
+var NATIVE_TO_TS = {
+  0: "TEXT" /* TEXT */,
+  1: "VARIABLE_START" /* VARIABLE_START */,
+  2: "VARIABLE_END" /* VARIABLE_END */,
+  3: "BLOCK_START" /* BLOCK_START */,
+  4: "BLOCK_END" /* BLOCK_END */,
+  5: "COMMENT_START" /* COMMENT_START */,
+  6: "COMMENT_END" /* COMMENT_END */,
+  7: "NAME" /* NAME */,
+  8: "STRING" /* STRING */,
+  9: "NUMBER" /* NUMBER */,
+  10: "NAME" /* NAME */,
+  11: "DOT" /* DOT */,
+  12: "COMMA" /* COMMA */,
+  13: "PIPE" /* PIPE */,
+  14: "COLON" /* COLON */,
+  15: "LPAREN" /* LPAREN */,
+  16: "RPAREN" /* RPAREN */,
+  17: "LBRACKET" /* LBRACKET */,
+  18: "RBRACKET" /* RBRACKET */,
+  19: "LBRACE" /* LBRACE */,
+  20: "RBRACE" /* RBRACE */,
+  21: "ASSIGN" /* ASSIGN */,
+  22: "EOF" /* EOF */
+};
+var OPERATOR_TO_TYPE = {
+  "==": "EQ" /* EQ */,
+  "!=": "NE" /* NE */,
+  "<": "LT" /* LT */,
+  ">": "GT" /* GT */,
+  "<=": "LE" /* LE */,
+  ">=": "GE" /* GE */,
+  "+": "ADD" /* ADD */,
+  "-": "SUB" /* SUB */,
+  "*": "MUL" /* MUL */,
+  "/": "DIV" /* DIV */,
+  "%": "MOD" /* MOD */,
+  "~": "TILDE" /* TILDE */
+};
+var KEYWORD_TO_TYPE = {
+  and: "AND" /* AND */,
+  or: "OR" /* OR */,
+  not: "NOT" /* NOT */
+};
+var ERROR_MESSAGES = {
+  1: "Unterminated string",
+  2: "Unclosed template tag",
+  3: "Invalid operator",
+  4: "Unexpected character"
+};
+function isNativeAccelerated() {
+  return checkNative();
+}
+function tokenizeNative(source) {
+  if (!checkNative() || !NativeLexerClass)
+    return null;
+  if (source.length === 0) {
+    return [{ type: "EOF" /* EOF */, value: "", line: 1, column: 1 }];
+  }
+  const lexer = new NativeLexerClass(source);
+  try {
+    if (lexer.hasError()) {
+      const errorCode = lexer.getErrorCode();
+      const errorLine = lexer.getErrorLine();
+      const message = ERROR_MESSAGES[errorCode] ?? "Unknown error";
+      throw new Error(`${message} at line ${errorLine}`);
+    }
+    const tokens = [];
+    const count = lexer.tokenCount;
+    const lineStarts = [0];
+    for (let i = 0;i < source.length; i++) {
+      if (source[i] === `
+`)
+        lineStarts.push(i + 1);
+    }
+    for (let i = 0;i < count; i++) {
+      const nativeType = lexer.getTokenType(i);
+      const value = lexer.getTokenValue(i);
+      const start = lexer.getTokenStart(i);
+      let lo = 0, hi = lineStarts.length - 1;
+      while (lo < hi) {
+        const mid = lo + hi + 1 >> 1;
+        if (lineStarts[mid] <= start)
+          lo = mid;
+        else
+          hi = mid - 1;
+      }
+      const line = lo + 1;
+      const column = start - lineStarts[lo] + 1;
+      let type = NATIVE_TO_TS[nativeType] ?? "NAME" /* NAME */;
+      let finalValue = value;
+      if (nativeType === 10 && OPERATOR_TO_TYPE[value]) {
+        type = OPERATOR_TO_TYPE[value];
+      } else if (type === "NAME" /* NAME */ && KEYWORD_TO_TYPE[value]) {
+        type = KEYWORD_TO_TYPE[value];
+      }
+      if (type === "STRING" /* STRING */ && finalValue.length >= 2) {
+        const first = finalValue[0];
+        const last = finalValue[finalValue.length - 1];
+        if (first === '"' && last === '"' || first === "'" && last === "'") {
+          finalValue = finalValue.slice(1, -1);
+        }
+      }
+      tokens.push({ type, value: finalValue, line, column });
+    }
+    return tokens;
+  } finally {
+    lexer.free();
+  }
+}
+
 // src/lexer/index.ts
 class Lexer {
   state;
@@ -29,6 +438,7 @@ class Lexer {
   blockEnd;
   commentStart;
   commentEnd;
+  useNative;
   constructor(source, options = {}) {
     this.state = {
       source,
@@ -43,8 +453,15 @@ class Lexer {
     this.blockEnd = options.blockEnd ?? "%}";
     this.commentStart = options.commentStart ?? "{#";
     this.commentEnd = options.commentEnd ?? "#}";
+    const hasCustomDelimiters = options.variableStart !== undefined || options.variableEnd !== undefined || options.blockStart !== undefined || options.blockEnd !== undefined || options.commentStart !== undefined || options.commentEnd !== undefined;
+    this.useNative = !hasCustomDelimiters && isNativeAccelerated();
   }
   tokenize() {
+    if (this.useNative) {
+      const nativeTokens = tokenizeNative(this.state.source);
+      if (nativeTokens)
+        return nativeTokens;
+    }
     while (!this.isAtEnd()) {
       this.scanToken();
     }