binja 0.3.3 → 0.4.0

package/dist/index.js

@@ -1,6 +1,286 @@
 // @bun
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __moduleCache = /* @__PURE__ */ new WeakMap;
+var __toCommonJS = (from) => {
+  var entry = __moduleCache.get(from), desc;
+  if (entry)
+    return entry;
+  entry = __defProp({}, "__esModule", { value: true });
+  if (from && typeof from === "object" || typeof from === "function")
+    __getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
+      get: () => from[key],
+      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+    }));
+  __moduleCache.set(from, entry);
+  return entry;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: (newValue) => all[name] = () => newValue
+    });
+};
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
 var __require = import.meta.require;
 
+// src/native/index.ts
+var exports_native = {};
+__export(exports_native, {
+  tokenizeCount: () => tokenizeCount,
+  tokenize: () => tokenize,
+  nativeVersion: () => nativeVersion,
+  isNativeAvailable: () => isNativeAvailable,
+  TokenType: () => TokenType2,
+  NativeLexer: () => NativeLexer
+});
+import { dlopen, FFIType, ptr, CString } from "bun:ffi";
+import { join } from "path";
+import { existsSync } from "fs";
+function getLibraryPath() {
+  const platform = process.platform;
+  const arch = process.arch;
+  const libExt = platform === "darwin" ? "dylib" : platform === "win32" ? "dll" : "so";
+  const libName = `libbinja.${libExt}`;
+  const projectRoot = join(import.meta.dir, "..", "..");
+  const searchPaths = [
+    join(projectRoot, "native", `${platform}-${arch}`, libName),
+    join(projectRoot, "native", libName),
+    join(projectRoot, "zig-native", "zig-out", "lib", libName),
+    join(projectRoot, "zig-native", libName),
+    join(import.meta.dir, libName)
+  ];
+  for (const p of searchPaths) {
+    if (existsSync(p)) {
+      return p;
+    }
+  }
+  return null;
+}
+function loadLibrary() {
+  if (_loadAttempted) {
+    return _lib;
+  }
+  _loadAttempted = true;
+  const libPath = getLibraryPath();
+  if (!libPath) {
+    console.warn("[binja] Native library not found, using pure JS fallback");
+    return null;
+  }
+  try {
+    _lib = dlopen(libPath, symbols);
+    _nativeAvailable = true;
+    return _lib;
+  } catch (e) {
+    console.warn(`[binja] Failed to load native library: ${e}`);
+    return null;
+  }
+}
+function isNativeAvailable() {
+  loadLibrary();
+  return _nativeAvailable;
+}
+function nativeVersion() {
+  const lib = loadLibrary();
+  if (!lib)
+    return null;
+  const versionPtr = lib.symbols.binja_version();
+  if (!versionPtr)
+    return null;
+  return new CString(versionPtr).toString();
+}
+function tokenizeCount(source) {
+  if (source.length === 0) {
+    return 1;
+  }
+  const lib = loadLibrary();
+  if (!lib) {
+    throw new Error("Native library not available");
+  }
+  const bytes = new TextEncoder().encode(source);
+  return Number(lib.symbols.binja_tokenize_count(ptr(bytes), bytes.length));
+}
+function tokenize(source) {
+  const lexer = new NativeLexer(source);
+  try {
+    return lexer.getAllTokens();
+  } finally {
+    lexer.free();
+  }
+}
+var TokenType2, symbols, _lib = null, _loadAttempted = false, _nativeAvailable = false, NativeLexer;
+var init_native = __esm(() => {
+  TokenType2 = {
+    TEXT: 0,
+    VAR_START: 1,
+    VAR_END: 2,
+    BLOCK_START: 3,
+    BLOCK_END: 4,
+    COMMENT_START: 5,
+    COMMENT_END: 6,
+    IDENTIFIER: 7,
+    STRING: 8,
+    NUMBER: 9,
+    OPERATOR: 10,
+    DOT: 11,
+    COMMA: 12,
+    PIPE: 13,
+    COLON: 14,
+    LPAREN: 15,
+    RPAREN: 16,
+    LBRACKET: 17,
+    RBRACKET: 18,
+    LBRACE: 19,
+    RBRACE: 20,
+    ASSIGN: 21,
+    EOF: 22
+  };
+  symbols = {
+    binja_lexer_new: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.ptr
+    },
+    binja_lexer_free: {
+      args: [FFIType.ptr],
+      returns: FFIType.void
+    },
+    binja_lexer_token_count: {
+      args: [FFIType.ptr],
+      returns: FFIType.u64
+    },
+    binja_lexer_token_type: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.u8
+    },
+    binja_lexer_token_start: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.u32
+    },
+    binja_lexer_token_end: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.u32
+    },
+    binja_lexer_has_error: {
+      args: [FFIType.ptr],
+      returns: FFIType.bool
+    },
+    binja_lexer_error_code: {
+      args: [FFIType.ptr],
+      returns: FFIType.u8
+    },
+    binja_lexer_error_line: {
+      args: [FFIType.ptr],
+      returns: FFIType.u32
+    },
+    binja_tokenize_count: {
+      args: [FFIType.ptr, FFIType.u64],
+      returns: FFIType.u64
+    },
+    binja_version: {
+      args: [],
+      returns: FFIType.ptr
+    }
+  };
+  NativeLexer = class NativeLexer {
+    lexerPtr = 0;
+    source;
+    sourceBuffer;
+    lib;
+    _tokenCount = 0;
+    _isEmpty = false;
+    constructor(source) {
+      const lib = loadLibrary();
+      if (!lib) {
+        throw new Error("Native library not available. Use isNativeAvailable() to check first.");
+      }
+      this.lib = lib;
+      this.source = source;
+      if (source.length === 0) {
+        this._isEmpty = true;
+        this._tokenCount = 1;
+        this.sourceBuffer = new Uint8Array(0);
+        return;
+      }
+      this.sourceBuffer = new TextEncoder().encode(source);
+      const result = this.lib.symbols.binja_lexer_new(ptr(this.sourceBuffer), this.sourceBuffer.length);
+      if (!result) {
+        throw new Error("Failed to create native lexer");
+      }
+      this.lexerPtr = result;
+      this._tokenCount = Number(this.lib.symbols.binja_lexer_token_count(this.lexerPtr));
+    }
+    get tokenCount() {
+      return this._tokenCount;
+    }
+    getTokenType(index) {
+      if (this._isEmpty)
+        return TokenType2.EOF;
+      return Number(this.lib.symbols.binja_lexer_token_type(this.lexerPtr, index));
+    }
+    getTokenStart(index) {
+      if (this._isEmpty)
+        return 0;
+      return Number(this.lib.symbols.binja_lexer_token_start(this.lexerPtr, index));
+    }
+    getTokenEnd(index) {
+      if (this._isEmpty)
+        return 0;
+      return Number(this.lib.symbols.binja_lexer_token_end(this.lexerPtr, index));
+    }
+    hasError() {
+      if (this._isEmpty)
+        return false;
+      return Boolean(this.lib.symbols.binja_lexer_has_error(this.lexerPtr));
+    }
+    getErrorCode() {
+      if (this._isEmpty)
+        return 0;
+      return Number(this.lib.symbols.binja_lexer_error_code(this.lexerPtr));
+    }
+    getErrorLine() {
+      if (this._isEmpty)
+        return 1;
+      return Number(this.lib.symbols.binja_lexer_error_line(this.lexerPtr));
+    }
+    getTokenValue(index) {
+      if (this._isEmpty)
+        return "";
+      const start = this.getTokenStart(index);
+      const end = this.getTokenEnd(index);
+      return new TextDecoder().decode(this.sourceBuffer.slice(start, end));
+    }
+    getToken(index) {
+      return {
+        type: this.getTokenType(index),
+        start: this.getTokenStart(index),
+        end: this.getTokenEnd(index),
+        value: this.getTokenValue(index)
+      };
+    }
+    getAllTokens() {
+      const tokens = [];
+      for (let i = 0;i < this._tokenCount; i++) {
+        tokens.push(this.getToken(i));
+      }
+      return tokens;
+    }
+    free() {
+      if (this.lexerPtr) {
+        this.lib.symbols.binja_lexer_free(this.lexerPtr);
+        this.lexerPtr = null;
+      }
+    }
+    [Symbol.dispose]() {
+      this.free();
+    }
+  };
+});
+
 // src/lexer/tokens.ts
 var TokenType;
 ((TokenType2) => {
@@ -56,6 +336,135 @@ var KEYWORDS = {
   in: "NAME" /* NAME */
 };
 
+// src/lexer/hybrid.ts
+var _nativeChecked = false;
+var _nativeAvailable2 = false;
+var NativeLexerClass = null;
+function checkNative() {
+  if (_nativeChecked)
+    return _nativeAvailable2;
+  _nativeChecked = true;
+  try {
+    const native = (init_native(), __toCommonJS(exports_native));
+    if (typeof native.isNativeAvailable === "function" && native.isNativeAvailable()) {
+      _nativeAvailable2 = true;
+      NativeLexerClass = native.NativeLexer;
+      return true;
+    }
+  } catch {}
+  return false;
+}
+var NATIVE_TO_TS = {
+  0: "TEXT" /* TEXT */,
+  1: "VARIABLE_START" /* VARIABLE_START */,
+  2: "VARIABLE_END" /* VARIABLE_END */,
+  3: "BLOCK_START" /* BLOCK_START */,
+  4: "BLOCK_END" /* BLOCK_END */,
+  5: "COMMENT_START" /* COMMENT_START */,
+  6: "COMMENT_END" /* COMMENT_END */,
+  7: "NAME" /* NAME */,
+  8: "STRING" /* STRING */,
+  9: "NUMBER" /* NUMBER */,
+  10: "NAME" /* NAME */,
+  11: "DOT" /* DOT */,
+  12: "COMMA" /* COMMA */,
+  13: "PIPE" /* PIPE */,
+  14: "COLON" /* COLON */,
+  15: "LPAREN" /* LPAREN */,
+  16: "RPAREN" /* RPAREN */,
+  17: "LBRACKET" /* LBRACKET */,
+  18: "RBRACKET" /* RBRACKET */,
+  19: "LBRACE" /* LBRACE */,
+  20: "RBRACE" /* RBRACE */,
+  21: "ASSIGN" /* ASSIGN */,
+  22: "EOF" /* EOF */
+};
+var OPERATOR_TO_TYPE = {
+  "==": "EQ" /* EQ */,
+  "!=": "NE" /* NE */,
+  "<": "LT" /* LT */,
+  ">": "GT" /* GT */,
+  "<=": "LE" /* LE */,
+  ">=": "GE" /* GE */,
+  "+": "ADD" /* ADD */,
+  "-": "SUB" /* SUB */,
+  "*": "MUL" /* MUL */,
+  "/": "DIV" /* DIV */,
+  "%": "MOD" /* MOD */,
+  "~": "TILDE" /* TILDE */
+};
+var KEYWORD_TO_TYPE = {
+  and: "AND" /* AND */,
+  or: "OR" /* OR */,
+  not: "NOT" /* NOT */
+};
+var ERROR_MESSAGES = {
+  1: "Unterminated string",
+  2: "Unclosed template tag",
+  3: "Invalid operator",
+  4: "Unexpected character"
+};
+function isNativeAccelerated() {
+  return checkNative();
+}
+function tokenizeNative(source) {
+  if (!checkNative() || !NativeLexerClass)
+    return null;
+  if (source.length === 0) {
+    return [{ type: "EOF" /* EOF */, value: "", line: 1, column: 1 }];
+  }
+  const lexer = new NativeLexerClass(source);
+  try {
+    if (lexer.hasError()) {
+      const errorCode = lexer.getErrorCode();
+      const errorLine = lexer.getErrorLine();
+      const message = ERROR_MESSAGES[errorCode] ?? "Unknown error";
+      throw new Error(`${message} at line ${errorLine}`);
+    }
+    const tokens = [];
+    const count = lexer.tokenCount;
+    const lineStarts = [0];
+    for (let i = 0;i < source.length; i++) {
+      if (source[i] === `
+`)
+        lineStarts.push(i + 1);
+    }
+    for (let i = 0;i < count; i++) {
+      const nativeType = lexer.getTokenType(i);
+      const value = lexer.getTokenValue(i);
+      const start = lexer.getTokenStart(i);
+      let lo = 0, hi = lineStarts.length - 1;
+      while (lo < hi) {
+        const mid = lo + hi + 1 >> 1;
+        if (lineStarts[mid] <= start)
+          lo = mid;
+        else
+          hi = mid - 1;
+      }
+      const line = lo + 1;
+      const column = start - lineStarts[lo] + 1;
+      let type = NATIVE_TO_TS[nativeType] ?? "NAME" /* NAME */;
+      let finalValue = value;
+      if (nativeType === 10 && OPERATOR_TO_TYPE[value]) {
+        type = OPERATOR_TO_TYPE[value];
+      } else if (type === "NAME" /* NAME */ && KEYWORD_TO_TYPE[value]) {
+        type = KEYWORD_TO_TYPE[value];
+      }
+      if (type === "STRING" /* STRING */ && finalValue.length >= 2) {
+        const first = finalValue[0];
+        const last = finalValue[finalValue.length - 1];
+        if (first === '"' && last === '"' || first === "'" && last === "'") {
+          finalValue = finalValue.slice(1, -1);
+        }
+      }
+      tokens.push({ type, value: finalValue, line, column });
+    }
+    return tokens;
+  } finally {
+    lexer.free();
+  }
+}
+
 // src/lexer/index.ts
 class Lexer {
   state;
@@ -65,6 +474,7 @@ class Lexer {
   blockEnd;
   commentStart;
   commentEnd;
+  useNative;
   constructor(source, options = {}) {
     this.state = {
       source,
@@ -79,8 +489,15 @@ class Lexer {
     this.blockEnd = options.blockEnd ?? "%}";
     this.commentStart = options.commentStart ?? "{#";
     this.commentEnd = options.commentEnd ?? "#}";
+    const hasCustomDelimiters = options.variableStart !== undefined || options.variableEnd !== undefined || options.blockStart !== undefined || options.blockEnd !== undefined || options.commentStart !== undefined || options.commentEnd !== undefined;
+    this.useNative = !hasCustomDelimiters && isNativeAccelerated();
   }
   tokenize() {
+    if (this.useNative) {
+      const nativeTokens = tokenizeNative(this.state.source);
+      if (nativeTokens)
+        return nativeTokens;
+    }
     while (!this.isAtEnd()) {
       this.scanToken();
     }
@@ -1589,7 +2006,7 @@ var last = (value) => {
     return value[value.length - 1];
   return value;
 };
-var join = (value, separator = "") => {
+var join2 = (value, separator = "") => {
   if (Array.isArray(value))
     return value.join(separator);
   return String(value);
@@ -2141,7 +2558,7 @@ var builtinFilters = {
   length_is,
   first,
   last,
-  join,
+  join: join2,
   slice,
   reverse,
   sort,
@@ -4559,6 +4976,8 @@ class Environment {
   runtime;
   templateCache = new Map;
   routes = new Map;
+  cacheHits = 0;
+  cacheMisses = 0;
   constructor(options = {}) {
     this.options = {
       templates: options.templates ?? "./templates",
@@ -4568,6 +4987,7 @@ class Environment {
       urlResolver: options.urlResolver ?? this.defaultUrlResolver.bind(this),
       staticResolver: options.staticResolver ?? this.defaultStaticResolver.bind(this),
       cache: options.cache ?? true,
+      cacheMaxSize: options.cacheMaxSize ?? 100,
       extensions: options.extensions ?? [".html", ".jinja", ".jinja2", ""],
       debug: options.debug ?? false,
       debugOptions: options.debugOptions ?? {},
@@ -4646,8 +5066,13 @@ class Environment {
   }
   async loadTemplate(templateName) {
     if (this.options.cache && this.templateCache.has(templateName)) {
-      return this.templateCache.get(templateName);
+      this.cacheHits++;
+      const ast2 = this.templateCache.get(templateName);
+      this.templateCache.delete(templateName);
+      this.templateCache.set(templateName, ast2);
+      return ast2;
     }
+    this.cacheMisses++;
     const templatePath = await this.resolveTemplatePath(templateName);
     if (!templatePath) {
       throw new Error(`Template not found: ${templateName}`);
@@ -4655,12 +5080,32 @@ class Environment {
     const source = await Bun.file(templatePath).text();
     const ast = this.compile(source);
     if (this.options.cache) {
+      while (this.templateCache.size >= this.options.cacheMaxSize) {
+        const oldestKey = this.templateCache.keys().next().value;
+        if (oldestKey)
+          this.templateCache.delete(oldestKey);
+      }
       this.templateCache.set(templateName, ast);
     }
     return ast;
   }
   clearCache() {
     this.templateCache.clear();
+    this.cacheHits = 0;
+    this.cacheMisses = 0;
+  }
+  cacheSize() {
+    return this.templateCache.size;
+  }
+  cacheStats() {
+    const total = this.cacheHits + this.cacheMisses;
+    return {
+      size: this.templateCache.size,
+      maxSize: this.options.cacheMaxSize,
+      hits: this.cacheHits,
+      misses: this.cacheMisses,
+      hitRate: total > 0 ? this.cacheHits / total * 100 : 0
+    };
   }
   addFilter(name, fn) {
     this.runtime.addFilter(name, fn);

package/dist/lexer/hybrid.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Hybrid Lexer - Uses Zig FFI when available, falls back to TypeScript
+ */
+import { Token } from './tokens';
+/**
+ * Check if native acceleration is available
+ */
+export declare function isNativeAccelerated(): boolean;
+/**
+ * Tokenize using native FFI (returns null if not available)
+ */
+export declare function tokenizeNative(source: string): Token[] | null;
+//# sourceMappingURL=hybrid.d.ts.map

package/dist/lexer/hybrid.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../src/lexer/hybrid.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,KAAK,EAAa,MAAM,UAAU,CAAA;AA0E3C;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,CAkE7D"}

package/dist/lexer/index.d.ts

@@ -1,6 +1,8 @@
 /**
  * Jinja2/DTL Lexer - Tokenizes template source into tokens
  * Compatible with both Jinja2 and Django Template Language
+ *
+ * Automatically uses Zig FFI acceleration when available and using default delimiters.
  */
 import { Token } from './tokens';
 export declare class Lexer {
@@ -11,6 +13,7 @@ export declare class Lexer {
     private blockEnd;
     private commentStart;
     private commentEnd;
+    private useNative;
     constructor(source: string, options?: {
         variableStart?: string;
         variableEnd?: string;
@@ -46,4 +49,5 @@ export declare class Lexer {
 }
 export { TokenType, KEYWORDS } from './tokens';
 export type { Token, LexerState } from './tokens';
+export { isNativeAccelerated } from './hybrid';
 //# sourceMappingURL=index.d.ts.map

package/dist/lexer/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lexer/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,KAAK,EAAmC,MAAM,UAAU,CAAA;AAEjE,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,aAAa,CAAQ;IAC7B,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,UAAU,CAAQ;gBAGxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;QACP,aAAa,CAAC,EAAE,MAAM,CAAA;QACtB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,UAAU,CAAC,EAAE,MAAM,CAAA;KACf;IAmBR,QAAQ,IAAI,KAAK,EAAE;IASnB,OAAO,CAAC,SAAS;IAsCjB,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,YAAY;IA0EpB,OAAO,CAAC,QAAQ;IAmChB,OAAO,CAAC,cAAc;IAwBtB,OAAO,CAAC,mBAAmB;IA6B3B,OAAO,CAAC,UAAU;IAyBlB,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,YAAY;IAuDpB,OAAO,CAAC,WAAW;IAgBnB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,OAAO;IAOf,OAAO,CAAC,KAAK;IAkBb,OAAO,CAAC,KAAK;IAab,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,QAAQ;CAQjB;AAED,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AAC9C,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lexer/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,KAAK,EAAmC,MAAM,UAAU,CAAA;AAGjE,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,aAAa,CAAQ;IAC7B,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,SAAS,CAAS;gBAGxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;QACP,aAAa,CAAC,EAAE,MAAM,CAAA;QACtB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,UAAU,CAAC,EAAE,MAAM,CAAA;KACf;IA8BR,QAAQ,IAAI,KAAK,EAAE;IAgBnB,OAAO,CAAC,SAAS;IAsCjB,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,YAAY;IA0EpB,OAAO,CAAC,QAAQ;IAmChB,OAAO,CAAC,cAAc;IAwBtB,OAAO,CAAC,mBAAmB;IA6B3B,OAAO,CAAC,UAAU;IAyBlB,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,YAAY;IAuDpB,OAAO,CAAC,WAAW;IAgBnB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,OAAO;IAOf,OAAO,CAAC,KAAK;IAkBb,OAAO,CAAC,KAAK;IAab,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,QAAQ;CAQjB;AAED,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AAC9C,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAA"}

package/dist/native/index.d.ts

@@ -0,0 +1,72 @@
+export interface NativeToken {
+    type: number;
+    start: number;
+    end: number;
+    value: string;
+}
+export declare const TokenType: {
+    readonly TEXT: 0;
+    readonly VAR_START: 1;
+    readonly VAR_END: 2;
+    readonly BLOCK_START: 3;
+    readonly BLOCK_END: 4;
+    readonly COMMENT_START: 5;
+    readonly COMMENT_END: 6;
+    readonly IDENTIFIER: 7;
+    readonly STRING: 8;
+    readonly NUMBER: 9;
+    readonly OPERATOR: 10;
+    readonly DOT: 11;
+    readonly COMMA: 12;
+    readonly PIPE: 13;
+    readonly COLON: 14;
+    readonly LPAREN: 15;
+    readonly RPAREN: 16;
+    readonly LBRACKET: 17;
+    readonly RBRACKET: 18;
+    readonly LBRACE: 19;
+    readonly RBRACE: 20;
+    readonly ASSIGN: 21;
+    readonly EOF: 22;
+};
+/**
+ * Check if native Zig library is available
+ */
+export declare function isNativeAvailable(): boolean;
+/**
+ * Get native library version
+ */
+export declare function nativeVersion(): string | null;
+/**
+ * Native Lexer - uses Zig FFI for maximum performance
+ */
+export declare class NativeLexer {
+    private lexerPtr;
+    private source;
+    private sourceBuffer;
+    private lib;
+    private _tokenCount;
+    private _isEmpty;
+    constructor(source: string);
+    get tokenCount(): number;
+    getTokenType(index: number): number;
+    getTokenStart(index: number): number;
+    getTokenEnd(index: number): number;
+    hasError(): boolean;
+    getErrorCode(): number;
+    getErrorLine(): number;
+    getTokenValue(index: number): string;
+    getToken(index: number): NativeToken;
+    getAllTokens(): NativeToken[];
+    free(): void;
+    [Symbol.dispose](): void;
+}
+/**
+ * Quick tokenize - returns token count only (fastest)
+ */
+export declare function tokenizeCount(source: string): number;
+/**
+ * Tokenize with native lexer, auto-cleanup
+ */
+export declare function tokenize(source: string): NativeToken[];
+//# sourceMappingURL=index.d.ts.map

package/dist/native/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/native/index.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;CACd;AAED,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;CAwBZ,CAAA;AA8HV;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAG3C;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,GAAG,IAAI,CAO7C;AAED;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,YAAY,CAAY;IAChC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAiB;gBAErB,MAAM,EAAE,MAAM;IAgC1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKnC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKpC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKlC,QAAQ,IAAI,OAAO;IAKnB,YAAY,IAAI,MAAM;IAKtB,YAAY,IAAI,MAAM;IAKtB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IASpC,YAAY,IAAI,WAAW,EAAE;IAQ7B,IAAI,IAAI,IAAI;IAOZ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAYpD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAOtD"}