binja 0.4.2 → 0.4.4

package/dist/index.js

@@ -1,287 +1,6 @@
 // @bun
-var __defProp = Object.defineProperty;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __moduleCache = /* @__PURE__ */ new WeakMap;
-var __toCommonJS = (from) => {
-  var entry = __moduleCache.get(from), desc;
-  if (entry)
-    return entry;
-  entry = __defProp({}, "__esModule", { value: true });
-  if (from && typeof from === "object" || typeof from === "function")
-    __getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
-      get: () => from[key],
-      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
-    }));
-  __moduleCache.set(from, entry);
-  return entry;
-};
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, {
-      get: all[name],
-      enumerable: true,
-      configurable: true,
-      set: (newValue) => all[name] = () => newValue
-    });
-};
-var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
 var __require = import.meta.require;
 
-// src/native/index.ts
-var exports_native = {};
-__export(exports_native, {
-  tokenizeCount: () => tokenizeCount,
-  tokenize: () => tokenize,
-  nativeVersion: () => nativeVersion,
-  isNativeAvailable: () => isNativeAvailable,
-  TokenType: () => TokenType2,
-  NativeLexer: () => NativeLexer
-});
-import { dlopen, FFIType, ptr, CString } from "bun:ffi";
-import { join, basename } from "path";
-import { existsSync } from "fs";
-function getLibraryPath() {
-  const platform = process.platform;
-  const arch = process.arch;
-  const libExt = platform === "darwin" ? "dylib" : platform === "win32" ? "dll" : "so";
-  const libName = `libbinja.${libExt}`;
-  const dirName = basename(import.meta.dir);
-  const projectRoot = dirName === "native" ? join(import.meta.dir, "..", "..") : join(import.meta.dir, "..");
-  const searchPaths = [
-    join(projectRoot, "native", `${platform}-${arch}`, libName),
-    join(projectRoot, "native", libName),
-    join(projectRoot, "zig-native", "zig-out", "lib", libName),
-    join(projectRoot, "zig-native", libName),
-    join(import.meta.dir, libName)
-  ];
-  for (const p of searchPaths) {
-    if (existsSync(p)) {
-      return p;
-    }
-  }
-  return null;
-}
-function loadLibrary() {
-  if (_loadAttempted) {
-    return _lib;
-  }
-  _loadAttempted = true;
-  const libPath = getLibraryPath();
-  if (!libPath) {
-    console.warn("[binja] Native library not found, using pure JS fallback");
-    return null;
-  }
-  try {
-    _lib = dlopen(libPath, symbols);
-    _nativeAvailable = true;
-    return _lib;
-  } catch (e) {
-    console.warn(`[binja] Failed to load native library: ${e}`);
-    return null;
-  }
-}
-function isNativeAvailable() {
-  loadLibrary();
-  return _nativeAvailable;
-}
-function nativeVersion() {
-  const lib = loadLibrary();
-  if (!lib)
-    return null;
-  const versionPtr = lib.symbols.binja_version();
-  if (!versionPtr)
-    return null;
-  return new CString(versionPtr).toString();
-}
-function tokenizeCount(source) {
-  if (source.length === 0) {
-    return 1;
-  }
-  const lib = loadLibrary();
-  if (!lib) {
-    throw new Error("Native library not available");
-  }
-  const bytes = new TextEncoder().encode(source);
-  return Number(lib.symbols.binja_tokenize_count(ptr(bytes), bytes.length));
-}
-function tokenize(source) {
-  const lexer = new NativeLexer(source);
-  try {
-    return lexer.getAllTokens();
-  } finally {
-    lexer.free();
-  }
-}
-var TokenType2, symbols, _lib = null, _loadAttempted = false, _nativeAvailable = false, NativeLexer;
-var init_native = __esm(() => {
-  TokenType2 = {
-    TEXT: 0,
-    VAR_START: 1,
-    VAR_END: 2,
-    BLOCK_START: 3,
-    BLOCK_END: 4,
-    COMMENT_START: 5,
-    COMMENT_END: 6,
-    IDENTIFIER: 7,
-    STRING: 8,
-    NUMBER: 9,
-    OPERATOR: 10,
-    DOT: 11,
-    COMMA: 12,
-    PIPE: 13,
-    COLON: 14,
-    LPAREN: 15,
-    RPAREN: 16,
-    LBRACKET: 17,
-    RBRACKET: 18,
-    LBRACE: 19,
-    RBRACE: 20,
-    ASSIGN: 21,
-    EOF: 22
-  };
-  symbols = {
-    binja_lexer_new: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.ptr
-    },
-    binja_lexer_free: {
-      args: [FFIType.ptr],
-      returns: FFIType.void
-    },
-    binja_lexer_token_count: {
-      args: [FFIType.ptr],
-      returns: FFIType.u64
-    },
-    binja_lexer_token_type: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u8
-    },
-    binja_lexer_token_start: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u32
-    },
-    binja_lexer_token_end: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u32
-    },
-    binja_lexer_has_error: {
-      args: [FFIType.ptr],
-      returns: FFIType.bool
-    },
-    binja_lexer_error_code: {
-      args: [FFIType.ptr],
-      returns: FFIType.u8
-    },
-    binja_lexer_error_line: {
-      args: [FFIType.ptr],
-      returns: FFIType.u32
-    },
-    binja_tokenize_count: {
-      args: [FFIType.ptr, FFIType.u64],
-      returns: FFIType.u64
-    },
-    binja_version: {
-      args: [],
-      returns: FFIType.ptr
-    }
-  };
-  NativeLexer = class NativeLexer {
-    lexerPtr = 0;
-    source;
-    sourceBuffer;
-    lib;
-    _tokenCount = 0;
-    _isEmpty = false;
-    constructor(source) {
-      const lib = loadLibrary();
-      if (!lib) {
-        throw new Error("Native library not available. Use isNativeAvailable() to check first.");
-      }
-      this.lib = lib;
-      this.source = source;
-      if (source.length === 0) {
-        this._isEmpty = true;
-        this._tokenCount = 1;
-        this.sourceBuffer = new Uint8Array(0);
-        return;
-      }
-      this.sourceBuffer = new TextEncoder().encode(source);
-      const result = this.lib.symbols.binja_lexer_new(ptr(this.sourceBuffer), this.sourceBuffer.length);
-      if (!result) {
-        throw new Error("Failed to create native lexer");
-      }
-      this.lexerPtr = result;
-      this._tokenCount = Number(this.lib.symbols.binja_lexer_token_count(this.lexerPtr));
-    }
-    get tokenCount() {
-      return this._tokenCount;
-    }
-    getTokenType(index) {
-      if (this._isEmpty)
-        return TokenType2.EOF;
-      return Number(this.lib.symbols.binja_lexer_token_type(this.lexerPtr, index));
-    }
-    getTokenStart(index) {
-      if (this._isEmpty)
-        return 0;
-      return Number(this.lib.symbols.binja_lexer_token_start(this.lexerPtr, index));
-    }
-    getTokenEnd(index) {
-      if (this._isEmpty)
-        return 0;
-      return Number(this.lib.symbols.binja_lexer_token_end(this.lexerPtr, index));
-    }
-    hasError() {
-      if (this._isEmpty)
-        return false;
-      return Boolean(this.lib.symbols.binja_lexer_has_error(this.lexerPtr));
-    }
-    getErrorCode() {
-      if (this._isEmpty)
-        return 0;
-      return Number(this.lib.symbols.binja_lexer_error_code(this.lexerPtr));
-    }
-    getErrorLine() {
-      if (this._isEmpty)
-        return 1;
-      return Number(this.lib.symbols.binja_lexer_error_line(this.lexerPtr));
-    }
-    getTokenValue(index) {
-      if (this._isEmpty)
-        return "";
-      const start = this.getTokenStart(index);
-      const end = this.getTokenEnd(index);
-      return new TextDecoder().decode(this.sourceBuffer.slice(start, end));
-    }
-    getToken(index) {
-      return {
-        type: this.getTokenType(index),
-        start: this.getTokenStart(index),
-        end: this.getTokenEnd(index),
-        value: this.getTokenValue(index)
-      };
-    }
-    getAllTokens() {
-      const tokens = [];
-      for (let i = 0;i < this._tokenCount; i++) {
-        tokens.push(this.getToken(i));
-      }
-      return tokens;
-    }
-    free() {
-      if (this.lexerPtr) {
-        this.lib.symbols.binja_lexer_free(this.lexerPtr);
-        this.lexerPtr = null;
-      }
-    }
-    [Symbol.dispose]() {
-      this.free();
-    }
-  };
-});
-
 // src/lexer/tokens.ts
 var TokenType;
 ((TokenType2) => {
@@ -338,21 +57,8 @@ var KEYWORDS = {
 };
 
 // src/lexer/hybrid.ts
-var _nativeChecked = false;
-var _nativeAvailable2 = false;
-var NativeLexerClass = null;
+var _tokenizeBatchFn = null;
 function checkNative() {
-  if (_nativeChecked)
-    return _nativeAvailable2;
-  _nativeChecked = true;
-  try {
-    const native = (init_native(), __toCommonJS(exports_native));
-    if (typeof native.isNativeAvailable === "function" && native.isNativeAvailable()) {
-      _nativeAvailable2 = true;
-      NativeLexerClass = native.NativeLexer;
-      return true;
-    }
-  } catch {}
   return false;
 }
 var NATIVE_TO_TS = {
@@ -399,71 +105,52 @@ var KEYWORD_TO_TYPE = {
   or: "OR" /* OR */,
   not: "NOT" /* NOT */
 };
-var ERROR_MESSAGES = {
-  1: "Unterminated string",
-  2: "Unclosed template tag",
-  3: "Invalid operator",
-  4: "Unexpected character"
-};
 function isNativeAccelerated() {
   return checkNative();
 }
 function tokenizeNative(source) {
-  if (!checkNative() || !NativeLexerClass)
+  if (!checkNative() || !_tokenizeBatchFn)
     return null;
   if (source.length === 0) {
     return [{ type: "EOF" /* EOF */, value: "", line: 1, column: 1 }];
   }
-  const lexer = new NativeLexerClass(source);
-  try {
-    if (lexer.hasError()) {
-      const errorCode = lexer.getErrorCode();
-      const errorLine = lexer.getErrorLine();
-      const message = ERROR_MESSAGES[errorCode] ?? "Unknown error";
-      throw new Error(`${message} at line ${errorLine}`);
-    }
-    const tokens = [];
-    const count = lexer.tokenCount;
-    const lineStarts = [0];
-    for (let i = 0;i < source.length; i++) {
-      if (source[i] === `
+  const rawTokens = _tokenizeBatchFn(source);
+  const lineStarts = [0];
+  for (let i = 0;i < source.length; i++) {
+    if (source[i] === `
 `)
-        lineStarts.push(i + 1);
-    }
-    for (let i = 0;i < count; i++) {
-      const nativeType = lexer.getTokenType(i);
-      const value = lexer.getTokenValue(i);
-      const start = lexer.getTokenStart(i);
-      let lo = 0, hi = lineStarts.length - 1;
-      while (lo < hi) {
-        const mid = lo + hi + 1 >> 1;
-        if (lineStarts[mid] <= start)
-          lo = mid;
-        else
-          hi = mid - 1;
-      }
-      const line = lo + 1;
-      const column = start - lineStarts[lo] + 1;
-      let type = NATIVE_TO_TS[nativeType] ?? "NAME" /* NAME */;
-      let finalValue = value;
-      if (nativeType === 10 && OPERATOR_TO_TYPE[value]) {
-        type = OPERATOR_TO_TYPE[value];
-      } else if (type === "NAME" /* NAME */ && KEYWORD_TO_TYPE[value]) {
-        type = KEYWORD_TO_TYPE[value];
-      }
-      if (type === "STRING" /* STRING */ && finalValue.length >= 2) {
-        const first = finalValue[0];
-        const last = finalValue[finalValue.length - 1];
-        if (first === '"' && last === '"' || first === "'" && last === "'") {
-          finalValue = finalValue.slice(1, -1);
-        }
+      lineStarts.push(i + 1);
+  }
+  const tokens = new Array(rawTokens.length);
+  for (let i = 0;i < rawTokens.length; i++) {
+    const [nativeType, start, end] = rawTokens[i];
+    let value = source.slice(start, end);
+    let lo = 0, hi = lineStarts.length - 1;
+    while (lo < hi) {
+      const mid = lo + hi + 1 >> 1;
+      if (lineStarts[mid] <= start)
+        lo = mid;
+      else
+        hi = mid - 1;
+    }
+    const line = lo + 1;
+    const column = start - lineStarts[lo] + 1;
+    let type = NATIVE_TO_TS[nativeType] ?? "NAME" /* NAME */;
+    if (nativeType === 10 && OPERATOR_TO_TYPE[value]) {
+      type = OPERATOR_TO_TYPE[value];
+    } else if (type === "NAME" /* NAME */ && KEYWORD_TO_TYPE[value]) {
+      type = KEYWORD_TO_TYPE[value];
+    }
+    if (type === "STRING" /* STRING */ && value.length >= 2) {
+      const first = value[0];
+      const last = value[value.length - 1];
+      if (first === '"' && last === '"' || first === "'" && last === "'") {
+        value = value.slice(1, -1);
       }
-      tokens.push({ type, value: finalValue, line, column });
     }
-    return tokens;
-  } finally {
-    lexer.free();
+    tokens[i] = { type, value, line, column };
   }
+  return tokens;
 }
 
 // src/lexer/index.ts
@@ -863,13 +550,16 @@ class Lexer {
 ` || c === "\r";
   }
   isDigit(c) {
-    return c >= "0" && c <= "9";
+    const code = c.charCodeAt(0);
+    return code >= 48 && code <= 57;
   }
   isAlpha(c) {
-    return c >= "a" && c <= "z" || c >= "A" && c <= "Z";
+    const code = c.charCodeAt(0);
+    return code >= 97 && code <= 122 || code >= 65 && code <= 90;
   }
   isAlphaNumeric(c) {
-    return this.isAlpha(c) || this.isDigit(c);
+    const code = c.charCodeAt(0);
+    return code >= 48 && code <= 57 || code >= 97 && code <= 122 || code >= 65 && code <= 90;
   }
   addToken(type, value) {
     this.state.tokens.push({
@@ -1777,7 +1467,7 @@ class Context {
   _currentScope;
   constructor(data = {}, parent = null) {
     this.parent = parent;
-    this._currentScope = Object.assign(Object.create(null), data);
+    this._currentScope = { ...data };
     this.scopes.push(this._currentScope);
   }
   get(name) {
@@ -1803,7 +1493,7 @@ class Context {
     return this.parent ? this.parent.has(name) : false;
   }
   push(data = {}) {
-    this._currentScope = Object.assign(Object.create(null), data);
+    this._currentScope = { ...data };
     this.scopes.push(this._currentScope);
   }
   pop() {
@@ -2007,7 +1697,7 @@ var last = (value) => {
     return value[value.length - 1];
   return value;
 };
-var join2 = (value, separator = "") => {
+var join = (value, separator = "") => {
   if (Array.isArray(value))
     return value.join(separator);
   return String(value);
@@ -2285,15 +1975,26 @@ var indent = (value, width = 4, first2 = false, blank = false) => {
 };
 var replace = (value, old, newStr, count) => {
   const str = String(value);
-  if (count === undefined) {
-    return str.replaceAll(String(old), String(newStr));
-  }
-  let result = str;
-  let remaining = Number(count);
-  while (remaining > 0 && result.includes(String(old))) {
-    result = result.replace(String(old), String(newStr));
-    remaining--;
+  const oldStr = String(old);
+  const newString = String(newStr);
+  if (count === undefined || count < 0) {
+    return str.replaceAll(oldStr, newString);
   }
+  const maxCount = Number(count);
+  if (maxCount === 0)
+    return str;
+  let result = "";
+  let pos = 0;
+  let replaced = 0;
+  while (pos < str.length && replaced < maxCount) {
+    const idx = str.indexOf(oldStr, pos);
+    if (idx === -1)
+      break;
+    result += str.slice(pos, idx) + newString;
+    pos = idx + oldStr.length;
+    replaced++;
+  }
+  result += str.slice(pos);
   return result;
 };
 var format = (value, ...args) => {
@@ -2473,11 +2174,12 @@ var PHONE_MAP = {
 };
 var phone2numeric = (value) => {
   const str = String(value).toLowerCase();
-  let result = "";
-  for (let i = 0;i < str.length; i++) {
-    result += PHONE_MAP[str[i]] ?? str[i];
+  const len = str.length;
+  const chars = new Array(len);
+  for (let i = 0;i < len; i++) {
+    chars[i] = PHONE_MAP[str[i]] ?? str[i];
   }
-  return result;
+  return chars.join("");
 };
 var linenumbers = (value) => {
   const lines = String(value).split(`
@@ -2559,7 +2261,7 @@ var builtinFilters = {
   length_is,
   first,
   last,
-  join: join2,
+  join,
   slice,
   reverse,
   sort,
@@ -2940,14 +2642,15 @@ class Runtime {
       return this.renderNodesSync(node.else_, ctx);
     }
     const parts = new Array(len);
-    const isUnpacking = Array.isArray(node.target);
     ctx.push();
     ctx.pushForLoop(items, 0);
-    for (let i = 0;i < len; i++) {
-      const item = items[i];
-      if (i > 0)
-        ctx.updateForLoop(i, items);
-      if (isUnpacking) {
+    if (Array.isArray(node.target)) {
+      const targets = node.target;
+      const targetsLen = targets.length;
+      for (let i = 0;i < len; i++) {
+        const item = items[i];
+        if (i > 0)
+          ctx.updateForLoop(i, items);
         let values;
         if (Array.isArray(item)) {
           values = item;
@@ -2956,14 +2659,19 @@ class Runtime {
         } else {
           values = [item, item];
         }
-        const targets = node.target;
-        for (let j = 0;j < targets.length; j++) {
+        for (let j = 0;j < targetsLen; j++) {
           ctx.set(targets[j], values[j]);
         }
-      } else {
-        ctx.set(node.target, item);
+        parts[i] = this.renderNodesSync(node.body, ctx);
+      }
+    } else {
+      const target = node.target;
+      for (let i = 0;i < len; i++) {
+        if (i > 0)
+          ctx.updateForLoop(i, items);
+        ctx.set(target, items[i]);
+        parts[i] = this.renderNodesSync(node.body, ctx);
       }
-      parts[i] = this.renderNodesSync(node.body, ctx);
     }
     ctx.popForLoop();
     ctx.pop();
@@ -3179,17 +2887,20 @@ class Runtime {
     const obj = this.eval(node.object, ctx);
     if (obj == null)
       return;
-    const numIndex = parseInt(node.attribute, 10);
-    if (!isNaN(numIndex) && Array.isArray(obj))
-      return obj[numIndex];
-    if (typeof obj === "object" && node.attribute in obj) {
-      const value = obj[node.attribute];
-      return typeof value === "function" ? value.call(obj) : value;
+    const attr2 = node.attribute;
+    if (Array.isArray(obj)) {
+      const numIndex = parseInt(attr2, 10);
+      if (!isNaN(numIndex))
+        return obj[numIndex];
+    }
+    const value = obj[attr2];
+    if (value === undefined && !(attr2 in Object(obj))) {
+      return;
     }
-    if (typeof obj[node.attribute] === "function") {
-      return obj[node.attribute].bind(obj);
+    if (typeof value === "function") {
+      return value.bind(obj);
     }
-    return;
+    return value;
   }
   evalGetItem(node, ctx) {
     const obj = this.eval(node.object, ctx);
@@ -3542,9 +3253,13 @@ class Runtime {
       if (typeof value[Symbol.iterator] === "function") {
         return Array.from(value);
       }
-      const result = [];
-      for (const key in value) {
-        result.push({ key, value: value[key], 0: key, 1: value[key] });
+      const keys = Object.keys(value);
+      const len = keys.length;
+      const result = new Array(len);
+      for (let i = 0;i < len; i++) {
+        const key = keys[i];
+        const val = value[key];
+        result[i] = { key, value: val, 0: key, 1: val };
       }
       return result;
     }

package/dist/lexer/hybrid.d.ts

@@ -7,7 +7,7 @@ import { Token } from './tokens';
  */
 export declare function isNativeAccelerated(): boolean;
 /**
- * Tokenize using native FFI (returns null if not available)
+ * Tokenize using native FFI with batch API (single FFI call)
  */
 export declare function tokenizeNative(source: string): Token[] | null;
 //# sourceMappingURL=hybrid.d.ts.map

package/dist/lexer/hybrid.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../src/lexer/hybrid.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,KAAK,EAAa,MAAM,UAAU,CAAA;AA0E3C;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,CAkE7D"}
+{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../src/lexer/hybrid.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,KAAK,EAAa,MAAM,UAAU,CAAA;AAiE3C;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,CAqD7D"}

package/dist/lexer/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lexer/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,KAAK,EAAmC,MAAM,UAAU,CAAA;AAGjE,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,aAAa,CAAQ;IAC7B,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,SAAS,CAAS;gBAGxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;QACP,aAAa,CAAC,EAAE,MAAM,CAAA;QACtB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,UAAU,CAAC,EAAE,MAAM,CAAA;KACf;IA8BR,QAAQ,IAAI,KAAK,EAAE;IAgBnB,OAAO,CAAC,SAAS;IAsCjB,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,YAAY;IA0EpB,OAAO,CAAC,QAAQ;IAmChB,OAAO,CAAC,cAAc;IAwBtB,OAAO,CAAC,mBAAmB;IA6B3B,OAAO,CAAC,UAAU;IAyBlB,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,YAAY;IAuDpB,OAAO,CAAC,WAAW;IAgBnB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,OAAO;IAOf,OAAO,CAAC,KAAK;IAkBb,OAAO,CAAC,KAAK;IAab,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,QAAQ;CAQjB;AAED,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AAC9C,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lexer/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,KAAK,EAAmC,MAAM,UAAU,CAAA;AAGjE,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,aAAa,CAAQ;IAC7B,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,SAAS,CAAS;gBAGxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;QACP,aAAa,CAAC,EAAE,MAAM,CAAA;QACtB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,UAAU,CAAC,EAAE,MAAM,CAAA;KACf;IA8BR,QAAQ,IAAI,KAAK,EAAE;IAgBnB,OAAO,CAAC,SAAS;IAsCjB,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,YAAY;IA0EpB,OAAO,CAAC,QAAQ;IAmChB,OAAO,CAAC,cAAc;IAwBtB,OAAO,CAAC,mBAAmB;IA6B3B,OAAO,CAAC,UAAU;IAyBlB,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,YAAY;IAuDpB,OAAO,CAAC,WAAW;IAgBnB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,OAAO;IAOf,OAAO,CAAC,KAAK;IAkBb,OAAO,CAAC,KAAK;IAab,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,OAAO;IAKf,OAAO,CAAC,OAAO;IAKf,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,QAAQ;CAQjB;AAED,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AAC9C,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAA"}

package/dist/native/index.d.ts

@@ -66,7 +66,16 @@ export declare class NativeLexer {
  */
 export declare function tokenizeCount(source: string): number;
 /**
- * Tokenize with native lexer, auto-cleanup
+ * Tokenize with native lexer, auto-cleanup (OLD - per-token FFI calls)
  */
 export declare function tokenize(source: string): NativeToken[];
+/**
+ * Batch tokenize - single FFI call for all tokens (FAST)
+ * Returns array of [type, start, end] tuples for maximum performance
+ */
+export declare function tokenizeBatch(source: string): Array<[number, number, number]>;
+/**
+ * Batch tokenize with full token objects (includes value extraction)
+ */
+export declare function tokenizeBatchFull(source: string): NativeToken[];
 //# sourceMappingURL=index.d.ts.map

package/dist/native/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/native/index.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;CACd;AAED,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;CAwBZ,CAAA;AAkIV;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAG3C;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,GAAG,IAAI,CAO7C;AAED;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,YAAY,CAAY;IAChC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAiB;gBAErB,MAAM,EAAE,MAAM;IAgC1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKnC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKpC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKlC,QAAQ,IAAI,OAAO;IAKnB,YAAY,IAAI,MAAM;IAKtB,YAAY,IAAI,MAAM;IAKtB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IASpC,YAAY,IAAI,WAAW,EAAE;IAQ7B,IAAI,IAAI,IAAI;IAOZ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAYpD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAOtD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/native/index.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;CACd;AAED,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;CAwBZ,CAAA;AA+IV;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAG3C;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,GAAG,IAAI,CAO7C;AAED;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,YAAY,CAAY;IAChC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAiB;gBAErB,MAAM,EAAE,MAAM;IAgC1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKnC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKpC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAKlC,QAAQ,IAAI,OAAO;IAKnB,YAAY,IAAI,MAAM;IAKtB,YAAY,IAAI,MAAM;IAKtB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IASpC,YAAY,IAAI,WAAW,EAAE;IAQ7B,IAAI,IAAI,IAAI;IAOZ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAYpD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAOtD;AAUD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CA2D7E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAQ/D"}