binja 0.4.2 → 0.4.4

package/dist/native/index.js

@@ -1,6 +1,6 @@
 // @bun
 // src/native/index.ts
-import { dlopen, FFIType, ptr, CString } from "bun:ffi";
+import { dlopen, FFIType, ptr, CString, toArrayBuffer } from "bun:ffi";
 import { join, basename } from "path";
 import { existsSync } from "fs";
 var TokenType = {
@@ -65,6 +65,18 @@ var symbols = {
     args: [FFIType.ptr],
     returns: FFIType.u32
   },
+  binja_lexer_get_tokens_buffer: {
+    args: [FFIType.ptr],
+    returns: FFIType.ptr
+  },
+  binja_lexer_tokens_buffer_size: {
+    args: [FFIType.ptr],
+    returns: FFIType.u64
+  },
+  binja_free_tokens_buffer: {
+    args: [FFIType.ptr, FFIType.u64],
+    returns: FFIType.void
+  },
   binja_tokenize_count: {
     args: [FFIType.ptr, FFIType.u64],
     returns: FFIType.u64
@@ -243,8 +255,71 @@ function tokenize(source) {
     lexer.free();
   }
 }
+var ERROR_MESSAGES = {
+  1: "Unterminated string",
+  2: "Unclosed template tag",
+  3: "Invalid operator",
+  4: "Unexpected character"
+};
+function tokenizeBatch(source) {
+  if (source.length === 0) {
+    return [[TokenType.EOF, 0, 0]];
+  }
+  const lib = loadLibrary();
+  if (!lib) {
+    throw new Error("Native library not available");
+  }
+  const sourceBuffer = new TextEncoder().encode(source);
+  const lexerPtr = lib.symbols.binja_lexer_new(ptr(sourceBuffer), sourceBuffer.length);
+  if (!lexerPtr) {
+    throw new Error("Failed to create native lexer");
+  }
+  try {
+    if (lib.symbols.binja_lexer_has_error(lexerPtr)) {
+      const errorCode = Number(lib.symbols.binja_lexer_error_code(lexerPtr));
+      const errorLine = Number(lib.symbols.binja_lexer_error_line(lexerPtr));
+      const message = ERROR_MESSAGES[errorCode] ?? "Unknown error";
+      throw new Error(`${message} at line ${errorLine}`);
+    }
+    const bufferSize = Number(lib.symbols.binja_lexer_tokens_buffer_size(lexerPtr));
+    const bufferPtr = lib.symbols.binja_lexer_get_tokens_buffer(lexerPtr);
+    if (!bufferPtr) {
+      throw new Error("Failed to get tokens buffer");
+    }
+    try {
+      const buffer = new Uint8Array(toArrayBuffer(bufferPtr, 0, bufferSize));
+      const view = new DataView(buffer.buffer);
+      const count = view.getUint32(0, true);
+      const tokens = new Array(count);
+      let offset = 4;
+      for (let i = 0;i < count; i++) {
+        const type = buffer[offset];
+        const start = view.getUint32(offset + 1, true);
+        const end = view.getUint32(offset + 5, true);
+        tokens[i] = [type, start, end];
+        offset += 9;
+      }
+      return tokens;
+    } finally {
+      lib.symbols.binja_free_tokens_buffer(bufferPtr, bufferSize);
+    }
+  } finally {
+    lib.symbols.binja_lexer_free(lexerPtr);
+  }
+}
+function tokenizeBatchFull(source) {
+  const tuples = tokenizeBatch(source);
+  return tuples.map(([type, start, end]) => ({
+    type,
+    start,
+    end,
+    value: source.slice(start, end)
+  }));
+}
 export {
   tokenizeCount,
+  tokenizeBatchFull,
+  tokenizeBatch,
   tokenize,
   nativeVersion,
   isNativeAvailable,

package/dist/runtime/context.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/runtime/context.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,OAAO,CAAA;IACd,IAAI,EAAE,OAAO,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IAEd,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAA;IAC9B,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAA;IAChC,QAAQ,CAAC,EAAE,GAAG,CAAA;IACd,QAAQ,CAAC,EAAE,GAAG,CAAA;CACf;AAuDD,qBAAa,OAAO;IAElB,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,eAAe,CAAyB;IAEhD,OAAO,CAAC,eAAe,CAAuB;IAE9C,OAAO,CAAC,aAAa,CAAqB;gBAE9B,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,EAAE,MAAM,GAAE,OAAO,GAAG,IAAW;IAOzE,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG;IAkBtB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI;IAKnC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAO1B,IAAI,CAAC,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GAAG,IAAI;IAM1C,GAAG,IAAI,IAAI;IAOX,OAAO,CAAC,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GAAG,OAAO;IAKhD,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO;IAYjD,UAAU,IAAI,IAAI;IAOlB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI;IAwBhD,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAehC"}
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/runtime/context.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,OAAO,CAAA;IACd,IAAI,EAAE,OAAO,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IAEd,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAA;IAC9B,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAA;IAChC,QAAQ,CAAC,EAAE,GAAG,CAAA;IACd,QAAQ,CAAC,EAAE,GAAG,CAAA;CACf;AAuDD,qBAAa,OAAO;IAElB,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,eAAe,CAAyB;IAEhD,OAAO,CAAC,eAAe,CAAuB;IAE9C,OAAO,CAAC,aAAa,CAAqB;gBAG9B,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,EAAE,MAAM,GAAE,OAAO,GAAG,IAAW;IAMzE,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG;IAkBtB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI;IAKnC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAQ1B,IAAI,CAAC,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GAAG,IAAI;IAK1C,GAAG,IAAI,IAAI;IAOX,OAAO,CAAC,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GAAG,OAAO;IAKhD,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO;IAYjD,UAAU,IAAI,IAAI;IAOlB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI;IAwBhD,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAehC"}

package/dist/runtime/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/runtime/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAkB,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3D,OAAO,EAAgB,YAAY,EAAE,MAAM,UAAU,CAAA;AACrD,OAAO,KAAK,EAEV,YAAY,EAcZ,cAAc,EAcf,MAAM,iBAAiB,CAAA;AAExB,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC7B,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,MAAM,CAAA;IAChF,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAA;IACzC,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,CAAA;IACxD,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,OAAO,CAAoE;IACnF,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,MAAM,CAAoC;IAClD,OAAO,CAAC,cAAc,CAA4B;gBAEtC,OAAO,GAAE,cAAmB;IAkClC,MAAM,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAuBnF,OAAO,CAAC,kBAAkB;IA0B1B,OAAO,CAAC,cAAc;IA4BtB,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,cAAc;IA+BtB,OAAO,CAAC,YAAY;IAYpB,OAAO,CAAC,aAAa;IA6CrB,OAAO,CAAC,eAAe;IAcvB,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,aAAa;IAgBrB,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,iBAAiB;IA+CzB,OAAO,CAAC,UAAU;IA4ClB,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,IAAI;IAwCZ,OAAO,CAAC,WAAW;IAenB,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,YAAY;IAmBpB,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,WAAW;IAwBnB,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,gBAAgB;IAWxB,OAAO,CAAC,QAAQ;IAuBhB,OAAO,CAAC,cAAc;YAWR,aAAa;YAYb,mBAAmB;YASnB,eAAe;YAkCf,aAAa;YAYb,cAAc;YA6Cd,gBAAgB;YAchB,aAAa;YAoBb,eAAe;YAUf,gBAAgB;IAUxB,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;IAMhE,OAAO,CAAC,SAAS;IAiBjB,OAAO,CAAC,QAAQ;IAchB,OAAO,CAAC,IAAI;IAOZ,OAAO,CAAC,UAAU;IAoBlB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,cAAc,GAAG,IAAI;IAKjD,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,GAAG,IAAI;IAK7C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI;CAG1C;AAED,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/runtime/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAkB,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3D,OAAO,EAAgB,YAAY,EAAE,MAAM,UAAU,CAAA;AACrD,OAAO,KAAK,EAEV,YAAY,EAcZ,cAAc,EAcf,MAAM,iBAAiB,CAAA;AAExB,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC7B,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,MAAM,CAAA;IAChF,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAA;IACzC,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,CAAA;IACxD,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,OAAO,CAAoE;IACnF,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,MAAM,CAAoC;IAClD,OAAO,CAAC,cAAc,CAA4B;gBAEtC,OAAO,GAAE,cAAmB;IAkClC,MAAM,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAuBnF,OAAO,CAAC,kBAAkB;IA0B1B,OAAO,CAAC,cAAc;IA4BtB,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,cAAc;IA+BtB,OAAO,CAAC,YAAY;IAYpB,OAAO,CAAC,aAAa;IAwDrB,OAAO,CAAC,eAAe;IAcvB,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,aAAa;IAgBrB,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,iBAAiB;IA+CzB,OAAO,CAAC,UAAU;IA4ClB,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,IAAI;IAyCZ,OAAO,CAAC,WAAW;IA0BnB,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,YAAY;IAmBpB,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,WAAW;IAwBnB,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,gBAAgB;IAWxB,OAAO,CAAC,QAAQ;IAuBhB,OAAO,CAAC,cAAc;YAWR,aAAa;YAYb,mBAAmB;YASnB,eAAe;YAkCf,aAAa;YAYb,cAAc;YA6Cd,gBAAgB;YAchB,aAAa;YAoBb,eAAe;YAUf,gBAAgB;IAUxB,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;IAMhE,OAAO,CAAC,SAAS;IAiBjB,OAAO,CAAC,QAAQ;IAchB,OAAO,CAAC,IAAI;IAOZ,OAAO,CAAC,UAAU;IA0BlB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,cAAc,GAAG,IAAI;IAKjD,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,GAAG,IAAI;IAK7C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI;CAG1C;AAED,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "binja",
-  "version": "0.4.2",
+  "version": "0.4.4",
   "description": "High-performance Jinja2/Django template engine for Bun",
   "main": "dist/index.js",
   "module": "dist/index.js",