binja 0.3.4 → 0.4.1

package/dist/native/index.js

@@ -0,0 +1,252 @@
+// @bun
+// src/native/index.ts
+import { dlopen, FFIType, ptr, CString } from "bun:ffi";
+import { join } from "path";
+import { existsSync } from "fs";
+var TokenType = {
+  TEXT: 0,
+  VAR_START: 1,
+  VAR_END: 2,
+  BLOCK_START: 3,
+  BLOCK_END: 4,
+  COMMENT_START: 5,
+  COMMENT_END: 6,
+  IDENTIFIER: 7,
+  STRING: 8,
+  NUMBER: 9,
+  OPERATOR: 10,
+  DOT: 11,
+  COMMA: 12,
+  PIPE: 13,
+  COLON: 14,
+  LPAREN: 15,
+  RPAREN: 16,
+  LBRACKET: 17,
+  RBRACKET: 18,
+  LBRACE: 19,
+  RBRACE: 20,
+  ASSIGN: 21,
+  EOF: 22
+};
+var symbols = {
+  binja_lexer_new: {
+    args: [FFIType.ptr, FFIType.u64],
+    returns: FFIType.ptr
+  },
+  binja_lexer_free: {
+    args: [FFIType.ptr],
+    returns: FFIType.void
+  },
+  binja_lexer_token_count: {
+    args: [FFIType.ptr],
+    returns: FFIType.u64
+  },
+  binja_lexer_token_type: {
+    args: [FFIType.ptr, FFIType.u64],
+    returns: FFIType.u8
+  },
+  binja_lexer_token_start: {
+    args: [FFIType.ptr, FFIType.u64],
+    returns: FFIType.u32
+  },
+  binja_lexer_token_end: {
+    args: [FFIType.ptr, FFIType.u64],
+    returns: FFIType.u32
+  },
+  binja_lexer_has_error: {
+    args: [FFIType.ptr],
+    returns: FFIType.bool
+  },
+  binja_lexer_error_code: {
+    args: [FFIType.ptr],
+    returns: FFIType.u8
+  },
+  binja_lexer_error_line: {
+    args: [FFIType.ptr],
+    returns: FFIType.u32
+  },
+  binja_tokenize_count: {
+    args: [FFIType.ptr, FFIType.u64],
+    returns: FFIType.u64
+  },
+  binja_version: {
+    args: [],
+    returns: FFIType.ptr
+  }
+};
+var _lib = null;
+var _loadAttempted = false;
+var _nativeAvailable = false;
+function getLibraryPath() {
+  const platform = process.platform;
+  const arch = process.arch;
+  const libExt = platform === "darwin" ? "dylib" : platform === "win32" ? "dll" : "so";
+  const libName = `libbinja.${libExt}`;
+  const projectRoot = join(import.meta.dir, "..", "..");
+  const searchPaths = [
+    join(projectRoot, "native", `${platform}-${arch}`, libName),
+    join(projectRoot, "native", libName),
+    join(projectRoot, "zig-native", "zig-out", "lib", libName),
+    join(projectRoot, "zig-native", libName),
+    join(import.meta.dir, libName)
+  ];
+  for (const p of searchPaths) {
+    if (existsSync(p)) {
+      return p;
+    }
+  }
+  return null;
+}
+function loadLibrary() {
+  if (_loadAttempted) {
+    return _lib;
+  }
+  _loadAttempted = true;
+  const libPath = getLibraryPath();
+  if (!libPath) {
+    console.warn("[binja] Native library not found, using pure JS fallback");
+    return null;
+  }
+  try {
+    _lib = dlopen(libPath, symbols);
+    _nativeAvailable = true;
+    return _lib;
+  } catch (e) {
+    console.warn(`[binja] Failed to load native library: ${e}`);
+    return null;
+  }
+}
+function isNativeAvailable() {
+  loadLibrary();
+  return _nativeAvailable;
+}
+function nativeVersion() {
+  const lib = loadLibrary();
+  if (!lib)
+    return null;
+  const versionPtr = lib.symbols.binja_version();
+  if (!versionPtr)
+    return null;
+  return new CString(versionPtr).toString();
+}
+
+class NativeLexer {
+  lexerPtr = 0;
+  source;
+  sourceBuffer;
+  lib;
+  _tokenCount = 0;
+  _isEmpty = false;
+  constructor(source) {
+    const lib = loadLibrary();
+    if (!lib) {
+      throw new Error("Native library not available. Use isNativeAvailable() to check first.");
+    }
+    this.lib = lib;
+    this.source = source;
+    if (source.length === 0) {
+      this._isEmpty = true;
+      this._tokenCount = 1;
+      this.sourceBuffer = new Uint8Array(0);
+      return;
+    }
+    this.sourceBuffer = new TextEncoder().encode(source);
+    const result = this.lib.symbols.binja_lexer_new(ptr(this.sourceBuffer), this.sourceBuffer.length);
+    if (!result) {
+      throw new Error("Failed to create native lexer");
+    }
+    this.lexerPtr = result;
+    this._tokenCount = Number(this.lib.symbols.binja_lexer_token_count(this.lexerPtr));
+  }
+  get tokenCount() {
+    return this._tokenCount;
+  }
+  getTokenType(index) {
+    if (this._isEmpty)
+      return TokenType.EOF;
+    return Number(this.lib.symbols.binja_lexer_token_type(this.lexerPtr, index));
+  }
+  getTokenStart(index) {
+    if (this._isEmpty)
+      return 0;
+    return Number(this.lib.symbols.binja_lexer_token_start(this.lexerPtr, index));
+  }
+  getTokenEnd(index) {
+    if (this._isEmpty)
+      return 0;
+    return Number(this.lib.symbols.binja_lexer_token_end(this.lexerPtr, index));
+  }
+  hasError() {
+    if (this._isEmpty)
+      return false;
+    return Boolean(this.lib.symbols.binja_lexer_has_error(this.lexerPtr));
+  }
+  getErrorCode() {
+    if (this._isEmpty)
+      return 0;
+    return Number(this.lib.symbols.binja_lexer_error_code(this.lexerPtr));
+  }
+  getErrorLine() {
+    if (this._isEmpty)
+      return 1;
+    return Number(this.lib.symbols.binja_lexer_error_line(this.lexerPtr));
+  }
+  getTokenValue(index) {
+    if (this._isEmpty)
+      return "";
+    const start = this.getTokenStart(index);
+    const end = this.getTokenEnd(index);
+    return new TextDecoder().decode(this.sourceBuffer.slice(start, end));
+  }
+  getToken(index) {
+    return {
+      type: this.getTokenType(index),
+      start: this.getTokenStart(index),
+      end: this.getTokenEnd(index),
+      value: this.getTokenValue(index)
+    };
+  }
+  getAllTokens() {
+    const tokens = [];
+    for (let i = 0;i < this._tokenCount; i++) {
+      tokens.push(this.getToken(i));
+    }
+    return tokens;
+  }
+  free() {
+    if (this.lexerPtr) {
+      this.lib.symbols.binja_lexer_free(this.lexerPtr);
+      this.lexerPtr = null;
+    }
+  }
+  [Symbol.dispose]() {
+    this.free();
+  }
+}
+function tokenizeCount(source) {
+  if (source.length === 0) {
+    return 1;
+  }
+  const lib = loadLibrary();
+  if (!lib) {
+    throw new Error("Native library not available");
+  }
+  const bytes = new TextEncoder().encode(source);
+  return Number(lib.symbols.binja_tokenize_count(ptr(bytes), bytes.length));
+}
+function tokenize(source) {
+  const lexer = new NativeLexer(source);
+  try {
+    return lexer.getAllTokens();
+  } finally {
+    lexer.free();
+  }
+}
+export {
+  tokenizeCount,
+  tokenize,
+  nativeVersion,
+  isNativeAvailable,
+  TokenType,
+  NativeLexer
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "binja",
-  "version": "0.3.4",
+  "version": "0.4.1",
   "description": "High-performance Jinja2/Django template engine for Bun",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -15,6 +15,12 @@
       "types": "./dist/index.d.ts",
       "bun": "./dist/index.js"
     },
+    "./native": {
+      "import": "./dist/native/index.js",
+      "require": "./dist/native/index.js",
+      "types": "./dist/native/index.d.ts",
+      "bun": "./dist/native/index.js"
+    },
     "./debug": {
       "import": "./dist/debug/index.js",
       "require": "./dist/debug/index.js",
@@ -24,22 +30,25 @@
   },
   "files": [
     "dist",
+    "native",
     "README.md",
     "LICENSE"
   ],
   "scripts": {
-    "build": "bun build ./src/index.ts --outdir ./dist --target bun && bun build ./src/cli.ts --outdir ./dist --target bun && bun run build:types",
+    "build": "bun build ./src/index.ts --outdir ./dist --target bun && bun build ./src/cli.ts --outdir ./dist --target bun && bun build ./src/native/index.ts --outdir ./dist/native --target bun && bun build ./src/debug/index.ts --outdir ./dist/debug --target bun && bun run build:types",
     "build:types": "tsc --declaration --emitDeclarationOnly --outDir ./dist",
+    "build:native": "./scripts/build-native.sh",
+    "build:all": "bun run build:native && bun run build",
     "test": "bun test",
     "dev": "bun --watch src/index.ts",
     "typecheck": "tsc --noEmit",
-    "prepublishOnly": "bun run build",
+    "prepublishOnly": "bun run build:all",
     "version:patch": "npm version patch --no-git-tag-version",
     "version:minor": "npm version minor --no-git-tag-version",
     "version:major": "npm version major --no-git-tag-version",
-    "release:patch": "bun run version:patch && bun run build",
-    "release:minor": "bun run version:minor && bun run build",
-    "release:major": "bun run version:major && bun run build"
+    "release:patch": "bun run version:patch && bun run build:all",
+    "release:minor": "bun run version:minor && bun run build:all",
+    "release:major": "bun run version:major && bun run build:all"
   },
   "repository": {
     "type": "git",
@@ -58,7 +67,10 @@
     "ssr",
     "template-engine",
     "django-templates",
-    "typescript"
+    "typescript",
+    "zig",
+    "native",
+    "ffi"
   ],
   "author": "egeominotti",
   "license": "BSD-3-Clause",