jiren 3.1.0 → 3.3.0

package/components/cache.ts

@@ -1,451 +0,0 @@
-import {
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  writeFileSync,
-  unlinkSync,
-  readdirSync,
-  statSync,
-} from "fs";
-import { gzipSync, gunzipSync } from "zlib";
-import { createHash } from "crypto";
-import { join } from "path";
-import type {
-  JirenResponse,
-  JirenResponseBody,
-  SerializableCacheEntry,
-  CacheEntry,
-} from "./types.js";
-
-function reconstructBody(bodyData: string, isText: boolean): JirenResponseBody {
-  let buffer: ArrayBuffer;
-
-  if (isText) {
-    // Text data stored directly
-    const encoder = new TextEncoder();
-    buffer = encoder.encode(bodyData).buffer as ArrayBuffer;
-  } else {
-    // Binary data stored as base64
-    const binaryString = atob(bodyData);
-    const bytes = new Uint8Array(binaryString.length);
-    for (let i = 0; i < binaryString.length; i++) {
-      bytes[i] = binaryString.charCodeAt(i);
-    }
-    buffer = bytes.buffer as ArrayBuffer;
-  }
-
-  return {
-    bodyUsed: false,
-    text: async () => {
-      if (isText) return bodyData;
-      const decoder = new TextDecoder();
-      return decoder.decode(buffer);
-    },
-    json: async () => {
-      const text = isText ? bodyData : new TextDecoder().decode(buffer);
-      return JSON.parse(text);
-    },
-    arrayBuffer: async () => buffer,
-    jsonFields: async <T extends Record<string, any> = any>(
-      _fields: (keyof T)[]
-    ) => ({}),
-    blob: async () => new Blob([buffer]),
-  };
-}
-
-interface LRUNode {
-  key: string;
-  entry: CacheEntry;
-  prev: LRUNode | null;
-  next: LRUNode | null;
-}
-
-class L1MemoryCache {
-  private capacity: number;
-  private cache: Map<string, LRUNode> = new Map();
-  private head: LRUNode | null = null; // Most recently used
-  private tail: LRUNode | null = null; // Least recently used
-
-  // Stats
-  public hits = 0;
-  public misses = 0;
-
-  constructor(capacity = 100) {
-    this.capacity = capacity;
-  }
-
-  get(key: string): CacheEntry | null {
-    const node = this.cache.get(key);
-    if (!node) {
-      this.misses++;
-      return null;
-    }
-
-    // Check if expired
-    const now = Date.now();
-    if (now - node.entry.timestamp > node.entry.ttl) {
-      this.delete(key);
-      this.misses++;
-      return null;
-    }
-
-    // Move to front (most recently used)
-    this.moveToFront(node);
-    this.hits++;
-    return node.entry;
-  }
-
-  set(key: string, entry: CacheEntry): void {
-    // If key exists, update and move to front
-    const existing = this.cache.get(key);
-    if (existing) {
-      existing.entry = entry;
-      this.moveToFront(existing);
-      return;
-    }
-
-    // Create new node
-    const node: LRUNode = {
-      key,
-      entry,
-      prev: null,
-      next: this.head,
-    };
-
-    // Add to front
-    if (this.head) {
-      this.head.prev = node;
-    }
-    this.head = node;
-    if (!this.tail) {
-      this.tail = node;
-    }
-
-    this.cache.set(key, node);
-
-    // Evict LRU if over capacity
-    if (this.cache.size > this.capacity) {
-      this.evictLRU();
-    }
-  }
-
-  delete(key: string): void {
-    const node = this.cache.get(key);
-    if (!node) return;
-
-    this.removeNode(node);
-    this.cache.delete(key);
-  }
-
-  clear(): void {
-    this.cache.clear();
-    this.head = null;
-    this.tail = null;
-  }
-
-  stats() {
-    return {
-      size: this.cache.size,
-      capacity: this.capacity,
-      hits: this.hits,
-      misses: this.misses,
-      hitRate:
-        this.hits + this.misses > 0
-          ? ((this.hits / (this.hits + this.misses)) * 100).toFixed(2) + "%"
-          : "0%",
-    };
-  }
-
-  private moveToFront(node: LRUNode): void {
-    if (node === this.head) return; // Already at front
-
-    this.removeNode(node);
-
-    // Add to front
-    node.prev = null;
-    node.next = this.head;
-    if (this.head) {
-      this.head.prev = node;
-    }
-    this.head = node;
-    if (!this.tail) {
-      this.tail = node;
-    }
-  }
-
-  private removeNode(node: LRUNode): void {
-    if (node.prev) {
-      node.prev.next = node.next;
-    } else {
-      this.head = node.next;
-    }
-
-    if (node.next) {
-      node.next.prev = node.prev;
-    } else {
-      this.tail = node.prev;
-    }
-  }
-
-  private evictLRU(): void {
-    if (!this.tail) return;
-
-    const key = this.tail.key;
-    this.removeNode(this.tail);
-    this.cache.delete(key);
-  }
-}
-
-export class ResponseCache {
-  private l1: L1MemoryCache;
-  private cacheDir: string;
-  private maxDiskSize: number;
-
-  constructor(l1Capacity = 100, cacheDir = ".cache/jiren", maxDiskSize = 100) {
-    this.l1 = new L1MemoryCache(l1Capacity);
-    this.maxDiskSize = maxDiskSize;
-    this.cacheDir = cacheDir;
-
-    // Create cache directory if it doesn't exist
-    if (!existsSync(this.cacheDir)) {
-      mkdirSync(this.cacheDir, { recursive: true });
-    }
-  }
-
-  private generateKey(url: string, path?: string, options?: any): string {
-    const fullUrl = path ? `${url}${path}` : url;
-    const method = options?.method || "GET";
-    const headers = JSON.stringify(options?.headers || {});
-    const key = `${method}:${fullUrl}:${headers}`;
-
-    // Hash the key to create a valid filename
-    return createHash("md5").update(key).digest("hex");
-  }
-
-  private getCacheFilePath(key: string): string {
-    return join(this.cacheDir, `${key}.json.gz`);
-  }
-
-  preloadL1(url: string, path?: string, options?: any): boolean {
-    const key = this.generateKey(url, path, options);
-
-    // Check if already in L1
-    if (this.l1.get(key)) {
-      return true;
-    }
-
-    // Try to load from L2 disk
-    const l2Entry = this.getFromDisk(key);
-    if (l2Entry) {
-      this.l1.set(key, l2Entry);
-      return true;
-    }
-
-    return false;
-  }
-
-  get(url: string, path?: string, options?: any): JirenResponse | null {
-    const key = this.generateKey(url, path, options);
-
-    // L1: Check in-memory cache first (~0.001ms)
-    const l1Entry = this.l1.get(key);
-    if (l1Entry) {
-      return l1Entry.response;
-    }
-
-    // L2: Check disk cache (~5ms)
-    const l2Entry = this.getFromDisk(key);
-    if (l2Entry) {
-      // Promote to L1 for faster future access
-      this.l1.set(key, l2Entry);
-      return l2Entry.response;
-    }
-
-    return null;
-  }
-
-  private getFromDisk(key: string): CacheEntry | null {
-    const filePath = this.getCacheFilePath(key);
-
-    if (!existsSync(filePath)) return null;
-
-    try {
-      // Read compressed file
-      const compressed = readFileSync(filePath);
-
-      // Decompress
-      const decompressed = gunzipSync(compressed);
-      const data = decompressed.toString("utf-8");
-      const serialized: SerializableCacheEntry = JSON.parse(data);
-
-      // Check if expired
-      const now = Date.now();
-      if (now - serialized.timestamp > serialized.ttl) {
-        // Delete expired cache file
-        try {
-          unlinkSync(filePath);
-        } catch {}
-        return null;
-      }
-
-      // Reconstruct the response with working body methods
-      const response: JirenResponse = {
-        status: serialized.status,
-        statusText: serialized.statusText,
-        headers: serialized.headers,
-        url: serialized.url,
-        ok: serialized.ok,
-        redirected: serialized.redirected,
-        type: serialized.type || "default",
-        body: reconstructBody(serialized.bodyData, serialized.bodyIsText),
-      };
-
-      return {
-        response,
-        timestamp: serialized.timestamp,
-        ttl: serialized.ttl,
-      };
-    } catch (error) {
-      // Invalid cache file, delete it
-      try {
-        unlinkSync(filePath);
-      } catch {}
-      return null;
-    }
-  }
-
-  set(
-    url: string,
-    response: JirenResponse,
-    ttl: number,
-    path?: string,
-    options?: any
-  ): void {
-    const key = this.generateKey(url, path, options);
-
-    const entry: CacheEntry = {
-      response,
-      timestamp: Date.now(),
-      ttl,
-    };
-
-    // L1: Store in memory (~0.001ms)
-    this.l1.set(key, entry);
-
-    // L2: Store on disk (async-ish, for persistence)
-    this.setToDisk(key, entry);
-  }
-
-  private async setToDisk(key: string, entry: CacheEntry): Promise<void> {
-    const filePath = this.getCacheFilePath(key);
-
-    try {
-      // Extract body text for serialization
-      let bodyData: string;
-      let bodyIsText = true;
-
-      try {
-        bodyData = await entry.response.body.text();
-      } catch {
-        // If text fails, try arrayBuffer and convert to base64
-        try {
-          const buffer = await entry.response.body.arrayBuffer();
-          const bytes = new Uint8Array(buffer);
-          let binary = "";
-          for (let i = 0; i < bytes.length; i++) {
-            binary += String.fromCharCode(bytes[i]!);
-          }
-          bodyData = btoa(binary);
-          bodyIsText = false;
-        } catch {
-          bodyData = "";
-        }
-      }
-
-      // Create serializable entry
-      const serialized: SerializableCacheEntry = {
-        status: entry.response.status,
-        statusText: entry.response.statusText,
-        headers: entry.response.headers as Record<string, string>,
-        url: entry.response.url,
-        ok: entry.response.ok,
-        redirected: entry.response.redirected,
-        type: entry.response.type || "default",
-        bodyData,
-        bodyIsText,
-        timestamp: entry.timestamp,
-        ttl: entry.ttl,
-      };
-
-      // Convert to JSON
-      const json = JSON.stringify(serialized);
-
-      // Compress with gzip
-      const compressed = gzipSync(json);
-
-      // Write compressed file
-      writeFileSync(filePath, compressed);
-    } catch (error) {
-      // Silently fail if can't write cache
-      console.warn("Failed to write cache:", error);
-    }
-  }
-
-  clear(url?: string): void {
-    // Clear L1
-    this.l1.clear();
-
-    // Clear L2
-    if (url) {
-      // Clear all cache files for this URL
-      // This is approximate since we hash the keys
-      // For now, just clear all to be safe
-      this.clearAllDisk();
-    } else {
-      this.clearAllDisk();
-    }
-  }
-
-  private clearAllDisk(): void {
-    try {
-      const files = readdirSync(this.cacheDir);
-      for (const file of files) {
-        if (file.endsWith(".json.gz")) {
-          unlinkSync(join(this.cacheDir, file));
-        }
-      }
-    } catch (error) {
-      // Silently fail
-    }
-  }
-
-  stats() {
-    const l1Stats = this.l1.stats();
-
-    // L2 disk stats
-    let diskSize = 0;
-    let diskFiles = 0;
-    let totalDiskBytes = 0;
-
-    try {
-      const files = readdirSync(this.cacheDir);
-      const cacheFiles = files.filter((f: string) => f.endsWith(".json.gz"));
-      diskFiles = cacheFiles.length;
-
-      for (const file of cacheFiles) {
-        const stats = statSync(join(this.cacheDir, file));
-        totalDiskBytes += stats.size;
-      }
-    } catch {}
-
-    return {
-      l1: l1Stats,
-      l2: {
-        size: diskFiles,
-        maxSize: this.maxDiskSize,
-        cacheDir: this.cacheDir,
-        totalSizeKB: (totalDiskBytes / 1024).toFixed(2),
-      },
-    };
-  }
-}

package/components/native-json.ts

@@ -1,195 +0,0 @@
-/**
- * Native JSON Parser
- *
- * Uses native Zig std.json for parsing large JSON payloads.
- * For small payloads (<10KB), JavaScript JSON.parse is faster due to FFI overhead.
- * For large payloads (>10KB), native parsing is 2-3x faster.
- */
-
-import { lib } from "./native";
-import { toArrayBuffer } from "bun:ffi";
-
-type Pointer = number | null;
-
-// Threshold in bytes above which native JSON parsing is faster
-const NATIVE_JSON_THRESHOLD = 10240; // 10KB
-
-/**
- * Parse JSON with automatic native/JS fallback
- * Uses native Zig parser for large payloads, JS for small ones
- *
- * Note: For full object conversion, JS JSON.parse is still used.
- * Native parsing benefits are:
- * 1. validateJson() - Fast validation without parsing
- * 2. NativeJsonHandle - Extract specific fields without full parse
- * 3. parseJsonFields() - Get only needed fields from large JSON
- */
-export function parseJson<T = any>(data: string | Buffer): T {
-  const buffer = typeof data === "string" ? Buffer.from(data) : data;
-  // For full object conversion, JS JSON.parse is most efficient
-  return JSON.parse(buffer.toString("utf-8"));
-}
-
-/**
- * Force native JSON validation before parsing
- * Validates natively (fast) then parses with JS
- * Use when you want to ensure valid JSON before parsing
- */
-export function parseJsonSafe<T = any>(data: string | Buffer): T {
-  const buffer = typeof data === "string" ? Buffer.from(data) : data;
-
-  // Validate natively first
-  const isValid = lib.symbols.zjson_validate(buffer, buffer.length);
-  if (!isValid) {
-    throw new SyntaxError("Invalid JSON");
-  }
-
-  // Parse with JS
-  return JSON.parse(buffer.toString("utf-8"));
-}
-
-/**
- * Validate JSON without parsing (fast check)
- * ~2x faster than try { JSON.parse } catch
- */
-export function validateJson(data: string | Buffer): boolean {
-  const buffer = typeof data === "string" ? Buffer.from(data) : data;
-  return lib.symbols.zjson_validate(buffer, buffer.length) as boolean;
-}
-
-/**
- * Native JSON handle for accessing parsed data without JS conversion
- * Useful for extracting specific fields from large JSON without full parse
- */
-export class NativeJsonHandle {
-  private ptr: Pointer;
-
-  constructor(data: string | Buffer) {
-    const buffer = typeof data === "string" ? Buffer.from(data) : data;
-    this.ptr = lib.symbols.zjson_parse(buffer, buffer.length) as Pointer;
-
-    if (!this.ptr) {
-      throw new SyntaxError("Failed to parse JSON");
-    }
-  }
-
-  /**
-   * Get string value by key
-   */
-  getString(key: string): string | null {
-    if (!this.ptr) return null;
-
-    const keyBuf = Buffer.from(key + "\0");
-    const strPtr = lib.symbols.zjson_get_string(
-      this.ptr as any,
-      keyBuf
-    ) as Pointer;
-
-    if (!strPtr) return null;
-
-    const len = Number(
-      lib.symbols.zjson_get_string_len(this.ptr as any, keyBuf)
-    );
-    if (len === 0) return "";
-
-    return Buffer.from(toArrayBuffer(strPtr as any, 0, len)).toString("utf-8");
-  }
-
-  /**
-   * Get integer value by key
-   */
-  getInt(key: string): number {
-    if (!this.ptr) return 0;
-    const keyBuf = Buffer.from(key + "\0");
-    return Number(lib.symbols.zjson_get_int(this.ptr as any, keyBuf));
-  }
-
-  /**
-   * Get boolean value by key
-   */
-  getBool(key: string): boolean {
-    if (!this.ptr) return false;
-    const keyBuf = Buffer.from(key + "\0");
-    return lib.symbols.zjson_get_bool(this.ptr as any, keyBuf) as boolean;
-  }
-
-  /**
-   * Check if key exists
-   */
-  hasKey(key: string): boolean {
-    if (!this.ptr) return false;
-    const keyBuf = Buffer.from(key + "\0");
-    return lib.symbols.zjson_has_key(this.ptr as any, keyBuf) as boolean;
-  }
-
-  /**
-   * Get type (0=null, 1=bool, 2=int, 3=float, 4=string, 5=array, 6=object)
-   */
-  getType(): number {
-    if (!this.ptr) return 0;
-    return Number(lib.symbols.zjson_get_type(this.ptr as any));
-  }
-
-  /**
-   * Get array length (if root is array)
-   */
-  getArrayLength(): number {
-    if (!this.ptr) return 0;
-    return Number(lib.symbols.zjson_array_len(this.ptr as any));
-  }
-
-  /**
-   * Get object key count (if root is object)
-   */
-  getObjectLength(): number {
-    if (!this.ptr) return 0;
-    return Number(lib.symbols.zjson_object_len(this.ptr as any));
-  }
-
-  /**
-   * Free native resources
-   */
-  close(): void {
-    if (this.ptr) {
-      lib.symbols.zjson_free(this.ptr as any);
-      this.ptr = null;
-    }
-  }
-}
-
-/**
- * Parse JSON and get specific fields without full JS object conversion
- * More efficient for extracting a few fields from large JSON
- */
-export function parseJsonFields<T extends Record<string, any>>(
-  data: string | Buffer,
-  fields: (keyof T)[]
-): Partial<T> {
-  const handle = new NativeJsonHandle(data);
-  const result: Partial<T> = {};
-
-  try {
-    for (const field of fields) {
-      const key = String(field);
-      if (handle.hasKey(key)) {
-        // Try string first (most common)
-        const strVal = handle.getString(key);
-        if (strVal !== null) {
-          (result as any)[field] = strVal;
-        } else {
-          // Try int
-          const intVal = handle.getInt(key);
-          if (intVal !== 0) {
-            (result as any)[field] = intVal;
-          } else {
-            // Try bool
-            (result as any)[field] = handle.getBool(key);
-          }
-        }
-      }
-    }
-    return result;
-  } finally {
-    handle.close();
-  }
-}

package/components/persistent-worker.ts

@@ -1,67 +0,0 @@
-import { CString, type Pointer } from "bun:ffi";
-import { lib } from "./native";
-
-const ptr: Pointer | null = lib.symbols.zclient_new();
-if (!ptr) {
-  console.error("ERROR: Failed to create client");
-  process.exit(1);
-}
-
-lib.symbols.zclient_set_benchmark_mode(ptr, true);
-
-const decoder = new TextDecoder();
-
-process.stdin.on("data", async (chunk: Buffer) => {
-  const lines = decoder.decode(chunk).trim().split("\n");
-
-  for (const line of lines) {
-    if (!line) continue;
-
-    try {
-      const { id, url, method = "GET" } = JSON.parse(line);
-
-      if (url === "PREFETCH") {
-        console.log(JSON.stringify({ id, type: "prefetch_done" }));
-        continue;
-      }
-
-      if (url === "QUIT") {
-        lib.symbols.zclient_free(ptr);
-        process.exit(0);
-      }
-
-      const methodBuffer = Buffer.from(method + "\0");
-      const urlBuffer = Buffer.from(url + "\0");
-
-      const respPtr = lib.symbols.zclient_request_full(
-        ptr,
-        methodBuffer,
-        urlBuffer,
-        null,
-        null,
-        5,
-        false
-      );
-
-      if (!respPtr) {
-        console.log(
-          JSON.stringify({ id, success: false, error: "Request failed" })
-        );
-        continue;
-      }
-
-      const status = lib.symbols.zfull_response_status(respPtr);
-      const bodyLen = Number(lib.symbols.zfull_response_body_len(respPtr));
-
-      lib.symbols.zclient_response_full_free(respPtr);
-
-      console.log(JSON.stringify({ id, success: true, status, bodyLen }));
-    } catch (err: any) {
-      console.log(
-        JSON.stringify({ id: 0, success: false, error: err.message })
-      );
-    }
-  }
-});
-
-console.log(JSON.stringify({ type: "ready" }));