@oh-my-pi/pi-natives 18.8.5 → 18.8.7

package/native/embedded-addon.js

@@ -1,6 +1,5 @@
-
 // Always null on disk. Standalone binary builds replace this module in memory
-// with the target's manifest and addon archive (scripts/embed-native.ts).
+// with the target's manifest and one zstd frame per addon (scripts/embed-native.ts).
 
 /** @typedef {"modern" | "baseline" | "default"} EmbeddedAddonVariant */
 
@@ -8,15 +7,8 @@
  * @typedef {Object} EmbeddedAddonFile
  * @property {EmbeddedAddonVariant} variant
  * @property {string} filename
- * @property {number} size
- * @property {string=} filePath
- */
-
-/**
- * @typedef {Object} EmbeddedAddonArchive
- * @property {"tar.gz"} format
- * @property {string} filename
- * @property {string} filePath
+ * @property {number} size Decompressed `.node` size in bytes.
+ * @property {string} zstdPath Embedded zstd frame holding the addon bytes.
  */
 
 /**
@@ -24,7 +16,6 @@
  * @property {string} platformTag
  * @property {string} version
  * @property {EmbeddedAddonFile[]} files
- * @property {EmbeddedAddonArchive=} archive
  */
 
 /** @type {EmbeddedAddon|null} */

package/native/index.d.ts

@@ -730,6 +730,13 @@ export declare class VcsRepo {
  */
 export declare function __ompInstallTokioRuntime(): void
 
+/**
+ * Points the addon at the directory holding downloaded wasm grammars
+ * (`<dir>/<file>`). The loader (`native/loader-state.js`) calls this once
+ * with `<natives dir>/grammars`; grammars then load lazily from it.
+ */
+export declare function __ompSetGrammarDir(dir: string): void
+
 /**
  * Release version stamped into this `.node` after linking.
  *
@@ -863,6 +870,11 @@ export interface AstFindResult {
   limitReached: boolean
   /** Non-fatal parse or pattern errors collected during the run. */
   parseErrors?: Array<string>
+  /**
+   * Languages whose on-demand grammar is not installed; their files were
+   * skipped (see `wasmGrammarFor`).
+   */
+  missingGrammars?: Array<string>
 }
 
 /**
@@ -1035,6 +1047,11 @@ export interface AstReplaceResult {
   limitReached: boolean
   /** Parse or pattern errors when not failing the whole operation. */
   parseErrors?: Array<string>
+  /**
+   * Languages whose on-demand grammar is not installed; their files were
+   * skipped (see `wasmGrammarFor`).
+   */
+  missingGrammars?: Array<string>
 }
 
 export interface AxNode {
@@ -3679,6 +3696,39 @@ export declare function warmBlockParse(options: BlockParseOptions): Promise<unde
  */
 export declare function warmHighlighter(): Promise<undefined>
 
+/**
+ * Wasm grammar backing `lang` (alias) or the language of `path`; `null` for
+ * built-in or unknown languages.
+ */
+export declare function wasmGrammarFor(query: WasmGrammarQuery): WasmGrammarInfo | null
+
+/** A grammar the host downloads on first use. */
+export interface WasmGrammarInfo {
+  /** Canonical language name, e.g. `verilog`, `csharp`. */
+  language: string
+  /** stencil-hq/wasm-grammars release tag hosting the grammar, e.g. `v1`. */
+  release: string
+  /**
+   * File name inside the grammar directory; the release asset is
+   * `<file>.zst`.
+   */
+  file: string
+  /** Lowercase hex SHA-256 of the decompressed `.wasm`. */
+  sha256: string
+  /** Byte size of the decompressed `.wasm`. */
+  size: number
+  /** Whether the grammar is downloaded (or already loaded). */
+  installed: boolean
+}
+
+/** Language to look up: an alias, or a path to infer it from. */
+export interface WasmGrammarQuery {
+  /** Language alias (e.g. `kotlin`, `sv`); wins over `path`. */
+  lang?: string
+  /** File whose extension selects the language. */
+  path?: string
+}
+
 /** Profiling results returned to JavaScript. */
 export interface WorkProfile {
   /** Folded stack format for flamegraph tools. */

package/native/index.js

@@ -46,6 +46,7 @@ export const VcsRepo = nativeBindings.VcsRepo;
 
 // functions
 export const __ompInstallTokioRuntime = nativeBindings.__ompInstallTokioRuntime ?? missingNativeExport("__ompInstallTokioRuntime");
+export const __ompSetGrammarDir = nativeBindings.__ompSetGrammarDir ?? missingNativeExport("__ompSetGrammarDir");
 export const __piNativesBuildVersion = nativeBindings.__piNativesBuildVersion;
 export const appleFmAvailability = nativeBindings.appleFmAvailability ?? missingNativeExport("appleFmAvailability");
 export const appleFmCancel = nativeBindings.appleFmCancel ?? missingNativeExport("appleFmCancel");
@@ -142,6 +143,7 @@ export const vectorIndexTopK = nativeBindings.vectorIndexTopK ?? missingNativeEx
 export const visibleWidth = nativeBindings.visibleWidth ?? missingNativeExport("visibleWidth");
 export const warmBlockParse = nativeBindings.warmBlockParse ?? missingNativeExport("warmBlockParse");
 export const warmHighlighter = nativeBindings.warmHighlighter ?? missingNativeExport("warmHighlighter");
+export const wasmGrammarFor = nativeBindings.wasmGrammarFor ?? missingNativeExport("wasmGrammarFor");
 export const wrapTextWithAnsi = nativeBindings.wrapTextWithAnsi ?? missingNativeExport("wrapTextWithAnsi");
 
 // string/numeric enums (napi-rs string_enum produces TS-only const enum)

package/native/loader-state.d.ts

@@ -1,21 +1,17 @@
 export interface EmbeddedAddonFile {
 	variant: "modern" | "baseline" | "default";
+	/** Basename of the extracted `.node` file. */
 	filename: string;
-	size?: number;
-	filePath?: string;
-}
-
-export interface EmbeddedAddonArchive {
-	format: "tar.gz";
-	filename: string;
-	filePath: string;
+	/** Decompressed addon size in bytes. */
+	size: number;
+	/** Embedded zstd frame holding the addon bytes. */
+	zstdPath: string;
 }
 
 export interface EmbeddedAddon {
 	platformTag: string;
 	version: string;
 	files: EmbeddedAddonFile[];
-	archive?: EmbeddedAddonArchive;
 }
 
 export interface DetectCompiledBinaryInput {
@@ -90,13 +86,13 @@ export function cleanupStaleNativeVersions(input: CleanupStaleNativeVersionsInpu
 
 export function prepareNativeVersionDir(versionedDir: string): void;
 
-export interface ExtractEmbeddedAddonArchiveInput {
-	archivePath: string;
+export interface ExtractEmbeddedAddonsInput {
 	files: EmbeddedAddonFile[];
 	targetDir: string;
 }
 
-export function extractEmbeddedAddonArchive(input: ExtractEmbeddedAddonArchiveInput): string[];
+/** Decompress every manifest addon missing or wrong-sized in `targetDir`; returns the paths written. */
+export function extractEmbeddedAddons(input: ExtractEmbeddedAddonsInput): string[];
 
 export interface SelectCpuVariantInput {
 	arch: string;

package/native/loader-state.js

@@ -3,7 +3,6 @@ import * as fs from "node:fs";
 import { createRequire } from "node:module";
 import * as os from "node:os";
 import * as path from "node:path";
-import * as zlib from "node:zlib";
 import packageJson from "../package.json" with { type: "json" };
 import { embeddedAddon } from "./embedded-addon.js";
 import { bindingsHaveReleaseIdentity, bindingsReleaseVersion, containsVersionStamp } from "./version-sentinel.js";
@@ -450,36 +449,6 @@ function selectEmbeddedAddonFile(selectedVariant) {
 	return embeddedAddon.files.find(file => file.variant === "baseline") || null;
 }
 
-function readTarString(buffer, offset, length) {
-	const end = Math.min(offset + length, buffer.length);
-	let stringEnd = offset;
-	while (stringEnd < end && buffer[stringEnd] !== 0) stringEnd++;
-	return buffer.toString("utf8", offset, stringEnd);
-}
-
-function readTarOctal(buffer, offset, length) {
-	const value = readTarString(buffer, offset, length).trim();
-	if (!value) return 0;
-	const parsed = Number.parseInt(value, 8);
-	if (!Number.isFinite(parsed)) {
-		throw new Error(`Invalid tar octal value: ${value}`);
-	}
-	return parsed;
-}
-
-function isZeroTarBlock(buffer, offset) {
-	for (let index = 0; index < 512; index++) {
-		if (buffer[offset + index] !== 0) return false;
-	}
-	return true;
-}
-
-function getTarEntryName(header) {
-	const name = readTarString(header, 0, 100);
-	const prefix = readTarString(header, 345, 155);
-	return prefix ? `${prefix}/${name}` : name;
-}
-
 function isSafeEmbeddedAddonFilename(filename) {
 	return filename.length > 0 && path.basename(filename) === filename && !filename.includes("/") && !filename.includes("\\");
 }
@@ -488,7 +457,7 @@ function isEmbeddedAddonFileCurrent(targetPath, file) {
 	try {
 		const stat = fs.statSync(targetPath);
 		if (!stat.isFile()) return false;
-		return typeof file.size !== "number" || stat.size === file.size;
+		return stat.size === file.size;
 	} catch (err) {
 		if (err && err.code === "ENOENT") return false;
 		throw err;
@@ -510,60 +479,23 @@ function writeEmbeddedAddonFile(targetPath, content) {
 	}
 }
 
-export function extractEmbeddedAddonArchive({ archivePath, files, targetDir }) {
-	const pending = new Map();
+export function extractEmbeddedAddons({ files, targetDir }) {
 	for (const file of files) {
 		if (!isSafeEmbeddedAddonFilename(file.filename)) {
 			throw new Error(`Unsafe embedded addon filename: ${file.filename}`);
 		}
-		const targetPath = path.join(targetDir, file.filename);
-		if (!isEmbeddedAddonFileCurrent(targetPath, file)) {
-			pending.set(file.filename, file);
-		}
 	}
-	if (pending.size === 0) return [];
-
-	const archive = zlib.gunzipSync(fs.readFileSync(archivePath));
 	const writtenPaths = [];
-	let offset = 0;
-
-	while (offset + 512 <= archive.length) {
-		if (isZeroTarBlock(archive, offset)) break;
-		const header = archive.subarray(offset, offset + 512);
-		const filename = getTarEntryName(header);
-		const size = readTarOctal(header, 124, 12);
-		const typeflag = header[156] === 0 ? "0" : String.fromCharCode(header[156]);
-		offset += 512;
-
-		if (offset + size > archive.length) {
-			throw new Error(`Truncated embedded addon archive entry: ${filename}`);
-		}
-
-		if (!isSafeEmbeddedAddonFilename(filename)) {
-			throw new Error(`Unsafe embedded addon archive entry: ${filename}`);
-		}
-		if (typeflag !== "0") {
-			throw new Error(`Unsupported embedded addon archive entry type ${typeflag}: ${filename}`);
-		}
-
-		const file = pending.get(filename);
-		if (file) {
-			if (typeof file.size === "number" && file.size !== size) {
-				throw new Error(`Embedded addon size mismatch for ${filename}: expected ${file.size}, got ${size}`);
-			}
-			const targetPath = path.join(targetDir, filename);
-			writeEmbeddedAddonFile(targetPath, archive.subarray(offset, offset + size));
-			pending.delete(filename);
-			writtenPaths.push(targetPath);
+	for (const file of files) {
+		const targetPath = path.join(targetDir, file.filename);
+		if (isEmbeddedAddonFileCurrent(targetPath, file)) continue;
+		const content = Bun.zstdDecompressSync(fs.readFileSync(file.zstdPath));
+		if (content.length !== file.size) {
+			throw new Error(`Embedded addon size mismatch for ${file.filename}: expected ${file.size}, got ${content.length}`);
 		}
-
-		offset += Math.ceil(size / 512) * 512;
-	}
-
-	if (pending.size > 0) {
-		throw new Error(`Embedded addon archive missing: ${[...pending.keys()].join(", ")}`);
+		writeEmbeddedAddonFile(targetPath, content);
+		writtenPaths.push(targetPath);
 	}
-
 	return writtenPaths;
 }
 
@@ -584,40 +516,12 @@ function maybeExtractEmbeddedAddon(ctx, errors) {
 		return null;
 	}
 
-	if (embeddedAddon.archive) {
-		try {
-			extractEmbeddedAddonArchive({
-				archivePath: embeddedAddon.archive.filePath,
-				files: embeddedAddon.files,
-				targetDir: ctx.versionedDir,
-			});
-			if (isEmbeddedAddonFileCurrent(targetPath, selectedEmbeddedFile)) {
-				return targetPath;
-			}
-			errors.push(`embedded addon archive (${embeddedAddon.archive.filename}): missing ${selectedEmbeddedFile.filename}`);
-			return null;
-		} catch (err) {
-			const message = err instanceof Error ? err.message : String(err);
-			errors.push(`embedded addon archive (${embeddedAddon.archive.filename}): ${message}`);
-			return null;
-		}
-	}
-
-	if (isEmbeddedAddonFileCurrent(targetPath, selectedEmbeddedFile)) {
-		return targetPath;
-	}
-	if (!selectedEmbeddedFile.filePath) {
-		errors.push(`embedded addon metadata missing file path for ${selectedEmbeddedFile.filename}`);
-		return null;
-	}
-
 	try {
-		const buffer = fs.readFileSync(selectedEmbeddedFile.filePath);
-		fs.writeFileSync(targetPath, buffer);
+		extractEmbeddedAddons({ files: embeddedAddon.files, targetDir: ctx.versionedDir });
 		return targetPath;
 	} catch (err) {
 		const message = err instanceof Error ? err.message : String(err);
-		errors.push(`embedded addon write (${selectedEmbeddedFile.filename}): ${message}`);
+		errors.push(`embedded addon extraction (${selectedEmbeddedFile.filename}): ${message}`);
 		return null;
 	}
 }
@@ -832,6 +736,18 @@ function installNativeTokioRuntime(bindings) {
 	}
 }
 
+/**
+ * Point the addon at `<nativesDir>/grammars`, where the agent downloads the
+ * tree-sitter grammars that load as WebAssembly on demand (pi-utils
+ * `getNativeGrammarsDir()` resolves the same path). Version cleanup never
+ * touches it: only release-version directories are removed. Older addons
+ * predating the export link every grammar and need no directory.
+ */
+function configureGrammarDir(bindings, nativesDir) {
+	const setGrammarDir = bindings.__ompSetGrammarDir;
+	if (typeof setGrammarDir === "function") setGrammarDir(path.join(nativesDir, "grammars"));
+}
+
 
 function buildHelpMessage(ctx) {
 	if (ctx.isCompiledBinary) {
@@ -957,6 +873,7 @@ export function loadNative() {
 			const bindings = require_(candidate);
 			validateLoadedBindings(ctx, bindings, candidate);
 			installNativeTokioRuntime(bindings);
+			configureGrammarDir(bindings, ctx.nativesDir);
 			loadedAddon = describeLoadedAddon(bindings, candidate, ctx);
 	        cleanupStaleNativeVersions({ nativesDir: ctx.nativesDir, currentVersion: ctx.packageVersion });
 			startupMarker("native:loadNative:done");

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@oh-my-pi/pi-natives",
-	"version": "18.8.5",
+	"version": "18.8.7",
 	"description": "Native Rust bindings for PDF conversion, audio, WebRTC, grep, clipboard, image processing, syntax highlighting, PTY, and shell operations via N-API",
 	"type": "module",
 	"homepage": "https://omp.sh",
@@ -112,11 +112,11 @@
 		}
 	},
 	"optionalDependencies": {
-		"@oh-my-pi/pi-natives-linux-x64": "18.8.5",
-		"@oh-my-pi/pi-natives-linux-arm64": "18.8.5",
-		"@oh-my-pi/pi-natives-darwin-x64": "18.8.5",
-		"@oh-my-pi/pi-natives-darwin-arm64": "18.8.5",
-		"@oh-my-pi/pi-natives-win32-x64": "18.8.5",
-		"@oh-my-pi/pi-natives-win32-arm64": "18.8.5"
+		"@oh-my-pi/pi-natives-linux-x64": "18.8.7",
+		"@oh-my-pi/pi-natives-linux-arm64": "18.8.7",
+		"@oh-my-pi/pi-natives-darwin-x64": "18.8.7",
+		"@oh-my-pi/pi-natives-darwin-arm64": "18.8.7",
+		"@oh-my-pi/pi-natives-win32-x64": "18.8.7",
+		"@oh-my-pi/pi-natives-win32-arm64": "18.8.7"
 	}
 }