@kumos/tree-sitter-parsers 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Ivan Porto Carrero
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,90 @@
+# Breeze Tree-sitter Parsers
+
+Pre-compiled Tree-sitter parsers for 163 programming languages, distributed as native binaries via npm.
+
+## Installation
+
+```bash
+npm install @kumos/tree-sitter-parsers
+```
+
+The appropriate binary for your platform will be automatically downloaded during installation.
+
+## Usage
+
+### Get the binary path
+
+```js
+const { binaryPath, metadataPath } = require('@kumos/tree-sitter-parsers');
+
+console.log(binaryPath); // Path to the static library
+console.log(metadataPath); // Path to the grammars metadata JSON
+```
+
+### Command-line usage
+
+```bash
+# Get the path to the binary
+npx tree-sitter-parsers-path
+```
+
+### Build tools integration
+
+The binary path can be used in build scripts:
+
+```bash
+# In a build script
+PARSER_LIB=$(npx tree-sitter-parsers-path)
+gcc myapp.c $PARSER_LIB -o myapp
+```
+
+## Supported Platforms
+
+- Linux x64 (glibc and musl)
+- Linux ARM64 (glibc and musl)  
+- macOS x64
+- macOS ARM64
+- Windows x64
+- Windows ARM64
+
+## Building from Source
+
+### Prerequisites
+
+- Node.js 20+
+- Git
+- Zig (for cross-compilation)
+- C/C++ compiler (for native builds)
+
+### Build Commands
+
+```bash
+# Clone the repository
+git clone https://github.com/casualjim/breeze-tree-sitter-parsers.git
+cd breeze-tree-sitter-parsers
+
+# Fetch all grammar repositories
+npm run fetch
+
+# Build for current platform
+npm run build
+
+# Build for all platforms (requires Zig)
+npm run build:all
+
+# Create npm packages for distribution
+npm run create-packages
+```
+
+## Architecture
+
+This project consists of:
+
+1. **Main package** (`@kumos/tree-sitter-parsers`) - Platform detection and binary resolution
+2. **Platform packages** - Platform-specific binaries (e.g., `@kumos/tree-sitter-parsers-darwin-arm64`)
+
+The main package uses npm's `optionalDependencies` to install only the relevant platform binary. If the platform package isn't available, it falls back to downloading the binary from GitHub releases.
+
+## License
+
+MIT

package/bin/tree-sitter-parsers-path.js

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+
+try {
+  const { binaryPath } = require('../index.js');
+  console.log(binaryPath);
+  process.exit(0);
+} catch (error) {
+  console.error(error.message);
+  process.exit(1);
+}

package/index.js

@@ -0,0 +1,128 @@
+const os = require('os');
+const path = require('path');
+const fs = require('fs');
+
+function getPlatformPackage() {
+  const platform = os.platform();
+  const arch = os.arch();
+  
+  let platformPackage;
+  
+  switch (platform) {
+    case 'darwin':
+      platformPackage = `@kumos/tree-sitter-parsers-darwin-${arch === 'arm64' ? 'arm64' : 'x64'}`;
+      break;
+    case 'linux':
+      // Check if using musl libc (Alpine Linux, etc.)
+      const isMusl = (() => {
+        try {
+          // First try ldd
+          const lddOutput = require('child_process').execSync('ldd --version 2>&1', { encoding: 'utf8' });
+          if (lddOutput.includes('musl')) return true;
+        } catch {
+          // ldd might not exist or fail
+        }
+        
+        // Check if /lib/ld-musl* exists (Alpine/musl systems)
+        try {
+          const files = fs.readdirSync('/lib');
+          return files.some(f => f.startsWith('ld-musl'));
+        } catch {
+          return false;
+        }
+      })();
+      
+      const archSuffix = arch === 'arm64' ? 'arm64' : 'x64';
+      platformPackage = `@kumos/tree-sitter-parsers-linux-${archSuffix}${isMusl ? '-musl' : ''}`;
+      break;
+    case 'win32':
+      platformPackage = `@kumos/tree-sitter-parsers-win32-${arch === 'arm64' ? 'arm64' : 'x64'}`;
+      break;
+    default:
+      throw new Error(`Unsupported platform: ${platform} ${arch}`);
+  }
+  
+  return platformPackage;
+}
+
+function getBinaryPath() {
+  const platformPackage = getPlatformPackage();
+  
+  try {
+    // Try to resolve the platform-specific package
+    const packagePath = require.resolve(platformPackage);
+    const packageDir = path.dirname(packagePath);
+    
+    // Map platform to expected binary filename
+    const platform = os.platform();
+    const arch = os.arch();
+    let binaryName;
+    
+    switch (platform) {
+      case 'darwin':
+        binaryName = `libtree-sitter-parsers-all-macos-${arch === 'arm64' ? 'aarch64' : 'x86_64'}.a`;
+        break;
+      case 'linux':
+        const isMusl = platformPackage.includes('musl');
+        const archName = arch === 'arm64' ? 'aarch64' : 'x86_64';
+        binaryName = `libtree-sitter-parsers-all-linux-${archName}-${isMusl ? 'musl' : 'glibc'}.a`;
+        break;
+      case 'win32':
+        binaryName = `libtree-sitter-parsers-all-windows-${arch === 'arm64' ? 'aarch64' : 'x86_64'}.a`;
+        break;
+    }
+    
+    const binaryPath = path.join(packageDir, binaryName);
+    
+    if (fs.existsSync(binaryPath)) {
+      return binaryPath;
+    }
+    
+    // Fallback to downloaded binary location
+    const fallbackPath = path.join(__dirname, 'binaries', binaryName);
+    if (fs.existsSync(fallbackPath)) {
+      return fallbackPath;
+    }
+    
+    throw new Error(`Binary not found at ${binaryPath} or ${fallbackPath}`);
+  } catch (error) {
+    // Platform package not installed, check fallback location
+    const fallbackDir = path.join(__dirname, 'binaries');
+    if (fs.existsSync(fallbackDir)) {
+      const files = fs.readdirSync(fallbackDir);
+      const binary = files.find(f => f.endsWith('.a'));
+      if (binary) {
+        return path.join(fallbackDir, binary);
+      }
+    }
+    
+    throw new Error(`Could not find tree-sitter parsers binary. Platform package ${platformPackage} not installed and no fallback binary found.`);
+  }
+}
+
+function getMetadataPath() {
+  const binaryPath = getBinaryPath();
+  const dir = path.dirname(binaryPath);
+  const binaryName = path.basename(binaryPath, '.a');
+  const metadataName = binaryName.replace('libtree-sitter-parsers-all-', 'grammars-') + '.json';
+  return path.join(dir, metadataName);
+}
+
+const binaryPath = getBinaryPath();
+const metadataPath = getMetadataPath();
+
+module.exports = {
+  binaryPath,
+  metadataPath,
+  platformPackage: getPlatformPackage(),
+  
+  // Utility function to get all available grammars
+  getGrammars() {
+    try {
+      return JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
+    } catch (error) {
+      console.error('Failed to load grammars metadata:', error);
+      return [];
+    }
+  }
+};

package/package.json

@@ -0,0 +1,54 @@
+{
+  "author": "",
+  "bin": {
+    "tree-sitter-parsers-path": "./bin/tree-sitter-parsers-path.js"
+  },
+  "description": "Pre-compiled Tree-sitter parsers for 163 programming languages",
+  "devDependencies": {
+    "tree-sitter-cli": "^0.25.6"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "files": [
+    "index.js",
+    "postinstall.js",
+    "bin/"
+  ],
+  "keywords": [
+    "tree-sitter",
+    "parser",
+    "syntax",
+    "ast"
+  ],
+  "license": "MIT",
+  "main": "index.js",
+  "name": "@kumos/tree-sitter-parsers",
+  "optionalDependencies": {
+    "@kumos/tree-sitter-parsers-darwin-arm64": "0.1.1",
+    "@kumos/tree-sitter-parsers-darwin-x64": "0.1.1",
+    "@kumos/tree-sitter-parsers-linux-arm64": "0.1.1",
+    "@kumos/tree-sitter-parsers-linux-arm64-musl": "0.1.1",
+    "@kumos/tree-sitter-parsers-linux-x64": "0.1.1",
+    "@kumos/tree-sitter-parsers-linux-x64-musl": "0.1.1",
+    "@kumos/tree-sitter-parsers-win32-arm64": "0.1.1",
+    "@kumos/tree-sitter-parsers-win32-x64": "0.1.1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/casualjim/breeze-tree-sitter-parsers.git"
+  },
+  "scripts": {
+    "build": "node build-grammars.js",
+    "build:all": "node build-grammars.js --all-platforms",
+    "compile": "node build-grammars.js --compile-only",
+    "create-packages": "node create-platform-packages.js",
+    "fetch": "node build-grammars.js --fetch-only",
+    "postinstall": "node postinstall.js",
+    "publish-all": "bash publish-packages.sh",
+    "test-distribution": "node test-distribution.js",
+    "test-distribution:arm64": "node test-distribution.js --arch arm64",
+    "test-distribution:x64": "node test-distribution.js --arch x64"
+  },
+  "version": "0.1.1"
+}

package/postinstall.js

@@ -0,0 +1,166 @@
+#!/usr/bin/env node
+
+const os = require('os');
+const path = require('path');
+const fs = require('fs');
+const https = require('https');
+const { promisify } = require('util');
+const pipeline = promisify(require('stream').pipeline);
+
+// Configuration
+const REPO_OWNER = 'casualjim';
+const REPO_NAME = 'breeze-tree-sitter-parsers';
+const VERSION = require('./package.json').version;
+
+async function downloadFile(url, destination) {
+  const dir = path.dirname(destination);
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+
+  return new Promise((resolve, reject) => {
+    https.get(url, { headers: { 'User-Agent': 'breeze-tree-sitter-parsers' } }, (response) => {
+      if (response.statusCode === 302 || response.statusCode === 301) {
+        // Follow redirect
+        downloadFile(response.headers.location, destination).then(resolve).catch(reject);
+        return;
+      }
+
+      if (response.statusCode !== 200) {
+        reject(new Error(`Failed to download: ${response.statusCode}`));
+        return;
+      }
+
+      const file = fs.createWriteStream(destination);
+      pipeline(response, file)
+        .then(() => resolve())
+        .catch(reject);
+    }).on('error', reject);
+  });
+}
+
+function getPlatformBinaryName() {
+  const platform = os.platform();
+  const arch = os.arch();
+  
+  switch (platform) {
+    case 'darwin':
+      return `libtree-sitter-parsers-all-macos-${arch === 'arm64' ? 'aarch64' : 'x86_64'}.a`;
+    case 'linux':
+      // Try to detect musl
+      const isMusl = (() => {
+        try {
+          // First try ldd
+          const lddOutput = require('child_process').execSync('ldd --version 2>&1', { encoding: 'utf8' });
+          if (lddOutput.includes('musl')) return true;
+        } catch {
+          // ldd might not exist or fail
+        }
+        
+        // Check if /lib/ld-musl* exists (Alpine/musl systems)
+        try {
+          const fs = require('fs');
+          const files = fs.readdirSync('/lib');
+          return files.some(f => f.startsWith('ld-musl'));
+        } catch {
+          return false;
+        }
+      })();
+      
+      const archName = arch === 'arm64' ? 'aarch64' : 'x86_64';
+      return `libtree-sitter-parsers-all-linux-${archName}-${isMusl ? 'musl' : 'glibc'}.a`;
+    case 'win32':
+      return `libtree-sitter-parsers-all-windows-${arch === 'arm64' ? 'aarch64' : 'x86_64'}.a`;
+    default:
+      throw new Error(`Unsupported platform: ${platform} ${arch}`);
+  }
+}
+
+function getMetadataFileName() {
+  const binaryName = getPlatformBinaryName();
+  return binaryName.replace('libtree-sitter-parsers-all-', 'grammars-').replace('.a', '.json');
+}
+
+async function checkOptionalDependency() {
+  try {
+    // Simply try to load the main module - if it works, we have what we need
+    const { binaryPath, metadataPath } = require('./index.js');
+    
+    // Verify the files actually exist
+    if (fs.existsSync(binaryPath) && fs.existsSync(metadataPath)) {
+      console.log('Tree-sitter parsers binary found.');
+      return true;
+    }
+    console.log(`Binary not found at: ${binaryPath}`);
+    console.log(`Metadata not found at: ${metadataPath}`);
+  } catch (error) {
+    // Binary not found via normal package installation
+    console.log('Error loading index.js:', error.message);
+  }
+  
+  console.log('Binary not found in platform package, will use fallback download.');
+  return false;
+}
+
+async function downloadBinaries() {
+  const binaryName = getPlatformBinaryName();
+  const metadataName = getMetadataFileName();
+  const binariesDir = path.join(__dirname, 'binaries');
+  
+  const binaryPath = path.join(binariesDir, binaryName);
+  const metadataPath = path.join(binariesDir, metadataName);
+  
+  // Check if already downloaded
+  if (fs.existsSync(binaryPath) && fs.existsSync(metadataPath)) {
+    console.log('Binaries already downloaded.');
+    return;
+  }
+  
+  console.log(`Downloading tree-sitter parsers for your platform...`);
+  
+  // Construct download URLs
+  const releaseUrl = `https://github.com/${REPO_OWNER}/${REPO_NAME}/releases/download/v${VERSION}`;
+  const binaryUrl = `${releaseUrl}/${binaryName}`;
+  const metadataUrl = `${releaseUrl}/${metadataName}`;
+  
+  try {
+    // Download both files
+    console.log(`Downloading ${binaryName}...`);
+    await downloadFile(binaryUrl, binaryPath);
+    
+    console.log(`Downloading ${metadataName}...`);
+    await downloadFile(metadataUrl, metadataPath);
+    
+    console.log('Download complete!');
+  } catch (error) {
+    console.error('Failed to download binaries:', error.message);
+    console.error('You may need to manually download the binaries from:');
+    console.error(`  ${binaryUrl}`);
+    console.error(`  ${metadataUrl}`);
+    process.exit(1);
+  }
+}
+
+async function main() {
+  // Skip in CI environments or when explicitly disabled
+  if (process.env.CI || process.env.BREEZE_SKIP_DOWNLOAD) {
+    console.log('Skipping postinstall download.');
+    return;
+  }
+  
+  // Check if optional dependency was installed
+  const hasOptionalDep = await checkOptionalDependency();
+  
+  if (!hasOptionalDep) {
+    console.log('Downloading tree-sitter parsers for your platform...');
+    await downloadBinaries();
+  }
+}
+
+// Only run if called directly
+if (require.main === module) {
+  main().catch(error => {
+    console.error('Postinstall failed:', error);
+    process.exit(1);
+  });
+}