git-sqlite-vfs 0.0.1 → 0.0.6

package/downloader.js

@@ -0,0 +1,63 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { execSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import os from 'node:os';
+import process from 'node:process';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+export async function downloadOrBuild(targetDir) {
+    let pkgVersion = '0.0.2';
+    try {
+        const pkg = JSON.parse(fs.readFileSync(path.join(__dirname, 'package.json'), 'utf-8'));
+        pkgVersion = pkg.version;
+    } catch(e) {}
+
+    const platform = os.platform();
+    const arch = os.arch();
+    
+    const repo = 'fur-tea-laser/git-sqlite-vfs';
+    const assetName = `git-sqlite-vfs-${platform}-${arch}.tar.gz`;
+    const url = `https://github.com/${repo}/releases/download/v${pkgVersion}/${assetName}`;
+
+    try {
+        console.log(`Attempting to download prebuilt binary: ${url}`);
+        
+        if (!fs.existsSync(targetDir)) {
+            fs.mkdirSync(targetDir, { recursive: true });
+        }
+        
+        const res = await fetch(url);
+        if (!res.ok) {
+            throw new Error(`Failed to fetch: ${res.status} ${res.statusText}`);
+        }
+        
+        const arrayBuffer = await res.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+        
+        const tarballPath = path.join(targetDir, 'temp.tar.gz');
+        fs.writeFileSync(tarballPath, buffer);
+        
+        execSync(`tar -xzf temp.tar.gz`, { cwd: targetDir });
+        fs.unlinkSync(tarballPath);
+        
+        console.log('Successfully downloaded and extracted prebuilt binary.');
+    } catch (err) {
+        console.warn(`Download failed: ${err.message}`);
+        console.log('Falling back to building from source...');
+        try {
+            execSync('npm run build', { stdio: 'inherit', cwd: __dirname });
+            
+            const defaultOutDir = path.join(__dirname, 'c', 'output');
+            if (path.resolve(defaultOutDir) !== path.resolve(targetDir)) {
+                fs.cpSync(defaultOutDir, targetDir, { recursive: true });
+            }
+            
+            console.log('Successfully built from source.');
+        } catch (buildErr) {
+            console.error('Failed to build from source.', buildErr.message);
+        }
+    }
+}

package/index.d.ts

@@ -0,0 +1,14 @@
+export const GITVFS_EXTENSION_PATH: string;
+
+export interface BootstrapOptions {
+    dir?: string;
+}
+
+export function bootstrapGitVFS(options?: BootstrapOptions): Promise<void>;
+
+export interface ConfigureGitOptions {
+    repoDir: string;
+    vfsDir: string;
+}
+
+export function configureGitIntegration(options: ConfigureGitOptions): Promise<void>;

package/index.js

@@ -1,55 +1,107 @@
-const Database = require('better-sqlite3');
-const { execSync } = require('child_process');
-const path = require('path');
-const fs = require('fs');
-
-class GitSQLite {
-    /**
-     * Opens a SQLite database utilizing the custom Git Virtual File System (VFS).
-     * By sharding the SQLite B-Tree into 4KB binary pages, it neutralizes cascading
-     * byte shifts, allowing native Git xdelta to achieve near-perfect compression.
-     * 
-     * @param {string} dbPath - The path to the sharded database directory (e.g., '.db')
-     * @returns {Database} - A native better-sqlite3 database connection
-     */
-    static open(dbPath) {
-        // 1. Open a temporary in-memory database to act as an extension loader
-        // better-sqlite3 requires an active connection to load an extension.
-        const tempDb = new Database(':memory:');
-        
-        // 2. Enable extensions and load our compiled C VFS extension (.so)
-        tempDb.loadExtension(path.resolve(__dirname, 'c/output/gitvfs'));
-        
-        // 3. Close tempDb. The SQLite runtime inside the Node process 
-        // will permanently retain the global 'gitvfs' registration!
-        tempDb.close();
-
-        // 4. Instantiate and return the actual database connection.
-        // Because our compiled extension registers itself as the default VFS, 
-        // better-sqlite3 will automatically route all physical I/O for this DB
-        // through our Git-sharded C engine!
-        return new Database(dbPath);
-    }
-
-    /**
-     * Configures the local Git repository with optimized binary thresholds 
-     * and strictly wires up our custom C engine as a Git Merge Strategy.
-     */
-    static setupGit() {
-        try {
-            // Optimize Git for 4KB binary pages to guarantee xdelta works nicely
-            // without prematurely terminating delta compression loops
-            execSync('git config core.bigFileThreshold 10m', { stdio: 'ignore' });
-            
-            // Wire up the custom merge strategy driver with absolute paths
-            // This natively binds our C executable to Git's conflict resolution pipeline
-            const driverPath = path.resolve(__dirname, 'c/output/git-merge-sqlitevfs');
-            execSync(`git config merge.sqlite_logical.name "SQLite Logical Merge Driver"`, { stdio: 'ignore' });
-            execSync(`git config merge.sqlite_logical.driver "${driverPath} %O %A %B %P"`, { stdio: 'ignore' });
-        } catch (err) {
-            console.warn("Warning: Could not configure git attributes automatically.", err.message);
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import process from 'node:process';
+import { downloadOrBuild } from './downloader.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Determine the correct extension based on OS
+const platform = os.platform();
+let ext = 'so';
+if (platform === 'darwin') {
+    ext = 'dylib';
+} else if (platform === 'win32') {
+    ext = 'dll';
+}
+
+const extensionPath = path.resolve(__dirname, 'c', 'output', `gitvfs.${ext}`);
+
+export const GITVFS_EXTENSION_PATH = extensionPath;
+
+export async function bootstrapGitVFS(options = {}) {
+    if (options.dir) {
+        if (typeof Deno !== 'undefined') {
+            Deno.env.set('GIT_SQLITE_VFS_DIR', options.dir);
+        } else {
+            process.env.GIT_SQLITE_VFS_DIR = options.dir;
         }
     }
+
+    let currentExtPath = extensionPath;
+    if (!fs.existsSync(currentExtPath)) {
+        const writableDir = path.join(process.cwd(), '.git-sqlite-vfs-bin');
+        await downloadOrBuild(writableDir);
+        currentExtPath = path.join(writableDir, `gitvfs.${ext}`);
+    }
+
+    // Dynamically import libsql so that we load the extension into its isolated native memory space.
+    let Database;
+    if (typeof Deno !== 'undefined') {
+        // Deno environment
+        const lib = await import('npm:libsql');
+        Database = lib.default || lib.Database || lib;
+    } else {
+        // Node.js environment
+        const lib = await import('libsql');
+        Database = lib.default || lib.Database || lib;
+    }
+
+    const db = new Database(':memory:');
+    db.loadExtension(currentExtPath);
+    db.close();
 }
 
-module.exports = GitSQLite;
+export async function configureGitIntegration({ repoDir, vfsDir }) {
+    let driverDir = path.resolve(__dirname, 'c', 'output');
+    let driverPath = path.join(driverDir, 'git-merge-sqlitevfs');
+    if (platform === 'win32' && !fs.existsSync(driverPath) && fs.existsSync(driverPath + '.exe')) {
+        driverPath += '.exe';
+    }
+
+    if (!fs.existsSync(driverPath)) {
+        driverDir = path.join(process.cwd(), '.git-sqlite-vfs-bin');
+        await downloadOrBuild(driverDir);
+        driverPath = path.join(driverDir, 'git-merge-sqlitevfs');
+        if (platform === 'win32' && !fs.existsSync(driverPath) && fs.existsSync(driverPath + '.exe')) {
+            driverPath += '.exe';
+        }
+    }
+    
+    // Set the merge driver
+    execSync(`git config merge.sqlitevfs.name "SQLite VFS Merge Driver"`, { cwd: repoDir, stdio: 'ignore' });
+    execSync(`git config merge.sqlitevfs.driver "${driverPath} %O %A %B %P"`, { cwd: repoDir, stdio: 'ignore' });
+
+    // Append to .gitattributes
+    const gitattributesPath = path.join(repoDir, '.gitattributes');
+    const attributeLine = `${vfsDir}/* merge=sqlitevfs\n`;
+    
+    let content = '';
+    if (fs.existsSync(gitattributesPath)) {
+        content = fs.readFileSync(gitattributesPath, 'utf-8');
+    }
+    if (!content.includes(attributeLine.trim())) {
+        fs.appendFileSync(gitattributesPath, attributeLine);
+    }
+
+    // Create or update .gitignore in the repo root to ignore SQLite transient files
+    const gitignorePath = path.join(repoDir, '.gitignore');
+    const ignoreLines = [
+        `${vfsDir}/*-journal`,
+        `${vfsDir}/*-wal`,
+        `${vfsDir}/*-shm`
+    ];
+
+    let gitignoreContent = '';
+    if (fs.existsSync(gitignorePath)) {
+        gitignoreContent = fs.readFileSync(gitignorePath, 'utf-8');
+    }
+    
+    const linesToAdd = ignoreLines.filter(line => !gitignoreContent.includes(line));
+    if (linesToAdd.length > 0) {
+        fs.appendFileSync(gitignorePath, (gitignoreContent.endsWith('\n') || gitignoreContent === '' ? '' : '\n') + linesToAdd.join('\n') + '\n');
+    }
+}

package/install.js

@@ -1,53 +1,17 @@
-const os = require('os');
-const fs = require('fs');
-const path = require('path');
-const { execSync } = require('child_process');
-const pkg = require('./package.json');
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import process from 'node:process';
+import { downloadOrBuild } from './downloader.js';
 
-const REPO = 'fur-tea-laser/git-sqlite-vfs'; 
-const VERSION = `v${pkg.version}`;
-const PLATFORM = os.platform();
-const ARCH = os.arch();
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 
-// Target filename from GitHub Release: e.g., git-sqlite-vfs-v1.0.0-linux-x64.tar.gz
-const ASSET_NAME = `git-sqlite-vfs-${VERSION}-${PLATFORM}-${ARCH}.tar.gz`;
-const DOWNLOAD_URL = `https://github.com/${REPO}/releases/download/${VERSION}/${ASSET_NAME}`;
-
-const OUT_DIR = path.join(__dirname, 'c', 'output');
-
-function buildFromSource() {
-    console.log('Building from source as fallback...');
-    try {
-        execSync('npm run build', { stdio: 'inherit', cwd: __dirname });
-        console.log('Successfully built from source.');
-    } catch (e) {
-        console.error('Failed to build from source.', e.message);
-        process.exit(1);
-    }
-}
-
-function downloadAndExtract() {
-    // If the SKIP_DOWNLOAD env var is set, or if we are building locally from the repo root
-    // we should just build from source.
-    if (process.env.SKIP_DOWNLOAD || !fs.existsSync(path.join(__dirname, 'node_modules'))) {
-        return buildFromSource();
-    }
-
-    console.log(`Attempting to download prebuilt binary: ${DOWNLOAD_URL}`);
-    
-    try {
-        if (!fs.existsSync(OUT_DIR)) {
-            fs.mkdirSync(OUT_DIR, { recursive: true });
-        }
-        
-        // Use native curl and tar to download and extract without requiring NPM dependencies.
-        // This is supported out-of-the-box on modern Linux, macOS, and Windows 10+
-        execSync(`curl -sLf ${DOWNLOAD_URL} | tar -xz -C "${OUT_DIR}"`, { stdio: 'inherit' });
-        console.log('Prebuilt binary successfully downloaded and extracted!');
-    } catch (err) {
-        console.log('Prebuilt binary not found or download failed. Falling back to source compilation...');
-        buildFromSource();
-    }
+async function run() {
+    const targetDir = path.join(__dirname, 'c', 'output');
+    await downloadOrBuild(targetDir);
 }
 
-downloadAndExtract();
+run().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

package/package.json

@@ -1,15 +1,34 @@
 {
   "name": "git-sqlite-vfs",
-  "version": "0.0.1",
+  "version": "0.0.6",
   "description": "A Git-Versioned SQLite Database via a Custom Virtual File System (VFS)",
   "main": "index.js",
+  "types": "index.d.ts",
+  "type": "module",
+  "bin": {
+    "git-sqlite-setup": "bin/cli.js"
+  },
+  "files": [
+    "bin/",
+    "c/",
+    "index.js",
+    "index.d.ts",
+    "downloader.js",
+    "install.js"
+  ],
   "scripts": {
     "build": "cd c && make",
     "postinstall": "node install.js",
-    "pretest": "npm run build && rm -rf .db .git && git init --initial-branch=master",
-    "test": "node --test test.js"
+    "pretest": "npm run build && rm -rf .db test.db .test-db .compaction-db .git && git init --initial-branch=master",
+    "test": "node --test test.node.js test.compaction.node.js test.e2e.node.js",
+    "test:deno": "npm run build && rm -rf .db test.db .test-db .git && git init --initial-branch=master && deno test -A test.deno.ts"
   },
   "dependencies": {
+    "@libsql/client": "^0.14.0",
+    "drizzle-orm": "^0.33.0",
+    "libsql": "^0.4.5"
+  },
+  "devDependencies": {
     "better-sqlite3": "^9.4.3"
   },
   "author": "",

package/test.js

@@ -1,209 +0,0 @@
-const { describe, it, before } = require('node:test');
-const assert = require('node:assert');
-const { execSync } = require('child_process');
-const fs = require('fs');
-const path = require('path');
-const GitSQLite = require('./index.js');
-
-// Helper to run git commands synchronously
-const runGit = (cmd) => execSync(cmd, { stdio: 'pipe' }).toString().trim();
-
-// Helper to recursively calculate total directory size
-const getDirSize = (dirPath) => {
-    let total = 0;
-    if (!fs.existsSync(dirPath)) return 0;
-    
-    const files = fs.readdirSync(dirPath, { withFileTypes: true });
-    for (const file of files) {
-        const fullPath = path.join(dirPath, file.name);
-        if (file.isDirectory()) {
-            total += getDirSize(fullPath);
-        } else {
-            total += fs.statSync(fullPath).size;
-        }
-    }
-    return total;
-};
-
-describe('GitSQLite Architecture Validation', () => {
-    let db;
-
-    before(() => {
-        // Initialize Git repo optimizations and register driver configurations
-        GitSQLite.setupGit();
-    });
-
-    it('Test 1: Initialization & VFS Sharding', () => {
-        db = GitSQLite.open('.db');
-
-        // Create initial schema
-        db.exec(`
-            CREATE TABLE test_data (id INTEGER PRIMARY KEY, name TEXT, value BLOB);
-            CREATE TABLE test_settings (config_key TEXT PRIMARY KEY, config_val TEXT);
-        `);
-
-        // Insert initial baseline data using parameterized transactions
-        const insertData = db.prepare("INSERT INTO test_data (name, value) VALUES (?, randomblob(100))");
-        const insertSettings = db.prepare("INSERT INTO test_settings (config_key, config_val) VALUES (?, ?)");
-
-        db.exec('BEGIN TRANSACTION;');
-        for (let i = 1; i <= 50; i++) {
-            insertData.run(`Initial ${i}`);
-        }
-        insertSettings.run('theme', 'dark');
-        db.exec('COMMIT;');
-
-        // Close to flush SQLite connections (VFS sync)
-        db.close(); 
-
-        // Assert our C VFS dynamically intercepted physical I/O and sharded the B-Tree!
-        assert.ok(fs.existsSync('.db/pages'), 'Pages directory should exist');
-        assert.ok(fs.existsSync('.db/pages/size.meta'), 'size.meta persistence state should exist');
-
-        // Verify git tracking structure
-        assert.ok(fs.existsSync('.db/.gitignore'), '.gitignore should be generated');
-        assert.ok(fs.existsSync('.db/pages/.gitattributes'), '.gitattributes should be generated');
-
-        // Stage and Commit Snapshot 1
-        runGit('git add -A -f .db/');
-        runGit('git commit -m "Snapshot 1: Initial DB state"');
-    });
-
-    it('Test 2: VACUUM and Physical B-Tree Shrinkage', () => {
-        db = GitSQLite.open('.db');
-
-        // Insert a massive amount of rows to forcibly expand the B-Tree footprint
-        db.exec('BEGIN TRANSACTION;');
-        const insertData = db.prepare("INSERT INTO test_data (name, value) VALUES ('Bulk', randomblob(100))");
-        for (let i = 0; i < 10000; i++) {
-            insertData.run();
-        }
-        db.exec('COMMIT;');
-        db.close();
-
-        // Capture total file system footprint of the expanded sharded VFS
-        const sizeBefore = getDirSize('.db/pages');
-
-        // Reopen, execute a massive deletion, and trigger a vacuum
-        db = GitSQLite.open('.db');
-        db.exec("DELETE FROM test_data WHERE id > 50;");
-        db.exec("VACUUM;");
-        db.close();
-
-        // Capture total file system footprint post-vacuum
-        const sizeAfter = getDirSize('.db/pages');
-
-        // Assert mathematical shrinkage: our xTruncate implementation successfully unlinked dead pages!
-        assert.ok(sizeAfter < sizeBefore, `Directory size must shrink after VACUUM. Before: ${sizeBefore}, After: ${sizeAfter}`);
-
-        // Stage and Commit Snapshot 2 (Unlinked files must be natively staged as Git deletions)
-        runGit('git add -A -f .db/');
-        runGit('git commit -m "Snapshot 2: Vacuumed database"');
-    });
-
-    it('Test 3: Git Branching & True 3-Way Merge (DDL + Row Conflicts)', () => {
-        // --- CONFLICT BRANCH MUTATIONS ---
-        runGit('git checkout -b conflict_branch');
-        db = GitSQLite.open('.db');
-        db.exec(`
-            INSERT INTO test_data (id, name, value) VALUES (10001, 'conflict_branch', randomblob(100));
-            INSERT INTO test_settings (config_key, config_val) VALUES ('plugin', 'enabled');
-            UPDATE test_data SET name = 'branch_update' WHERE id = 10;
-            DELETE FROM test_data WHERE id = 15;
-            UPDATE test_data SET name = 'branch_wins' WHERE id = 50;
-            CREATE TABLE new_feature (id INTEGER PRIMARY KEY, feature_name TEXT);
-            INSERT INTO new_feature (id, feature_name) VALUES (1, 'version_control');
-            CREATE INDEX idx_test_name ON test_data(name);
-            DROP TABLE test_settings;
-        `);
-        db.close();
-        runGit('git add -A -f .db/');
-        runGit('git commit -m "conflict_branch: Schema evolution and row updates"');
-
-        // --- MASTER BRANCH MUTATIONS ---
-        runGit('git checkout master');
-        db = GitSQLite.open('.db');
-        db.exec(`
-            INSERT INTO test_data (id, name, value) VALUES (10002, 'master', randomblob(100));
-            CREATE TABLE unrelated_table (id INTEGER);
-            UPDATE test_data SET name = 'master_update' WHERE id = 20;
-            UPDATE test_data SET name = 'master_wins' WHERE id = 50;
-        `);
-        db.close();
-        runGit('git add -A -f .db/');
-        runGit('git commit -m "master: Insertions and row updates"');
-
-        // --- THE CUSTOM GIT MERGE STRATEGY ---
-        const binPath = path.resolve(__dirname, 'c/output');
-        try {
-            // By utilizing the -s sqlitevfs strategy and augmenting our PATH, Git natively
-            // delegates the entire branch resolution to our SQLite C engine!
-            execSync(`PATH=$PATH:${binPath} git merge -s sqlitevfs conflict_branch -m "Merge conflict_branch into master"`, { stdio: 'pipe' });
-        } catch (e) {
-            console.error("Merge failed:\n", e.stdout?.toString(), e.stderr?.toString());
-            throw e;
-        }
-
-        // --- ASSERTIONS (Mathematical verification of 3-Way Logical Merge) ---
-        db = GitSQLite.open('.db');
-        
-        // Assert True Row Conflict Resolution (Master Wins)
-        const row50 = db.prepare("SELECT name FROM test_data WHERE id = 50").get();
-        assert.strictEqual(row50.name, 'master_wins', 'Master must win true row-level collisions by Custom Merge Strategy logic');
-
-        // Assert standard branch updates
-        const row10 = db.prepare("SELECT name FROM test_data WHERE id = 10").get();
-        assert.strictEqual(row10.name, 'branch_update');
-
-        const row20 = db.prepare("SELECT name FROM test_data WHERE id = 20").get();
-        assert.strictEqual(row20.name, 'master_update');
-
-        // Assert branch deletions
-        const row15 = db.prepare("SELECT name FROM test_data WHERE id = 15").get();
-        assert.strictEqual(row15, undefined, 'Row 15 must have been deleted by conflict_branch');
-
-        // Assert 3-Way Schema Evolution (DDL Merge)
-        const settingsTable = db.prepare("SELECT count(*) as cnt FROM sqlite_schema WHERE name='test_settings'").get();
-        assert.strictEqual(settingsTable.cnt, 0, 'test_settings table must be mathematically DROPPED');
-
-        const newFeatureRow = db.prepare("SELECT feature_name FROM new_feature WHERE id = 1").get();
-        assert.strictEqual(newFeatureRow.feature_name, 'version_control', 'new_feature table and its row data must exist');
-
-        const idx = db.prepare("SELECT name FROM sqlite_schema WHERE type='index' AND name='idx_test_name'").get();
-        assert.ok(idx, 'idx_test_name index must have been created');
-
-        db.close();
-    });
-
-    it('Test 4: Time Travel (Disaster Recovery via Git)', () => {
-        db = GitSQLite.open('.db');
-        
-        // Assert initial baseline existence
-        let featureTable = db.prepare("SELECT count(*) as cnt FROM sqlite_schema WHERE name='new_feature'").get();
-        assert.strictEqual(featureTable.cnt, 1);
-
-        // Execute a catastrophic, destructive operation
-        db.exec("DROP TABLE new_feature;");
-        featureTable = db.prepare("SELECT count(*) as cnt FROM sqlite_schema WHERE name='new_feature'").get();
-        assert.strictEqual(featureTable.cnt, 0, 'Table must be completely dropped from SQLite');
-        db.close();
-
-        // Commit the disaster
-        runGit('git add -A -f .db/');
-        runGit('git commit -m "Oops, accidentally dropped new_feature"');
-
-        // Initiate Time Travel (Git Reset)
-        // Because the database is perfectly versioned, Git instantly restores the .bin files
-        runGit('git reset --hard HEAD~1');
-
-        // Reopen DB and Verify absolute recovery
-        db = GitSQLite.open('.db');
-        featureTable = db.prepare("SELECT count(*) as cnt FROM sqlite_schema WHERE name='new_feature'").get();
-        assert.strictEqual(featureTable.cnt, 1, 'Table schema must be fully resurrected natively by Git!');
-        
-        const row = db.prepare("SELECT feature_name FROM new_feature WHERE id = 1").get();
-        assert.strictEqual(row.feature_name, 'version_control', 'Physical row data must be fully intact after time travel!');
-
-        db.close();
-    });
-});