@joinezco/codeblock 0.0.8

package/dist/utils/snapshot.js

@@ -0,0 +1,299 @@
+import fsPromises from 'fs/promises';
+import multimatch from 'multimatch';
+import { CborEncoder } from '@jsonjoy.com/json-pack/lib/cbor/CborEncoder';
+import { CborDecoder } from '@jsonjoy.com/json-pack/lib/cbor/CborDecoder';
+import { Writer } from '@jsonjoy.com/util/lib/buffers/Writer';
+export const writer = new Writer(1024 * 32);
+export const encoder = new CborEncoder(writer);
+export const decoder = new CborDecoder();
+// Cross-platform compression utilities
+const isNode = typeof process !== 'undefined' && process.versions?.node;
+/**
+ * Compress data using gzip compression.
+ * Uses Node.js zlib in Node.js environment, browser CompressionStream in browser.
+ */
+export const compress = async (data) => {
+    if (isNode) {
+        // Node.js environment
+        try {
+            const { gzip } = await import('zlib');
+            const { promisify } = await import('util');
+            const gzipAsync = promisify(gzip);
+            return new Uint8Array(await gzipAsync(data));
+        }
+        catch (error) {
+            console.warn('Node.js compression failed, returning uncompressed data:', error);
+            return data;
+        }
+    }
+    else {
+        // Browser environment
+        if (typeof CompressionStream === 'undefined') {
+            // Fallback: return uncompressed data if CompressionStream is not available
+            console.warn('CompressionStream not available, returning uncompressed data');
+            return data;
+        }
+        try {
+            const stream = new CompressionStream('gzip');
+            const writer = stream.writable.getWriter();
+            const reader = stream.readable.getReader();
+            writer.write(new Uint8Array(data));
+            writer.close();
+            const chunks = [];
+            let done = false;
+            while (!done) {
+                const { value, done: readerDone } = await reader.read();
+                done = readerDone;
+                if (value)
+                    chunks.push(value);
+            }
+            // Concatenate all chunks
+            const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+            const result = new Uint8Array(totalLength);
+            let offset = 0;
+            for (const chunk of chunks) {
+                result.set(chunk, offset);
+                offset += chunk.length;
+            }
+            return result;
+        }
+        catch (error) {
+            console.warn('Browser compression failed, returning uncompressed data:', error);
+            return data;
+        }
+    }
+};
+/**
+ * Decompress gzip-compressed data.
+ * Uses Node.js zlib in Node.js environment, browser DecompressionStream in browser.
+ */
+/**
+ * Check if data appears to be gzip compressed by looking at the magic bytes
+ */
+const isGzipCompressed = (data) => {
+    return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
+};
+export const decompress = async (data) => {
+    console.debug('decompressData: Starting decompression, data length:', data.length);
+    console.debug('decompressData: First few bytes:', Array.from(data.slice(0, 10)).map(b => '0x' + b.toString(16).padStart(2, '0')).join(' '));
+    // Check if data is actually compressed
+    if (!isGzipCompressed(data)) {
+        console.debug('decompressData: Data does not appear to be gzip compressed, returning as-is');
+        return data;
+    }
+    console.debug('decompressData: Data appears to be gzip compressed');
+    if (isNode) {
+        console.debug('decompressData: Using Node.js zlib');
+        // Node.js environment
+        const { gunzip } = await import('zlib');
+        const { promisify } = await import('util');
+        const gunzipAsync = promisify(gunzip);
+        const result = new Uint8Array(await gunzipAsync(data));
+        console.debug('decompressData: Node.js decompression successful, result length:', result.length);
+        return result;
+    }
+    else {
+        console.debug('decompressData: Using browser DecompressionStream');
+        // Browser environment
+        if (typeof DecompressionStream === 'undefined') {
+            // Fallback: assume data is uncompressed if DecompressionStream is not available
+            console.warn('decompressData: DecompressionStream not available, assuming uncompressed data');
+            return data;
+        }
+        try {
+            const stream = new DecompressionStream('gzip');
+            const writer = stream.writable.getWriter();
+            const reader = stream.readable.getReader();
+            console.debug('decompressData: Writing data to decompression stream');
+            writer.write(new Uint8Array(data));
+            writer.close();
+            const chunks = [];
+            let done = false;
+            console.debug('decompressData: Reading decompressed chunks');
+            while (!done) {
+                const { value, done: readerDone } = await reader.read();
+                done = readerDone;
+                if (value) {
+                    console.debug('decompressData: Received chunk of length:', value.length);
+                    chunks.push(value);
+                }
+            }
+            // Concatenate all chunks
+            const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+            console.debug('decompressData: Total decompressed length:', totalLength);
+            const result = new Uint8Array(totalLength);
+            let offset = 0;
+            for (const chunk of chunks) {
+                result.set(chunk, offset);
+                offset += chunk.length;
+            }
+            console.debug('decompressData: Browser decompression successful');
+            return result;
+        }
+        catch (error) {
+            console.warn('decompressData: Browser decompression failed, returning original data:', error);
+            return data;
+        }
+    }
+};
+export const buildFilter = ({ include, exclude }) => {
+    return (path) => {
+        if (!(include || exclude))
+            return true;
+        const included = include ? !!multimatch(path, include, { partial: true }).length : true;
+        const excluded = exclude ? !!multimatch(path, exclude).length : false;
+        return included && !excluded;
+    };
+};
+export const getGitignored = async (path, fs = typeof fsPromises) => {
+    // @ts-expect-error
+    const content = await fs.readFile(path, 'utf-8');
+    // @ts-ignore
+    return parse(content).patterns;
+};
+export const snapshotDefaults = {
+    root: typeof process !== 'undefined' ? process.cwd() : './',
+    filter: () => Promise.resolve(true),
+};
+/**
+ * Takes a snapshot of the file system based on the provided properties.
+ * The snapshot is encoded with CBOR and compressed with gzip.
+ *
+ * @param props - The properties to configure the snapshot.
+ */
+export const takeSnapshot = async (props = {}) => {
+    const { root, filter } = { ...snapshotDefaults, ...props };
+    console.debug('Taking snapshot of filesystem', { root, filter });
+    const snapshot = await Snapshot
+        .take({ fs: fsPromises, path: root, filter })
+        .then((snapshot) => encoder.encode(snapshot))
+        .then((encoded) => compress(encoded));
+    return snapshot;
+};
+export var Snapshot;
+(function (Snapshot) {
+    Snapshot.take = async ({ fs, path, filter, separator = '/' }) => {
+        if (filter && !(await filter(path)))
+            return null;
+        // TODO: think about handling snapshotting symlinks better
+        // for now we just resolve and include
+        const stats = await fs.stat(path);
+        if (stats.isDirectory()) {
+            const list = await fs.readdir(path);
+            const entries = {};
+            const dir = path.endsWith(separator) ? path : path + separator;
+            for (const child of list) {
+                const childSnapshot = await Snapshot.take({ fs, path: `${dir}${child}`, separator, filter });
+                if (childSnapshot)
+                    entries[child] = childSnapshot;
+            }
+            return [0 /* Folder */, {}, entries];
+        }
+        else if (stats.isFile()) {
+            const buf = (await fs.readFile(path));
+            const uint8 = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
+            return [1 /* File */, stats, uint8];
+        }
+        else if (stats.isSymbolicLink()) {
+            // TODO: branch never actually reached as `fs.stat` doesn't return symlinks
+            return [
+                2 /* Symlink */,
+                {
+                    target: (await fs.readlink(path, { encoding: 'utf8' })),
+                },
+            ];
+        }
+        return null;
+    };
+    Snapshot.mount = async (buffer, { fs, path = '/', separator = '/' }) => {
+        try {
+            console.debug('Snapshot.mount: Starting mount process');
+            console.debug('Snapshot.mount: Buffer type:', typeof buffer);
+            console.debug('Snapshot.mount: Buffer length:', buffer?.byteLength || buffer?.length || 'unknown');
+            console.debug('Snapshot.mount: Buffer constructor:', buffer?.constructor?.name);
+            // Convert buffer to Uint8Array if needed
+            const uint8Buffer = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
+            console.debug('Snapshot.mount: Converted to Uint8Array, length:', uint8Buffer.length);
+            // Check if data appears to be compressed
+            const isCompressed = isGzipCompressed(uint8Buffer);
+            console.debug('Snapshot.mount: Data appears compressed:', isCompressed);
+            // Try to decompress the buffer first, then decode
+            let decompressed;
+            try {
+                decompressed = await decompress(uint8Buffer);
+                console.debug('Snapshot.mount: Successfully processed buffer, decompressed length:', decompressed.length);
+            }
+            catch (decompressError) {
+                console.warn('Snapshot.mount: Decompression failed, using original buffer:', decompressError);
+                // Fallback: assume the buffer is already uncompressed (for backward compatibility)
+                decompressed = uint8Buffer;
+            }
+            console.debug('Snapshot.mount: Attempting to decode CBOR data...');
+            const snapshot = await decoder.decode(decompressed);
+            console.debug('Snapshot.mount: Successfully decoded snapshot, type:', typeof snapshot);
+            console.debug('Snapshot.mount: Snapshot structure:', Array.isArray(snapshot) ? `Array[${snapshot.length}]` : snapshot);
+            if (snapshot) {
+                console.debug('Snapshot.mount: Starting fromSnapshot process...');
+                await fromSnapshot(snapshot, { fs, path, separator });
+                console.debug('Snapshot.mount: Successfully mounted snapshot');
+            }
+            else {
+                console.warn('Snapshot.mount: Decoded snapshot is null or undefined');
+            }
+        }
+        catch (error) {
+            console.error('Snapshot.mount: Failed to mount snapshot:', error);
+            throw error;
+        }
+    };
+    /**
+     * Load and mount a snapshot directly from a URL in a web worker environment.
+     * This is more efficient for large snapshots as it avoids transferring data through the main thread.
+     */
+    Snapshot.loadAndMount = async (url, { fs, path = '/', separator = '/' }) => {
+        try {
+            console.debug('Snapshot.loadAndMount: Starting direct load from URL:', url);
+            // Fetch the snapshot data directly in the worker
+            const response = await fetch(url);
+            if (!response.ok) {
+                throw new Error(`Failed to fetch snapshot: ${response.status} ${response.statusText}`);
+            }
+            console.debug('Snapshot.loadAndMount: Response received, content-length:', response.headers.get('content-length'));
+            // Get the response as ArrayBuffer for better performance
+            const arrayBuffer = await response.arrayBuffer();
+            const uint8Buffer = new Uint8Array(arrayBuffer);
+            console.debug('Snapshot.loadAndMount: Downloaded buffer length:', uint8Buffer.length);
+            // Use the existing mount logic
+            await Snapshot.mount(uint8Buffer, { fs, path, separator });
+        }
+        catch (error) {
+            console.error('Snapshot.loadAndMount: Failed to load and mount snapshot:', error);
+            throw error;
+        }
+    };
+})(Snapshot || (Snapshot = {}));
+export const fromSnapshot = async (snapshot, { fs, path = '/', separator = '/' }) => {
+    if (!snapshot)
+        return;
+    switch (snapshot[0]) {
+        case 0: {
+            if (!path.endsWith(separator))
+                path = path + separator;
+            const [, , entries] = snapshot;
+            fs.mkdirSync(path, { recursive: true });
+            for (const [name, child] of Object.entries(entries))
+                await fromSnapshot(child, { fs, path: `${path}${name}`, separator });
+            break;
+        }
+        case 1: {
+            const [, , data] = snapshot;
+            fs.writeFileSync(path, data);
+            break;
+        }
+        case 2: {
+            const [, { target }] = snapshot;
+            fs.symlinkSync(target, path);
+            break;
+        }
+    }
+};

package/dist/workers/fs.worker.d.ts

@@ -0,0 +1,11 @@
+import { MountArgs, MountResult } from "../types";
+export declare const mount: ({ buffer, mountPoint }: MountArgs) => Promise<MountResult>;
+/**
+ * Optimized mount function that loads snapshots directly from URLs.
+ * This is much more efficient for large snapshots as it avoids transferring
+ * data through the main thread.
+ */
+export declare const mountFromUrl: ({ url, mountPoint }: {
+    url: string;
+    mountPoint?: string;
+}) => Promise<MountResult>;

package/dist/workers/fs.worker.js

@@ -0,0 +1,93 @@
+import * as Comlink from "comlink";
+import { watchOptionsTransferHandler, asyncGeneratorTransferHandler } from '../rpc/serde';
+import { Snapshot } from "../utils";
+Comlink.transferHandlers.set('asyncGenerator', asyncGeneratorTransferHandler);
+Comlink.transferHandlers.set('watchOptions', watchOptionsTransferHandler);
+let filesystems = [];
+export const mount = async ({ buffer, mountPoint = '/' }) => {
+    let filesystem;
+    try {
+        console.log('Importing memfs after FS mount...');
+        const { fs } = await import('@joinezco/memfs');
+        console.log("FS imported");
+        try {
+            if (buffer) {
+                console.log(`Mounting filesystem snapshot at [${mountPoint}]...`, buffer);
+                // Convert Node Buffer to ArrayBuffer if needed
+                const uint8 = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
+                const aligned = uint8.byteOffset === 0 && uint8.byteLength === uint8.buffer.byteLength
+                    ? uint8.buffer
+                    : uint8.buffer.slice(uint8.byteOffset, uint8.byteOffset + uint8.byteLength);
+                console.log('Aligned ArrayBuffer:', aligned);
+                await Snapshot.mount(new Uint8Array(aligned), {
+                    // @ts-ignore
+                    fs,
+                });
+            }
+            else {
+                console.log('Getting storage directory...');
+                // const handle = await navigator.storage.getDirectory();
+                console.log('Got storage directory');
+                console.log('Attempting to remove directory...');
+                try {
+                    // TODO: clear storage button
+                    // @ts-ignore
+                    // await handle.remove({ recursive: true });
+                    console.log('Successfully removed directory');
+                }
+                catch (removeErr) {
+                    console.error('Error removing directory:', removeErr);
+                    // Continue anyway, this might not be critical
+                }
+            }
+            console.log('Returning proxy from worker', fs);
+            filesystem = Comlink.proxy({ fs });
+            filesystems.push(filesystem);
+        }
+        catch (e) {
+            console.error('Worker initialization failed with error:', e);
+            throw e; // Make sure error propagates
+        }
+    }
+    catch (e) {
+        console.error('Error importing memfs:', e);
+    }
+    console.log('mounting fs', { buffer, mountPoint });
+    return filesystem;
+};
+/**
+ * Optimized mount function that loads snapshots directly from URLs.
+ * This is much more efficient for large snapshots as it avoids transferring
+ * data through the main thread.
+ */
+export const mountFromUrl = async ({ url, mountPoint = '/' }) => {
+    let filesystem;
+    try {
+        const { fs } = await import('@joinezco/memfs');
+        console.log(`Loading and mounting filesystem snapshot from URL: ${url} at [${mountPoint}]...`);
+        const startTime = performance.now();
+        await Snapshot.loadAndMount(url, {
+            // @ts-ignore
+            fs,
+            path: mountPoint
+        });
+        const endTime = performance.now();
+        console.log(`Snapshot loaded and mounted in ${Math.round(endTime - startTime)}ms`);
+        console.log('Returning proxy from worker', fs);
+        filesystem = Comlink.proxy({ fs });
+        filesystems.push(filesystem);
+    }
+    catch (e) {
+        console.error('Error loading snapshot from URL:', e);
+        throw e;
+    }
+    return filesystem;
+};
+onconnect = async function (event) {
+    const [port] = event.ports;
+    console.log('workers/fs connected on port: ', port);
+    port.addEventListener('close', () => {
+        console.log('fs port closed');
+    });
+    Comlink.expose({ mount, mountFromUrl }, port);
+};

package/dist/workers/javascript.worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/javascript.worker.js

@@ -0,0 +1,20 @@
+import * as Comlink from 'comlink';
+import { createLanguageServer } from '../lsps/typescript';
+import { createConnection } from 'vscode-languageserver/browser';
+import { BrowserMessageReader, BrowserMessageWriter } from '@volar/language-server/browser';
+// TODO: get rid of this
+// instead, create language specific workers (with a smarter client)
+// i.e typescript.worker.ts / rust.worker.ts / ...
+onconnect = async (event) => {
+    const [port] = event.ports;
+    console.debug('LSP worker connected on port: ', port);
+    const reader = new BrowserMessageReader(port);
+    const writer = new BrowserMessageWriter(port);
+    const connection = createConnection(reader, writer);
+    connection.listen();
+    const proxy = async ({ fs }) => {
+        console.log('creating language server');
+        await createLanguageServer({ fs, connection });
+    };
+    Comlink.expose({ createLanguageServer: proxy }, port);
+};

package/package.json

@@ -0,0 +1,95 @@
+{
+  "name": "@joinezco/codeblock",
+  "version": "0.0.8",
+  "type": "module",
+  "module": "dist/index.js",
+  "main": "dist/index.js",
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "devDependencies": {
+    "@playwright/test": "^1.52.0",
+    "@types/lodash": "^4.17.16",
+    "@types/markdown-it": "^14.1.2",
+    "@types/node": "^24.3.1",
+    "@types/parse-gitignore": "^1.0.2",
+    "@types/sharedworker": "^0.0.181",
+    "multimatch": "^7.0.0",
+    "process": "^0.11.10",
+    "tsx": "^4.19.4",
+    "vite": "^7.0.6",
+    "vitest": "^3.2.4"
+  },
+  "dependencies": {
+    "@babel/runtime": "^7.27.1",
+    "@codemirror/autocomplete": "^6.18.6",
+    "@codemirror/commands": "^6.8.1",
+    "@codemirror/lang-cpp": "^6.0.3",
+    "@codemirror/lang-css": "^6.3.1",
+    "@codemirror/lang-html": "^6.4.9",
+    "@codemirror/lang-java": "^6.0.2",
+    "@codemirror/lang-javascript": "^6.2.3",
+    "@codemirror/lang-json": "^6.0.2",
+    "@codemirror/lang-less": "^6.0.2",
+    "@codemirror/lang-markdown": "^6.3.4",
+    "@codemirror/lang-php": "^6.0.2",
+    "@codemirror/lang-python": "^6.2.0",
+    "@codemirror/lang-rust": "^6.0.1",
+    "@codemirror/lang-sass": "^6.0.2",
+    "@codemirror/lang-sql": "^6.9.1",
+    "@codemirror/lang-xml": "^6.1.0",
+    "@codemirror/lang-yaml": "^6.1.2",
+    "@codemirror/language": "^6.11.0",
+    "@codemirror/legacy-modes": "^6.5.1",
+    "@codemirror/lint": "^6.8.5",
+    "@codemirror/search": "^6.5.10",
+    "@codemirror/state": "^6.5.2",
+    "@codemirror/view": "^6.36.7",
+    "@jsonjoy.com/json-pack": "^1.2.0",
+    "@jsonjoy.com/util": "^1.6.0",
+    "@lezer/highlight": "^1.2.1",
+    "@open-rpc/client-js": "^1.8.1",
+    "@typescript/vfs": "^1.6.1",
+    "@uiw/codemirror-theme-vscode": "^4.23.12",
+    "@volar/language-server": "2.4.13",
+    "@volar/language-service": "2.4.13",
+    "@volar/typescript": "2.4.13",
+    "comlink": "^4.4.2",
+    "events": "^3.3.0",
+    "ignore": "^7.0.4",
+    "lodash": "^4.17.21",
+    "lz-string": "^1.5.0",
+    "markdown-it": "^14.1.0",
+    "minisearch": "^7.1.2",
+    "parse-gitignore": "^2.0.0",
+    "path-browserify": "^1.0.1",
+    "style-mod": "^4.1.2",
+    "typescript": "~5.9.2",
+    "uuid": "^11.1.0",
+    "vite-plugin-node-polyfills": "^0.24.0",
+    "volar-service-typescript": "0.0.65",
+    "vscode-languageserver": "^9.0.1",
+    "vscode-languageserver-protocol": "^3.17.5",
+    "vscode-languageserver-textdocument": "^1.0.12",
+    "vscode-uri": "^3.1.0",
+    "@marimo-team/codemirror-languageserver": "1.16.0",
+    "@joinezco/jswasi": "0.0.1",
+    "@joinezco/memfs": "4.23.0"
+  },
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc",
+    "build:preview": "vite build",
+    "preview": "vite preview"
+  }
+}