@joinezco/codeblock 0.0.8

package/dist/utils/fs.js

@@ -0,0 +1,310 @@
+import * as Comlink from "comlink";
+import { watchOptionsTransferHandler, asyncGeneratorTransferHandler } from "../rpc/serde";
+import { FileType } from '@volar/language-service';
+import { constants } from "@joinezco/jswasi";
+Comlink.transferHandlers.set("asyncGenerator", asyncGeneratorTransferHandler);
+Comlink.transferHandlers.set("watchOptions", watchOptionsTransferHandler);
+export var Vfs;
+(function (Vfs) {
+    Vfs.fromJswasiFs = async (jswasiFs) => {
+        // Map WASI filetype to @volar/language-service FileType
+        const toVolarFileType = (filetype) => {
+            // WASI preview1 common values: 3 = directory, 4 = regular file, 7 = symlink
+            switch (filetype) {
+                case 3: return FileType.Directory;
+                case 7: return FileType.SymbolicLink;
+                default: return FileType.File;
+            }
+        };
+        // Normalize to absolute path for TopLevelFs
+        const ensureAbs = (p) => (p && p.startsWith("/")) ? p : `/${p ?? ""}`;
+        // Pull constants if available (fall back to literals when missing)
+        const WASI_ESUCCESS = constants.WASI_ESUCCESS ?? 0;
+        const WASI_EEXIST = constants.WASI_EEXIST ?? 20;
+        const WASI_O_TRUNC = constants.WASI_O_TRUNC ?? 0x00000010;
+        const WASI_O_CREAT = constants.WASI_O_CREAT ?? 0x00000001;
+        const WASI_O_DIRECTORY = constants.WASI_O_DIRECTORY ?? 0x00020000;
+        return {
+            async readFile(path) {
+                const abs = ensureAbs(path);
+                const { desc, err } = await jswasiFs.open(abs);
+                if (err !== WASI_ESUCCESS)
+                    throw new Error(`readFile open failed (${err}) for ${abs}`);
+                const { content, err: readErr } = await desc.read_str();
+                if (readErr !== WASI_ESUCCESS)
+                    throw new Error(`readFile read_str failed (${readErr}) for ${abs}`);
+                desc.close(); // Close after reading
+                return content;
+            },
+            async writeFile(path, data) {
+                const abs = ensureAbs(path);
+                const { desc, err } = await jswasiFs.open(abs, 0, WASI_O_CREAT | WASI_O_TRUNC);
+                if (err !== WASI_ESUCCESS)
+                    throw new Error(`writeFile open failed (${err}) for ${abs}`);
+                const encoder = new TextEncoder();
+                const buf = encoder.encode(data);
+                const { err: writeErr } = await desc.pwrite(buf.buffer, 0n);
+                desc.close(); // Close after writing
+                if (writeErr !== WASI_ESUCCESS)
+                    throw new Error(`writeFile pwrite failed (${writeErr}) for ${abs}`);
+            },
+            async *watch(_path, { signal }) {
+                return jswasiFs.watch(_path, { signal });
+            },
+            async mkdir(path, options) {
+                const abs = ensureAbs(path);
+                if (options?.recursive) {
+                    const parts = abs.split("/").filter(Boolean);
+                    let cur = "/";
+                    for (const part of parts) {
+                        cur = cur === "/" ? `/${part}` : `${cur}/${part}`;
+                        console.log('creating', { cur, abs });
+                        const exists = await this.exists(cur);
+                        if (exists)
+                            continue;
+                        const res = await jswasiFs.createDir(cur);
+                        if (res !== WASI_ESUCCESS && res !== WASI_EEXIST) {
+                            throw new Error(`mkdir recursive failed (${res}) at ${cur}`);
+                        }
+                    }
+                }
+                else {
+                    const res = await jswasiFs.createDir(abs);
+                    if (res !== WASI_ESUCCESS) {
+                        throw new Error(`mkdir failed (${res}) for ${abs}`);
+                    }
+                }
+            },
+            async readDir(path) {
+                const abs = ensureAbs(path);
+                const { desc, err } = await jswasiFs.open(abs, 0, WASI_O_DIRECTORY);
+                if (err !== WASI_ESUCCESS)
+                    throw new Error(`readDir open failed (${err}) for ${abs}`);
+                const { err: rerr, dirents } = await desc.readdir(true);
+                if (rerr !== WASI_ESUCCESS)
+                    throw new Error(`readDir readdir failed (${rerr}) for ${abs}`);
+                return dirents.map((d) => [d.name, toVolarFileType(d.d_type)]);
+            },
+            async exists(path) {
+                const abs = ensureAbs(path);
+                const { desc, err } = await jswasiFs.open(abs);
+                if (err !== WASI_ESUCCESS)
+                    return false;
+                const stat = await desc.getFilestat();
+                desc.close(); // Close after getting file status
+                return stat.err === WASI_ESUCCESS;
+            },
+            async stat(path) {
+                const abs = ensureAbs(path);
+                const { desc, err } = await jswasiFs.open(abs);
+                if (err !== WASI_ESUCCESS)
+                    return null;
+                const res = await desc.getFilestat();
+                desc.close(); // Close after getting file status
+                if (res.err !== WASI_ESUCCESS)
+                    return null;
+                const filestat = res.filestat;
+                // filestat times are typically in ns; convert to ms for Date
+                const nsToDate = (ns) => new Date(Number(ns / 1000000n));
+                return {
+                    name: abs,
+                    atime: nsToDate(filestat.atim),
+                    mtime: nsToDate(filestat.mtim),
+                    ctime: nsToDate(filestat.ctim),
+                    size: Number(filestat.size),
+                    type: toVolarFileType(filestat.filetype),
+                };
+            },
+        };
+    };
+    // TODO: this is incorrect, fs is a Comlink proxy
+    Vfs.fromMemfs = (fs) => {
+        return {
+            async readFile(path) {
+                return fs.promises.readFile(path, { encoding: "utf-8" });
+            },
+            async writeFile(path, data) {
+                await fs.promises.writeFile(path, data);
+            },
+            async *watch(path, { signal }) {
+                for await (const e of await fs.promises.watch(path, { signal, encoding: "utf-8", recursive: true })) {
+                    yield e;
+                }
+            },
+            async mkdir(path, options) {
+                await fs.promises.mkdir(path, options);
+            },
+            async readDir(path) {
+                const files = await fs.readdirSync(path, { withFileTypes: true, encoding: "utf-8" });
+                // @ts-expect-error
+                return files.map((ent) => {
+                    let type = FileType.File;
+                    switch (ent.mode & 0o170000) {
+                        case 0o040000:
+                            type = FileType.Directory;
+                            break;
+                        case 0o120000:
+                            type = FileType.SymbolicLink;
+                            break;
+                    }
+                    return [ent.name, type];
+                });
+            },
+            async exists(path) {
+                return fs.existsSync(path);
+            },
+            async stat(path) {
+                try {
+                    const stat = await fs.promises.stat(path);
+                    let type = FileType.File;
+                    switch (stat.mode & 0o170000) {
+                        case 0o040000:
+                            type = FileType.Directory;
+                            break;
+                        case 0o120000:
+                            type = FileType.SymbolicLink;
+                            break;
+                    }
+                    // console.debug(`Stat success "${path}"`);
+                    return {
+                        name: path,
+                        atime: stat.atime,
+                        mtime: stat.mtime,
+                        ctime: stat.ctime,
+                        size: stat.size,
+                        type,
+                    };
+                }
+                catch (err) {
+                    return null;
+                }
+            }
+        };
+    };
+    Vfs.fromNodelike = (fs) => {
+        return {
+            async readFile(path) {
+                return fs.readFile(path, { encoding: "utf-8" });
+            },
+            async writeFile(path, data) {
+                await fs.writeFile(path, data);
+            },
+            async *watch(path, { signal }) {
+                for await (const e of await fs.watch(path, { signal, encoding: "utf-8", recursive: true })) {
+                    yield e;
+                }
+            },
+            async mkdir(path, options) {
+                await fs.mkdir(path, options);
+            },
+            async readDir(path) {
+                const files = await fs.readdir(path, { withFileTypes: true, encoding: "utf-8" });
+                return files.map((ent) => {
+                    let type = FileType.File;
+                    switch (ent.stats.mode & 0o170000) {
+                        case 0o040000:
+                            type = FileType.Directory;
+                            break;
+                        case 0o120000:
+                            type = FileType.SymbolicLink;
+                            break;
+                    }
+                    return [ent.path, type];
+                });
+            },
+            async exists(path) {
+                try {
+                    await fs.access(path);
+                    return true;
+                }
+                catch {
+                    return false;
+                }
+            },
+            async stat(path) {
+                try {
+                    const stat = await fs.stat(path);
+                    let type = FileType.File;
+                    switch (stat.mode & 0o170000) {
+                        case 0o040000:
+                            type = FileType.Directory;
+                            break;
+                        case 0o120000:
+                            type = FileType.SymbolicLink;
+                            break;
+                    }
+                    // console.debug(`Stat success "${path}"`);
+                    return {
+                        name: path,
+                        atime: stat.atime,
+                        mtime: stat.mtime,
+                        ctime: stat.ctime,
+                        size: stat.size,
+                        type,
+                    };
+                }
+                catch (err) {
+                    return null;
+                }
+            }
+        };
+    };
+    /**
+     * Create a filesystem worker with optional snapshot data.
+     *
+     * @param bufferOrUrl - Either a snapshot buffer or URL to a snapshot file.
+     *                     If a URL is provided, it will be loaded directly in the worker
+     *                     for better performance with large files.
+     */
+    Vfs.worker = async (bufferOrUrl) => {
+        const url = new URL('../workers/fs.worker.js', import.meta.url);
+        const worker = new SharedWorker(url, { type: 'module' });
+        worker.port.start();
+        const proxy = Comlink.wrap(worker.port);
+        let fs;
+        if (!bufferOrUrl) {
+            // No buffer or URL provided - create empty filesystem
+            ({ fs } = await proxy.mount({ mountPoint: '/' }));
+        }
+        else if (typeof bufferOrUrl === 'string') {
+            // URL provided - use optimized mountFromUrl for better performance
+            ({ fs } = await proxy.mountFromUrl({
+                url: bufferOrUrl,
+                mountPoint: '/'
+            }));
+        }
+        else {
+            // Buffer provided - use traditional mount method
+            ({ fs } = await proxy.mount(Comlink.transfer({ buffer: bufferOrUrl, mountPoint: "/" }, [bufferOrUrl])));
+        }
+        return Comlink.proxy(Vfs.fromMemfs(fs));
+    };
+    async function* walk(fs, path) {
+        const files = await fs.readDir(path);
+        for (const [filename, type] of files) {
+            const joined = `${path === '/' ? '' : path}/${filename}`;
+            if (type === FileType.Directory) {
+                yield* walk(fs, joined);
+            }
+            else {
+                yield joined;
+            }
+        }
+    }
+    Vfs.walk = walk;
+})(Vfs || (Vfs = {}));
+export class VolarFs {
+    #fs;
+    constructor(fs) {
+        this.#fs = fs;
+    }
+    async stat(uri) {
+        return this.#fs.stat(uri.path);
+    }
+    async readDirectory(uri) {
+        return this.#fs.readDir(uri.path);
+    }
+    async readFile(uri) {
+        return this.#fs.readFile(uri.path);
+    }
+}

package/dist/utils/indent.d.ts

@@ -0,0 +1 @@
+export declare const detectIndentationUnit: (content: string, limit?: number) => string | null;

package/dist/utils/indent.js

@@ -0,0 +1,38 @@
+export const detectIndentationUnit = (content, limit = 1024) => {
+    const lines = content.split(/\r?\n/, limit).filter(line => line.trim().length > 0);
+    const indentCounts = {};
+    let indentation = new Set();
+    let tabLineCount = 0;
+    let spaceLineCount = 0;
+    let unit = '\t';
+    for (const line of lines) {
+        const match = line.match(/^(\s+)/);
+        if (!match)
+            continue;
+        const indent = match[0];
+        const isTab = indent.includes('\t');
+        if (isTab) {
+            tabLineCount++;
+        }
+        else {
+            spaceLineCount++;
+        }
+        indentation.add(indent);
+        indentCounts[indent] = (indentCounts[indent] || 0) + 1;
+    }
+    if (Object.keys(indentCounts).length === 0) {
+        return null;
+    }
+    unit = tabLineCount > spaceLineCount ? '\t' : ' ';
+    const counts = Array.from(indentation)
+        .filter(indent => indent.startsWith(unit))
+        .map(indent => indent.length);
+    if (counts.length > 0) {
+        const gcd = (...numbers) => {
+            return numbers.reduce((a, b) => (b === 0 ? a : gcd(b, a % b)));
+        };
+        const commonUnit = gcd(...counts);
+        return unit.repeat(commonUnit);
+    }
+    return null;
+};

package/dist/utils/index.d.ts

@@ -0,0 +1,2 @@
+export * from './snapshot';
+export * from './indent';

package/dist/utils/index.js

@@ -0,0 +1,2 @@
+export * from './snapshot';
+export * from './indent';

package/dist/utils/lsp.d.ts

@@ -0,0 +1,26 @@
+import { VfsInterface } from "../types";
+import { LanguageServerClient } from "@marimo-team/codemirror-languageserver";
+import { Extension } from "@codemirror/state";
+import { LanguageServer } from "@volar/language-server";
+import { EditorView } from "@codemirror/view";
+export type LSPClientExtension = {
+    client: LanguageServerClient;
+} & Extension;
+export type ClientOptions = {
+    view: EditorView;
+    language: string;
+    path: string;
+    fs: VfsInterface;
+};
+export declare const languageServerFactory: Map<string, (args: {
+    fs: VfsInterface;
+}) => Promise<{
+    server: LanguageServer;
+}>>;
+export declare const lspWorkers: Map<string, SharedWorker>;
+export declare namespace LSP {
+    function worker(language: string, fs: VfsInterface): Promise<{
+        worker: SharedWorker;
+    }>;
+    function client({ fs, language, path, view }: ClientOptions): Promise<LSPClientExtension>;
+}

package/dist/utils/lsp.js

@@ -0,0 +1,74 @@
+import * as Comlink from 'comlink';
+import { LanguageServerClient, languageServerWithClient } from "@marimo-team/codemirror-languageserver";
+import MessagePortTransport from "../rpc/transport";
+import { HighlightStyle } from "@codemirror/language";
+import { languageSupportCompartment, renderMarkdownCode } from "../editor";
+import markdownit from 'markdown-it';
+import { vscodeLightDark } from "../themes/vscode";
+const clients = new Map();
+// TODO: better fix for this reference sticking around to prevent Comlink from releasing the port
+export const languageServerFactory = new Map();
+export const lspWorkers = new Map();
+export var LSP;
+(function (LSP) {
+    async function worker(language, fs) {
+        let factory, worker;
+        console.debug('language', { language });
+        switch (language) {
+            case 'javascript':
+            case 'typescript':
+                factory = languageServerFactory.get('javascript');
+                worker = lspWorkers.get('javascript');
+                console.debug('got worker', { worker, factory });
+                if (!factory) {
+                    worker = new SharedWorker(new URL('../workers/javascript.worker.js', import.meta.url), { type: 'module' });
+                    worker.port.start();
+                    lspWorkers.set('javascript', worker);
+                    const { createLanguageServer } = Comlink.wrap(worker.port);
+                    factory = createLanguageServer;
+                    languageServerFactory.set('javascript', factory);
+                }
+                break;
+        }
+        await factory?.(Comlink.proxy({ fs }));
+        return { worker };
+    }
+    LSP.worker = worker;
+    async function client({ fs, language, path, view }) {
+        let client = clients.get(language);
+        let clientExtension;
+        const uri = `file:///${path}`;
+        if (!client) {
+            const { worker } = await LSP.worker(language, fs);
+            if (!worker)
+                return null;
+            console.debug('got worker', { worker });
+            client = new LanguageServerClient({
+                transport: new MessagePortTransport(worker.port),
+                rootUri: 'file:///',
+                workspaceFolders: [{ name: 'workspace', uri: 'file:///' }]
+            });
+        }
+        clients.set(language, client);
+        clientExtension = { client, extension: [] };
+        clientExtension.extension = languageServerWithClient({
+            client: clientExtension.client,
+            documentUri: uri,
+            languageId: language,
+            allowHTMLContent: true,
+            markdownRenderer(markdown) {
+                const support = languageSupportCompartment.get(view.state);
+                const highlighter = vscodeLightDark[1].find(item => item.value instanceof HighlightStyle)?.value;
+                const parser = support.language?.parser;
+                const md = markdownit({
+                    highlight: (str) => {
+                        return renderMarkdownCode(str, parser, highlighter);
+                    }
+                });
+                return md.render(markdown);
+            },
+        });
+        return clientExtension;
+    }
+    LSP.client = client;
+})(LSP || (LSP = {}));

package/dist/utils/search.d.ts

@@ -0,0 +1,30 @@
+import { VfsInterface } from "../types";
+import MiniSearch, { Options, SearchResult } from 'minisearch';
+export declare const validFields: readonly ["path", "basename", "dirname", "extension"];
+export type IndexFields = typeof validFields[number][];
+export declare const defaultFields: IndexFields;
+export declare const defaultFilter: (path: string) => boolean;
+export type SearchIndexOptions = Options & {
+    filter?: (path: string) => boolean;
+};
+export type SearchHighlights = {
+    fields: Record<string, [number, number][]>;
+};
+export type HighlightedSearch = SearchResult & {
+    highlights: SearchHighlights;
+};
+export declare class SearchIndex {
+    index: MiniSearch;
+    constructor(index: MiniSearch);
+    search(...params: Parameters<MiniSearch['search']>): HighlightedSearch[];
+    /**
+     *
+     * @param results
+     * @returns ranges of found term matched by each field
+     */
+    highlight(_results: SearchResult[]): SearchHighlights[];
+    save(fs: VfsInterface, path: string): Promise<this>;
+    static from(fs: string, fields: IndexFields): SearchIndex;
+    static get(fs: VfsInterface, path: string, fields?: IndexFields): Promise<SearchIndex>;
+    static build(fs: VfsInterface, { filter, ...rest }: SearchIndexOptions): Promise<SearchIndex>;
+}

package/dist/utils/search.js

@@ -0,0 +1,68 @@
+import MiniSearch from 'minisearch';
+import { Vfs } from "./fs";
+export const validFields = ['path', 'basename', 'dirname', 'extension'];
+export const defaultFields = ['path', 'basename', 'dirname', 'extension'];
+export const defaultFilter = (path) => {
+    return !path.endsWith('.crswap');
+};
+export class SearchIndex {
+    index;
+    constructor(index) {
+        this.index = index;
+    }
+    search(...params) {
+        const results = this.index.search(...params);
+        const highlights = this.highlight(results);
+        return results.map((result, i) => ({
+            ...result,
+            highlights: highlights[i]
+        }));
+    }
+    /**
+     *
+     * @param results
+     * @returns ranges of found term matched by each field
+     */
+    highlight(_results) {
+        // TODO: implement
+        return [];
+    }
+    async save(fs, path) {
+        try {
+            // Extract directory from the file path and create it
+            const dir = path.substring(0, path.lastIndexOf('/'));
+            if (dir) {
+                await fs.mkdir(dir, { recursive: true });
+            }
+            await fs.writeFile(path, JSON.stringify(this.index));
+        }
+        catch (error) {
+            console.error('Failed to save search index:', error);
+        }
+        finally {
+            return this;
+        }
+    }
+    static from(fs, fields) {
+        const index = MiniSearch.loadJSON(fs, { fields, idField: 'path' });
+        return new SearchIndex(index);
+    }
+    static async get(fs, path, fields = defaultFields) {
+        const index = await fs.exists(path) ? await fs.readFile(path) : null;
+        return index ?
+            SearchIndex.from(index, fields) :
+            SearchIndex.build(fs, { fields, idField: 'path' }).then(index => index.save(fs, path));
+    }
+    static async build(fs, { filter = defaultFilter, ...rest }) {
+        const index = new MiniSearch({ ...rest });
+        for await (const path of Vfs.walk(fs, '/')) {
+            if (!filter(path)) {
+                continue;
+            }
+            if (!index.has(path.slice(1))) {
+                index.add({ path: path.slice(1) });
+            }
+        }
+        return new SearchIndex(index);
+    }
+}

package/dist/utils/snapshot.d.ts

@@ -0,0 +1,60 @@
+import fsPromises from 'fs/promises';
+import { SnapshotNode } from '@joinezco/memfs/snapshot';
+import { CborEncoder } from '@jsonjoy.com/json-pack/lib/cbor/CborEncoder';
+import { CborDecoder } from '@jsonjoy.com/json-pack/lib/cbor/CborDecoder';
+import { Writer } from '@jsonjoy.com/util/lib/buffers/Writer';
+import { CborUint8Array } from '@jsonjoy.com/json-pack/lib/cbor/types';
+import { FsApi } from '@joinezco/memfs/node/types';
+export declare const writer: Writer;
+export declare const encoder: CborEncoder<Writer>;
+export declare const decoder: CborDecoder<import("@jsonjoy.com/util/lib/buffers").IReader & import("@jsonjoy.com/util/lib/buffers").IReaderResettable>;
+/**
+ * Compress data using gzip compression.
+ * Uses Node.js zlib in Node.js environment, browser CompressionStream in browser.
+ */
+export declare const compress: (data: Uint8Array) => Promise<Uint8Array>;
+export declare const decompress: (data: Uint8Array) => Promise<Uint8Array>;
+export type BuildPathFilterArgs = {
+    include?: string[];
+    exclude?: string[];
+};
+export declare const buildFilter: ({ include, exclude }: BuildPathFilterArgs) => (path: string) => boolean;
+export type IgnoreArgs = {
+    fs: typeof fsPromises;
+    root: string;
+    exclude: string[];
+    gitignore: string | null;
+};
+export declare const getGitignored: (path: string, fs?: "string" | "number" | "bigint" | "boolean" | "symbol" | "undefined" | "object" | "function") => Promise<any>;
+export type TakeSnapshotProps = {
+    root: string;
+    filter: (path: string) => (Promise<boolean> | boolean);
+};
+export declare const snapshotDefaults: TakeSnapshotProps;
+/**
+ * Takes a snapshot of the file system based on the provided properties.
+ * The snapshot is encoded with CBOR and compressed with gzip.
+ *
+ * @param props - The properties to configure the snapshot.
+ */
+export declare const takeSnapshot: (props?: Partial<TakeSnapshotProps>) => Promise<Uint8Array<ArrayBufferLike>>;
+export type SnapshotOptions = {
+    fs: FsApi;
+    path?: string;
+    separator?: string;
+};
+export declare namespace Snapshot {
+    const take: ({ fs, path, filter, separator }: {
+        fs: typeof fsPromises;
+        path: TakeSnapshotProps["root"];
+        filter?: TakeSnapshotProps["filter"];
+        separator?: string;
+    }) => Promise<SnapshotNode>;
+    const mount: (buffer: CborUint8Array<SnapshotNode>, { fs, path, separator }: SnapshotOptions) => Promise<void>;
+    /**
+     * Load and mount a snapshot directly from a URL in a web worker environment.
+     * This is more efficient for large snapshots as it avoids transferring data through the main thread.
+     */
+    const loadAndMount: (url: string, { fs, path, separator }: SnapshotOptions) => Promise<void>;
+}
+export declare const fromSnapshot: (snapshot: SnapshotNode, { fs, path, separator }: SnapshotOptions) => Promise<void>;