@spyglassmc/core 0.4.34 → 0.4.35

package/lib/common/externals/BrowserExternals.js

@@ -35,24 +35,6 @@ export class BrowserEventEmitter {
         return this;
     }
 }
-class BrowserExternalDownloader {
-    async get(uri, options = {}) {
-        const headers = new Headers();
-        for (const [name, value] of Object.entries(options?.headers ?? {})) {
-            const values = typeof value === 'string' ? [value] : value;
-            for (const v of values) {
-                headers.append(name, v);
-            }
-        }
-        const res = await fetch(uri, { headers, redirect: 'follow' });
-        if (!res.ok) {
-            throw new Error(`Status code ${res.status}: ${res.ok}`);
-        }
-        else {
-            return new Uint8Array(await res.arrayBuffer());
-        }
-    }
-}
 class BrowserFsWatcher {
     on(event, listener) {
         if (event === 'ready') {
@@ -157,7 +139,6 @@ export const BrowserExternals = {
             return uint8ArrayToHex(new Uint8Array(hash));
         },
     },
-    downloader: new BrowserExternalDownloader(),
     error: {
         createKind(kind, message) {
             return new Error(`${kind}: ${message}`);
@@ -168,6 +149,10 @@ export const BrowserExternals = {
     },
     event: { EventEmitter: BrowserEventEmitter },
     fs: new BrowserFileSystem(),
+    web: {
+        fetch,
+        getCache: () => window.caches.open('spyglassmc'),
+    },
 };
 function uint8ArrayToHex(array) {
     let ans = '';

package/lib/common/externals/NodeJsExternals.d.ts

@@ -1,3 +1,116 @@
-import type { Externals } from './index.js';
-export declare const NodeJsExternals: Externals;
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+import chokidar from 'chokidar';
+import decompress from 'decompress';
+import { Buffer } from 'node:buffer';
+import { EventEmitter } from 'node:events';
+import fs from 'node:fs';
+import type { RootUriString } from '../../index.js';
+import type { FsLocation, FsWatcher } from './index.js';
+export declare function getNodeJsExternals({ cacheRoot }?: {
+    cacheRoot?: RootUriString;
+}): Readonly<{
+    archive: {
+        decompressBall(buffer: Uint8Array, options: {
+            stripLevel?: number | undefined;
+        } | undefined): Promise<decompress.File[]>;
+        gunzip(buffer: Uint8Array): Promise<Buffer>;
+        gzip(buffer: Uint8Array): Promise<Buffer>;
+    };
+    crypto: {
+        getSha1(data: string | Uint8Array): Promise<string>;
+    };
+    error: {
+        createKind(kind: import("./index.js").ExternalErrorKind, message: string): Error;
+        isKind(e: unknown, kind: import("./index.js").ExternalErrorKind): boolean;
+    };
+    event: {
+        EventEmitter: typeof EventEmitter;
+    };
+    fs: {
+        chmod(location: FsLocation, mode: number): Promise<void>;
+        mkdir(location: FsLocation, options: {
+            mode?: number | undefined;
+            recursive?: boolean | undefined;
+        } | undefined): Promise<undefined>;
+        readdir(location: FsLocation): Promise<fs.Dirent[]>;
+        readFile(location: FsLocation): Promise<Buffer>;
+        showFile(location: FsLocation): Promise<void>;
+        stat(location: FsLocation): Promise<fs.Stats>;
+        unlink(location: FsLocation): Promise<void>;
+        watch(locations: FsLocation[], { usePolling }?: {
+            usePolling?: boolean | undefined;
+        }): ChokidarWatcherWrapper;
+        writeFile(location: FsLocation, data: string | Uint8Array, options: {
+            mode: number;
+        } | undefined): Promise<void>;
+    };
+    web: {
+        fetch: typeof fetch;
+        getCache: () => Promise<HttpCache>;
+    };
+}>;
+export declare const NodeJsExternals: Readonly<{
+    archive: {
+        decompressBall(buffer: Uint8Array, options: {
+            stripLevel?: number | undefined;
+        } | undefined): Promise<decompress.File[]>;
+        gunzip(buffer: Uint8Array): Promise<Buffer>;
+        gzip(buffer: Uint8Array): Promise<Buffer>;
+    };
+    crypto: {
+        getSha1(data: string | Uint8Array): Promise<string>;
+    };
+    error: {
+        createKind(kind: import("./index.js").ExternalErrorKind, message: string): Error;
+        isKind(e: unknown, kind: import("./index.js").ExternalErrorKind): boolean;
+    };
+    event: {
+        EventEmitter: typeof EventEmitter;
+    };
+    fs: {
+        chmod(location: FsLocation, mode: number): Promise<void>;
+        mkdir(location: FsLocation, options: {
+            mode?: number | undefined;
+            recursive?: boolean | undefined;
+        } | undefined): Promise<undefined>;
+        readdir(location: FsLocation): Promise<fs.Dirent[]>;
+        readFile(location: FsLocation): Promise<Buffer>;
+        showFile(location: FsLocation): Promise<void>;
+        stat(location: FsLocation): Promise<fs.Stats>;
+        unlink(location: FsLocation): Promise<void>;
+        watch(locations: FsLocation[], { usePolling }?: {
+            usePolling?: boolean | undefined;
+        }): ChokidarWatcherWrapper;
+        writeFile(location: FsLocation, data: string | Uint8Array, options: {
+            mode: number;
+        } | undefined): Promise<void>;
+    };
+    web: {
+        fetch: typeof fetch;
+        getCache: () => Promise<HttpCache>;
+    };
+}>;
+declare class ChokidarWatcherWrapper extends EventEmitter implements FsWatcher {
+    #private;
+    constructor(watcher: chokidar.FSWatcher);
+    close(): Promise<void>;
+}
+/**
+ * A non-spec-compliant, non-complete implementation of the Cache Web API for use in Spyglass.
+ * This class stores the cached response on the file system under the cache root.
+ */
+declare class HttpCache implements Cache {
+    #private;
+    constructor(cacheRoot: RootUriString | undefined);
+    match(request: RequestInfo | URL, _options?: CacheQueryOptions | undefined): Promise<Response | undefined>;
+    put(request: RequestInfo | URL, response: Response): Promise<void>;
+    add(): Promise<void>;
+    addAll(): Promise<void>;
+    delete(): Promise<boolean>;
+    keys(): Promise<readonly Request[]>;
+    matchAll(): Promise<readonly Response[]>;
+}
+export {};
 //# sourceMappingURL=NodeJsExternals.d.ts.map

package/lib/common/externals/NodeJsExternals.js

@@ -1,119 +1,111 @@
 // https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/60592
 import chokidar from 'chokidar';
 import decompress from 'decompress';
-import followRedirects from 'follow-redirects';
 import { Buffer } from 'node:buffer';
 import cp from 'node:child_process';
 import crypto from 'node:crypto';
 import { EventEmitter } from 'node:events';
-import { promises as fsp } from 'node:fs';
+import fs, { promises as fsp } from 'node:fs';
+import os from 'node:os';
 import process from 'node:process';
+import stream from 'node:stream';
 import url from 'node:url';
 import { promisify } from 'node:util';
 import zlib from 'node:zlib';
-import { promisifyAsyncIterable, Uri } from '../util.js';
-const { http, https } = followRedirects;
+import { Uri } from '../util.js';
 const gunzip = promisify(zlib.gunzip);
 const gzip = promisify(zlib.gzip);
-class NodeJsExternalDownloader {
-    get(uri, options = {}) {
-        const protocol = new Uri(uri).protocol;
-        return new Promise((resolve, reject) => {
-            const backend = protocol === 'http:' ? http : https;
-            backend.get(uri, options, (res) => {
-                if (res.statusCode !== 200) {
-                    reject(new Error(`Status code ${res.statusCode}: ${res.statusMessage}`));
+export function getNodeJsExternals({ cacheRoot } = {}) {
+    return Object.freeze({
+        archive: {
+            decompressBall(buffer, options) {
+                if (!(buffer instanceof Buffer)) {
+                    buffer = Buffer.from(buffer);
                 }
-                else {
-                    resolve(promisifyAsyncIterable(res, (chunks) => Buffer.concat(chunks)));
-                }
-            }).on('error', (e) => {
-                reject(e);
-            });
-        });
-    }
-}
-export const NodeJsExternals = {
-    archive: {
-        decompressBall(buffer, options) {
-            if (buffer instanceof Buffer) {
                 return decompress(buffer, { strip: options?.stripLevel });
-            }
-            throw new TypeError(`The 'buffer' argument for 'decompressBall' on Node.js must be an instance of 'Buffer'. Got '${buffer}' instead.`);
-        },
-        gunzip(buffer) {
-            return gunzip(buffer);
-        },
-        gzip(buffer) {
-            return gzip(buffer);
-        },
-    },
-    crypto: {
-        async getSha1(data) {
-            const hash = crypto.createHash('sha1');
-            hash.update(data);
-            return hash.digest('hex');
-        },
-    },
-    downloader: new NodeJsExternalDownloader(),
-    error: {
-        createKind(kind, message) {
-            const error = new Error(message);
-            error.code = kind;
-            return error;
-        },
-        isKind(e, kind) {
-            return e instanceof Error && e.code === kind;
-        },
-    },
-    event: { EventEmitter },
-    fs: {
-        chmod(location, mode) {
-            return fsp.chmod(toFsPathLike(location), mode);
-        },
-        async mkdir(location, options) {
-            return void (await fsp.mkdir(toFsPathLike(location), options));
-        },
-        readdir(location) {
-            return fsp.readdir(toFsPathLike(location), { encoding: 'utf-8', withFileTypes: true });
-        },
-        readFile(location) {
-            return fsp.readFile(toFsPathLike(location));
-        },
-        async showFile(location) {
-            const execFile = promisify(cp.execFile);
-            let command;
-            switch (process.platform) {
-                case 'darwin':
-                    command = 'open';
-                    break;
-                case 'win32':
-                    command = 'explorer';
-                    break;
-                default:
-                    command = 'xdg-open';
-                    break;
-            }
-            return void (await execFile(command, [toPath(location)]));
+            },
+            gunzip(buffer) {
+                return gunzip(buffer);
+            },
+            gzip(buffer) {
+                return gzip(buffer);
+            },
         },
-        stat(location) {
-            return fsp.stat(toFsPathLike(location));
+        crypto: {
+            async getSha1(data) {
+                const hash = crypto.createHash('sha1');
+                hash.update(data);
+                return hash.digest('hex');
+            },
         },
-        unlink(location) {
-            return fsp.unlink(toFsPathLike(location));
+        error: {
+            createKind(kind, message) {
+                const error = new Error(message);
+                error.code = kind;
+                return error;
+            },
+            isKind(e, kind) {
+                return e instanceof Error && e.code === kind;
+            },
         },
-        watch(locations, { usePolling = false } = {}) {
-            return new ChokidarWatcherWrapper(chokidar.watch(locations.map(toPath), {
-                usePolling,
-                disableGlobbing: true,
-            }));
+        event: { EventEmitter },
+        fs: {
+            chmod(location, mode) {
+                return fsp.chmod(toFsPathLike(location), mode);
+            },
+            async mkdir(location, options) {
+                return void (await fsp.mkdir(toFsPathLike(location), options));
+            },
+            readdir(location) {
+                return fsp.readdir(toFsPathLike(location), {
+                    encoding: 'utf-8',
+                    withFileTypes: true,
+                });
+            },
+            readFile(location) {
+                return fsp.readFile(toFsPathLike(location));
+            },
+            async showFile(location) {
+                const execFile = promisify(cp.execFile);
+                let command;
+                switch (process.platform) {
+                    case 'darwin':
+                        command = 'open';
+                        break;
+                    case 'win32':
+                        command = 'explorer';
+                        break;
+                    default:
+                        command = 'xdg-open';
+                        break;
+                }
+                return void (await execFile(command, [toPath(location)]));
+            },
+            stat(location) {
+                return fsp.stat(toFsPathLike(location));
+            },
+            unlink(location) {
+                return fsp.unlink(toFsPathLike(location));
+            },
+            watch(locations, { usePolling = false } = {}) {
+                return new ChokidarWatcherWrapper(chokidar.watch(locations.map(toPath), {
+                    usePolling,
+                    disableGlobbing: true,
+                }));
+            },
+            writeFile(location, data, options) {
+                return fsp.writeFile(toFsPathLike(location), data, options);
+            },
         },
-        writeFile(location, data, options) {
-            return fsp.writeFile(toFsPathLike(location), data, options);
+        web: {
+            fetch,
+            getCache: async () => {
+                return new HttpCache(cacheRoot);
+            },
         },
-    },
-};
-Object.freeze(NodeJsExternals);
+    });
+}
+export const NodeJsExternals = getNodeJsExternals();
 /**
  * @returns A {@link fs.PathLike}.
  */
@@ -151,4 +143,70 @@ class ChokidarWatcherWrapper extends EventEmitter {
         return this.#watcher.close();
     }
 }
+/**
+ * A non-spec-compliant, non-complete implementation of the Cache Web API for use in Spyglass.
+ * This class stores the cached response on the file system under the cache root.
+ */
+class HttpCache {
+    #cacheRoot;
+    constructor(cacheRoot) {
+        if (cacheRoot) {
+            this.#cacheRoot = `${cacheRoot}http/`;
+        }
+    }
+    async match(request, _options) {
+        if (!this.#cacheRoot) {
+            return undefined;
+        }
+        const fileName = this.#getFileName(request);
+        try {
+            const etag = (await fsp.readFile(new URL(`${fileName}.etag`, this.#cacheRoot), 'utf8'))
+                .trim();
+            const bodyStream = fs.createReadStream(new URL(`${fileName}.bin`, this.#cacheRoot));
+            return new Response(stream.Readable.toWeb(bodyStream), 
+            //              \___/
+            // stream Readable -> stream/web ReadableStream
+            //                                \_______________/
+            //                 stream/web ReadableStream -> DOM ReadableStream
+            { headers: { etag } });
+        }
+        catch (e) {
+            if (e?.code === 'ENOENT') {
+                return undefined;
+            }
+            throw e;
+        }
+    }
+    async put(request, response) {
+        const etag = response.headers.get('etag');
+        if (!(this.#cacheRoot && response.body && etag)) {
+            return;
+        }
+        const fileName = this.#getFileName(request);
+        await fsp.mkdir(new URL(this.#cacheRoot), { recursive: true });
+        await Promise.all([
+            fsp.writeFile(new URL(`${fileName}.bin`, this.#cacheRoot), stream.Readable.fromWeb(response.body)),
+            fsp.writeFile(new URL(`${fileName}.etag`, this.#cacheRoot), `${etag}${os.EOL}`),
+        ]);
+    }
+    #getFileName(request) {
+        const uriString = request instanceof Request ? request.url : request.toString();
+        return Buffer.from(uriString, 'utf8').toString('base64url');
+    }
+    async add() {
+        throw new Error('Method not implemented.');
+    }
+    async addAll() {
+        throw new Error('Method not implemented.');
+    }
+    async delete() {
+        throw new Error('Method not implemented.');
+    }
+    async keys() {
+        throw new Error('Method not implemented.');
+    }
+    async matchAll() {
+        throw new Error('Method not implemented.');
+    }
+}
 //# sourceMappingURL=NodeJsExternals.js.map

package/lib/common/externals/index.d.ts

@@ -1,6 +1,4 @@
 import type { Uri } from '../util.js';
-import type { ExternalDownloader } from './downloader.js';
-export * from './downloader.js';
 export interface Externals {
     archive: {
         decompressBall: (buffer: Uint8Array, options?: {
@@ -15,7 +13,6 @@ export interface Externals {
          */
         getSha1: (data: string | Uint8Array) => Promise<string>;
     };
-    downloader: ExternalDownloader;
     error: {
         /**
          * @returns an error of the specified kind
@@ -30,6 +27,10 @@ export interface Externals {
         EventEmitter: new () => ExternalEventEmitter;
     };
     fs: ExternalFileSystem;
+    web: {
+        fetch: typeof fetch;
+        getCache: () => Promise<Cache>;
+    };
 }
 export interface DecompressedFile {
     data: Uint8Array;

package/lib/common/externals/index.js

@@ -1,2 +1,2 @@
-export * from './downloader.js';
+export {};
 //# sourceMappingURL=index.js.map

package/lib/service/Dependency.d.ts

@@ -1,6 +1,15 @@
 export type Dependency = {
+    type: 'directory';
     uri: string;
-    info?: Record<string, any>;
+} | {
+    type: 'tarball-file';
+    uri: string;
+    stripLevel?: number;
+} | {
+    type: 'tarball-ram';
+    name: string;
+    data: Uint8Array;
+    stripLevel?: number;
 };
 export type DependencyKey = `@${string}`;
 export declare namespace DependencyKey {

package/lib/service/FileService.js

@@ -137,7 +137,11 @@ export class FileUriSupporter {
     static async create(dependencies, externals, logger) {
         const roots = [];
         const files = new Map();
-        for (let { uri } of dependencies) {
+        for (const dependency of dependencies) {
+            if (dependency.type !== 'directory') {
+                continue;
+            }
+            let { uri } = dependency;
             try {
                 if (fileUtil.isFileUri(uri) && (await externals.fs.stat(uri)).isDirectory()) {
                     uri = fileUtil.ensureEndingSlash(uri);
@@ -232,23 +236,26 @@ export class ArchiveUriSupporter {
     }
     static async create(dependencies, externals, logger) {
         const entries = new Map();
-        for (const { uri, info } of dependencies) {
+        for (const dependency of dependencies) {
+            if (dependency.type === 'directory') {
+                continue;
+            }
+            const archiveName = dependency.type === 'tarball-file'
+                ? fileUtil.basename(dependency.uri)
+                : dependency.name;
             try {
-                if (uri.startsWith('file:')
-                    && ArchiveUriSupporter.SupportedArchiveExtnames.some((ext) => uri.endsWith(ext))
-                    && (await externals.fs.stat(uri)).isFile()) {
-                    const archiveName = fileUtil.basename(uri);
-                    if (entries.has(archiveName)) {
-                        throw new Error(`A different URI with ${archiveName} already exists`);
-                    }
-                    const files = await externals.archive.decompressBall(await externals.fs.readFile(uri), { stripLevel: typeof info?.startDepth === 'number' ? info.startDepth : 0 });
-                    /// Debug message for #1609
-                    logger.info(`[ArchiveUriSupporter#create] Extracted ${files.length} files from ${archiveName}`);
-                    entries.set(archiveName, new Map(files.map((f) => [f.path.replace(/\\/g, '/'), f])));
+                if (entries.has(archiveName)) {
+                    throw new Error(`A different archive with ${archiveName} already exists`);
                 }
+                const files = await externals.archive.decompressBall(dependency.type === 'tarball-file'
+                    ? await externals.fs.readFile(dependency.uri)
+                    : dependency.data, { stripLevel: dependency.stripLevel ?? 0 });
+                /// Debug message for #1609
+                logger.info(`[ArchiveUriSupporter#create] Extracted ${files.length} files from ${archiveName}`);
+                entries.set(archiveName, new Map(files.map((f) => [f.path.replace(/\\/g, '/'), f])));
             }
             catch (e) {
-                logger.error(`[ArchiveUriSupporter#create] Bad dependency ${uri}`, e);
+                logger.error(`[ArchiveUriSupporter#create] Bad dependency ${archiveName}`, e);
             }
         }
         return new ArchiveUriSupporter(externals, logger, entries);

package/lib/service/Project.d.ts

@@ -8,19 +8,17 @@ import type { PosRangeLanguageError } from '../source/index.js';
 import { SymbolUtil } from '../symbol/index.js';
 import { CacheService } from './CacheService.js';
 import type { Config } from './Config.js';
-import { Downloader } from './Downloader.js';
 import { FileService } from './FileService.js';
 import type { RootUriString } from './fileUtil.js';
 import { MetaRegistry } from './MetaRegistry.js';
 import { ProfilerFactory } from './Profiler.js';
-export type ProjectInitializerContext = Pick<Project, 'cacheRoot' | 'config' | 'downloader' | 'externals' | 'isDebugging' | 'logger' | 'meta' | 'projectRoots'>;
+export type ProjectInitializerContext = Pick<Project, 'cacheRoot' | 'config' | 'externals' | 'isDebugging' | 'logger' | 'meta' | 'projectRoots'>;
 export type SyncProjectInitializer = (this: void, ctx: ProjectInitializerContext) => Record<string, string> | void;
 export type AsyncProjectInitializer = (this: void, ctx: ProjectInitializerContext) => PromiseLike<Record<string, string> | void>;
 export type ProjectInitializer = SyncProjectInitializer | AsyncProjectInitializer;
 export interface ProjectOptions {
     cacheRoot: RootUriString;
     defaultConfig?: Config;
-    downloader?: Downloader;
     externals: Externals;
     fs?: FileService;
     initializers?: readonly ProjectInitializer[];
@@ -56,7 +54,7 @@ interface SymbolRegistrarEvent {
     id: string;
     checksum: string | undefined;
 }
-export type ProjectData = Pick<Project, 'cacheRoot' | 'config' | 'downloader' | 'ensureBindingStarted' | 'externals' | 'fs' | 'isDebugging' | 'logger' | 'meta' | 'profilers' | 'projectRoots' | 'roots' | 'symbols' | 'ctx'>;
+export type ProjectData = Pick<Project, 'cacheRoot' | 'config' | 'ensureBindingStarted' | 'externals' | 'fs' | 'isDebugging' | 'logger' | 'meta' | 'profilers' | 'projectRoots' | 'roots' | 'symbols' | 'ctx'>;
 /**
  * Manage all tracked documents and errors.
  *
@@ -100,7 +98,6 @@ export declare class Project implements ExternalEventEmitter {
     readonly cacheService: CacheService;
     get isReady(): boolean;
     config: Config;
-    readonly downloader: Downloader;
     readonly externals: Externals;
     readonly fs: FileService;
     readonly isDebugging: boolean;
@@ -156,7 +153,7 @@ export declare class Project implements ExternalEventEmitter {
      * are not loaded into the memory.
      */
     getTrackedFiles(): string[];
-    constructor({ cacheRoot, defaultConfig, downloader, externals, fs, initializers, isDebugging, logger, profilers, projectRoots, }: ProjectOptions);
+    constructor({ cacheRoot, defaultConfig, externals, fs, initializers, isDebugging, logger, profilers, projectRoots, }: ProjectOptions);
     private setInitPromise;
     private setReadyPromise;
     /**