@occultist/extensions 0.0.1

package/dist/static-extension.d.ts

@@ -0,0 +1,79 @@
+import { type StaticExtension as StaticExt, type Extension, type Cache, type HintLink, type ImplementedAction, type Registry, type StaticAsset } from '@occultist/occultist';
+import { DependancyGraph } from './dependancy-graph.ts';
+import { type FileInfo } from './file-info.ts';
+import type { Directory, ReferenceParser, ReferencePreprocessor } from './types.ts';
+export declare const defaultExtensions: {
+    readonly txt: "text/plain";
+    readonly html: "text/html";
+    readonly css: "text/css";
+    readonly xhtml: "application/xhtml+xml";
+    readonly xht: "application/xhtml+xml";
+    readonly js: "application/javascript";
+    readonly ts: "application/javascript";
+    readonly jpg: "image/jpeg";
+    readonly png: "image/png";
+    readonly woff: "font/woff";
+    readonly woff2: "font/woff2";
+};
+export declare const defaultParsers: ReferenceParser[];
+export declare const defaultPreprocessors: ReferencePreprocessor[];
+export declare const defaultCSPTypes: string[];
+export type StaticExtensionArgs = {
+    /**
+     * An occultist registry.
+     */
+    registry: Registry;
+    /**
+     * A cache instance
+     */
+    cache?: Cache;
+    /**
+     * Directories to serve as static content.
+     */
+    directories: Directory[];
+    /**
+     * A javascript object mapping file extensions
+     * to their content types.
+     */
+    extensions?: Record<string, string>;
+    /**
+     *
+     */
+    parsers?: ReferenceParser[];
+    /**
+     *
+     */
+    preprocessors?: ReferencePreprocessor[];
+    /**
+     * A path prefix where static assets should be served.
+     */
+    prefix?: string;
+};
+/**
+ * Serves a directory of files up as static assets using hashed urls
+ * and immutable cache headers.
+ *
+ * Other endpoints can use the hint method to register early hints linking
+ * to hashed actions.
+ */
+export declare class StaticExtension implements Extension, StaticExt {
+    #private;
+    name: string;
+    constructor(args: StaticExtensionArgs);
+    /**
+     * Promise the action cache for all static actions.
+     */
+    onAfterFinalize: () => Promise<void>;
+    get dependancies(): DependancyGraph | undefined;
+    get staticAliases(): string[];
+    get(name: string): ImplementedAction | undefined;
+    getFile(alias: string): FileInfo | undefined;
+    getAsset(alias: string): StaticAsset | undefined;
+    hint(name: string, args: Omit<HintLink, 'href' | 'contentType'>): HintLink | null;
+    /**
+     * Called when registered with Occultist to
+     * run any async tasks and hook into Occultist's
+     * extension event system.
+     */
+    setup: () => ReadableStream & Promise<void>;
+}

package/dist/static-extension.js

@@ -0,0 +1,273 @@
+import { joinPaths } from '@occultist/occultist';
+import { createHash } from "crypto";
+import { createReadStream } from "fs";
+import { opendir, readFile } from "fs/promises";
+import { join } from "path";
+import { Readable } from "stream";
+import { DependancyGraph, DependancyMap } from "./dependancy-graph.js";
+import { WorkingFileInfo } from "./file-info.js";
+import { CSSReferenceParser } from "./css-parser.js";
+import { JSReferenceParser } from "./js-parser.js";
+import { HTMLParser } from "./html-parser.js";
+import { TSReferencePreprocessor } from "./ts-preprocessor.js";
+export const defaultExtensions = {
+    txt: 'text/plain',
+    html: 'text/html',
+    css: 'text/css',
+    xhtml: 'application/xhtml+xml',
+    xht: 'application/xhtml+xml',
+    js: 'application/javascript',
+    ts: 'application/javascript',
+    jpg: 'image/jpeg',
+    png: 'image/png',
+    woff: 'font/woff',
+    woff2: 'font/woff2',
+};
+export const defaultParsers = [
+    new HTMLParser(),
+    new CSSReferenceParser(),
+    new JSReferenceParser(),
+];
+export const defaultPreprocessors = [
+    new TSReferencePreprocessor(),
+];
+export const defaultCSPTypes = [
+    'text/html',
+    'application/xhtml+xml',
+];
+/**
+ * Serves a directory of files up as static assets using hashed urls
+ * and immutable cache headers.
+ *
+ * Other endpoints can use the hint method to register early hints linking
+ * to hashed actions.
+ */
+export class StaticExtension {
+    name = 'static';
+    #loaded = false;
+    #registry;
+    #cache;
+    #directories;
+    #staticAliases = [];
+    #extensions;
+    #parsers = new Map();
+    #preprocessors = new Map();
+    #prefix;
+    #filesByAlias = new Map();
+    #filesByURL = new Map();
+    #filesByExtension = new Map();
+    #filesByContentType = new Map();
+    #hashes = new Map();
+    #actions = new Map();
+    #dependancies;
+    constructor(args) {
+        this.#registry = args.registry;
+        this.#cache = args.cache;
+        this.#directories = args.directories;
+        this.#extensions = new Map(Object.entries(args.extensions ?? defaultExtensions));
+        this.#prefix = args.prefix ?? '/';
+        this.#registry.registerExtension(this);
+        this.#registry.addEventListener('afterfinalize', this.onAfterFinalize);
+        for (let i = 0; i < args.directories.length; i++) {
+            this.#staticAliases.push(args.directories[i].alias);
+        }
+        Object.freeze(this.#staticAliases);
+        for (const parser of args.parsers ?? defaultParsers) {
+            for (const contentType of parser.supports.values()) {
+                this.#parsers.set(contentType, parser);
+            }
+        }
+        for (const preprocessor of args.preprocessors ?? defaultPreprocessors) {
+            for (const extension of preprocessor.supports.values()) {
+                this.#preprocessors.set(extension, preprocessor);
+            }
+        }
+    }
+    /**
+     * Promise the action cache for all static actions.
+     */
+    onAfterFinalize = async () => {
+        const promises = [];
+        for (const action of this.#actions.values()) {
+            promises.push(this.#registry.primeCache(new Request(action.url())));
+        }
+        await Promise.all(promises);
+    };
+    get dependancies() {
+        return this.#dependancies;
+    }
+    get staticAliases() {
+        return this.#staticAliases;
+    }
+    get(name) {
+        return this.#actions.get(name);
+    }
+    getFile(alias) {
+        return this.#filesByAlias.get(alias);
+    }
+    getAsset(alias) {
+        return this.#filesByAlias.get(alias);
+    }
+    hint(name, args) {
+        const file = this.#filesByAlias.get(name);
+        const action = this.#actions.get(name);
+        if (file == null || action == null)
+            return null;
+        const href = action.url();
+        if (href == null)
+            return null;
+        return {
+            ...args,
+            href,
+            type: file.contentType,
+        };
+    }
+    /**
+     * Called when registered with Occultist to
+     * run any async tasks and hook into Occultist's
+     * extension event system.
+     */
+    setup = () => {
+        if (this.#loaded)
+            throw new Error('Static extension already loaded');
+        const { writable, readable } = new TransformStream();
+        readable.then = async (resolve, reject) => {
+            try {
+                for await (const _ of readable) { }
+            }
+            catch (err) {
+                return reject(err);
+            }
+            resolve();
+        };
+        this.#load(writable.getWriter());
+        return readable;
+    };
+    async #load(writer) {
+        writer.write('Gathering files');
+        for await (const file of this.#traverse(this.#directories)) {
+            this.#filesByAlias.set(file.alias, file);
+        }
+        let file;
+        const files = Array.from(this.#filesByAlias.values());
+        writer.write('Generating hashes');
+        for (let i = 0; i < files.length; i++) {
+            file = files[i];
+            const content = await readFile(file.absolutePath);
+            const hash = createHash('sha1').update(content).digest('hex');
+            const parts = file.name.split('/');
+            const friendly = parts[parts.length - 1].split('.')[0];
+            const rootURL = this.#registry.rootIRI;
+            const aliasURL = joinPaths(rootURL, this.#prefix, file.alias);
+            const url = joinPaths(rootURL, this.#prefix, `${friendly}-${hash}.${file.extension}`);
+            file.finalize(hash, url, aliasURL);
+            this.#hashes.set(file.alias, hash);
+            this.#filesByAlias.set(file.alias, file);
+            this.#filesByURL.set(aliasURL, file);
+            if (!this.#filesByExtension.has(file.extension)) {
+                this.#filesByExtension.set(file.extension, []);
+            }
+            if (!this.#filesByContentType.has(file.contentType)) {
+                this.#filesByContentType.set(file.contentType, []);
+            }
+            this.#filesByExtension.get(file.extension).push(file);
+            this.#filesByContentType.get(file.contentType).push(file);
+        }
+        const dependancyMaps = [];
+        writer.write('Building dependancy tree');
+        for (const [extension, contentType] of this.#extensions.entries()) {
+            const preprocessor = this.#preprocessors.get(extension);
+            if (preprocessor != null) {
+                const files = this.#filesByExtension.get(extension);
+                if (files == null)
+                    continue;
+                const dependancies = new Map();
+                for (let i = 0; i < files.length; i++) {
+                    let file = files[i];
+                    const content = await readFile(file.absolutePath);
+                    const references = await preprocessor.parse(new Blob([content]), file, this.#filesByURL);
+                    dependancies.set(file.alias, new DependancyMap(file, references));
+                }
+                dependancyMaps.push(dependancies);
+                continue;
+            }
+            const parser = this.#parsers.get(contentType);
+            if (parser == null)
+                continue;
+            const files = this.#filesByContentType.get(contentType);
+            if (files == null)
+                continue;
+            const dependancies = new Map();
+            for (let i = 0; i < files.length; i++) {
+                let file = files[i];
+                const content = await readFile(file.absolutePath);
+                const references = await parser.parse(new Blob([content]), file, this.#filesByURL);
+                dependancies.set(file.alias, new DependancyMap(file, references));
+            }
+            dependancyMaps.push(dependancies);
+        }
+        this.#dependancies = new DependancyGraph(new Map(dependancyMaps.flatMap((map => Array.from(map.entries())))));
+        writer.write('Registering actions');
+        for (const [name, file] of this.#filesByAlias.entries()) {
+            const preprocessor = this.#preprocessors.get(file.extension);
+            const parser = this.#parsers.get(file.contentType);
+            const parts = name.split('/');
+            const friendly = parts[parts.length - 1].split('.')[0];
+            const hash = this.#hashes.get(name);
+            let action = this.#registry.http.get(name, joinPaths(this.#prefix, `${friendly}-${hash}.${file.extension}`))
+                .public();
+            if (this.#cache) {
+                action = action.cache(this.#cache.store({ immutable: true }));
+            }
+            const implemented = action.handle(file.contentType, async (ctx) => {
+                if (preprocessor != null) {
+                    const content = await readFile(file.absolutePath);
+                    ctx.body = await preprocessor.process(new Blob([content]), file, this.#filesByURL);
+                }
+                else if (parser != null) {
+                    const content = await readFile(file.absolutePath);
+                    ctx.body = await parser.update(new Blob([content]), file, this.#filesByURL);
+                }
+                else {
+                    ctx.body = Readable.toWeb(createReadStream(file.absolutePath));
+                }
+            });
+            this.#actions.set(name, implemented);
+        }
+        writer.write('Finished');
+        writer.close();
+        this.#loaded = true;
+    }
+    #traverseRe = /.(?:\.(?<lang>[a-zA-Z\-]+))?(?:\.(?<extension>[a-zA-Z0-9]+))$/;
+    /**
+     * Traverses into a list of directories outputting a file info object for every file
+     * of a configured file extension.
+     *
+     * @param directories The directories to traverse into.
+     */
+    async *#traverse(directories, root = '') {
+        for (let i = 0; i < directories.length; i++) {
+            const dir = await opendir(directories[i].path);
+            for await (const entry of dir) {
+                const name = entry.name;
+                const alias = directories[i].alias;
+                const match = this.#traverseRe.exec(entry.name);
+                const absolutePath = join(directories[i].path, entry.name);
+                const directory = root === '' ? directories[i].path : root;
+                const relativePath = absolutePath.replace(directory, '');
+                const { lang, extension } = match?.groups ?? {};
+                const contentType = this.#extensions.get(extension);
+                if (contentType == null || extension == null) {
+                    console.warn(`File ${joinPaths(alias, relativePath)} extension not known, skipping...`);
+                    continue;
+                }
+                if (entry.isDirectory()) {
+                    yield* this.#traverse([{ alias, path: absolutePath }], root === '' ? directories[i].path : root);
+                }
+                else {
+                    yield new WorkingFileInfo(name, alias, relativePath, absolutePath, extension, contentType, lang);
+                }
+            }
+        }
+    }
+}

package/dist/ts-preprocessor.d.ts

@@ -0,0 +1,8 @@
+import { type FileInfo } from './file-info.ts';
+import type { FilesByURL, ReferenceDetails, ReferencePreprocessor } from './types.ts';
+export declare class TSReferencePreprocessor implements ReferencePreprocessor {
+    supports: Set<string>;
+    readonly output: 'application/javascript';
+    parse(content: Blob, file: FileInfo, filesByURL: FilesByURL): Promise<ReferenceDetails[]>;
+    process(content: Blob, file: FileInfo, filesByURL: FilesByURL): Promise<Blob>;
+}

package/dist/ts-preprocessor.js

@@ -0,0 +1,69 @@
+import { createPrinter, createSourceFile, factory, forEachChild, isCallExpression, isImportDeclaration, isStringLiteral, NewLineKind, ScriptTarget, SyntaxKind, transform, visitEachChild, visitNode } from 'typescript';
+export class TSReferencePreprocessor {
+    supports = new Set(['ts', 'mts', 'cts']);
+    output;
+    async parse(content, file, filesByURL) {
+        const sourceText = await content.text();
+        const source = createSourceFile(file.absolutePath, sourceText, ScriptTarget.ES2022, true);
+        const references = [];
+        function visit(node) {
+            if (isImportDeclaration(node) && node.moduleSpecifier) {
+                const path = node.moduleSpecifier.text;
+                const url = new URL(path, file.aliasURL).toString();
+                references.push({
+                    url,
+                    directive: 'script-src',
+                    file: filesByURL.get(url),
+                });
+            }
+            else if (isCallExpression(node) &&
+                node.expression.kind === SyntaxKind.ImportKeyword &&
+                node.arguments.length === 1 &&
+                isStringLiteral(node.arguments[0])) {
+                const path = node.arguments[0].text;
+                const url = new URL(path, file.aliasURL).toString();
+                references.push({
+                    url,
+                    directive: 'script-src',
+                    file: filesByURL.get(url),
+                });
+            }
+            forEachChild(node, visit);
+        }
+        visit(source);
+        return references;
+    }
+    async process(content, file, filesByURL) {
+        const sourceText = await content.text();
+        const source = createSourceFile(file.absolutePath, sourceText, ScriptTarget.ES2022, true);
+        const transformerFactory = (context) => {
+            function visitor(node) {
+                if (isImportDeclaration(node) && node.moduleSpecifier) {
+                    const path = node.moduleSpecifier.text;
+                    const url = new URL(path, file.aliasURL).toString();
+                    const ref = filesByURL.get(url);
+                    const literal = factory.createStringLiteral(ref.url);
+                    return factory.updateImportDeclaration(node, node.modifiers, node.importClause, literal, node.attributes);
+                }
+                else if (isCallExpression(node) &&
+                    node.expression.kind === SyntaxKind.ImportKeyword &&
+                    node.arguments.length === 1 &&
+                    isStringLiteral(node.arguments[0])) {
+                    const path = node.arguments[0].text;
+                    const url = new URL(path, file.aliasURL).toString();
+                    const ref = filesByURL.get(url);
+                    const literal = factory.createStringLiteral(ref.url);
+                    return factory.updateCallExpression(node, node.expression, node.typeArguments, [literal]);
+                }
+                return visitEachChild(node, visitor, context);
+            }
+            return (node) => visitNode(node, visitor);
+        };
+        const result = transform(source, [transformerFactory]);
+        const printer = createPrinter({ newLine: NewLineKind.LineFeed });
+        const transformed = result.transformed[0];
+        const code = printer.printFile(transformed);
+        result.dispose();
+        return new Blob([code], { type: this.output });
+    }
+}

package/dist/types.d.ts

@@ -0,0 +1,68 @@
+import type { FileInfo } from "./file-info.ts";
+export type FilesByURL = Map<string, FileInfo>;
+export type PolicyDirective = 'child-src' | 'connect-src' | 'default-src' | 'fenced-frame-src' | 'font-src' | 'frame-src' | 'img-src' | 'manifest-src' | 'media-src' | 'object-src' | 'script-src' | 'script-src-elm' | 'script-src-attr' | 'style-src' | 'style-src-elm' | 'style-src-attr' | 'worker-src';
+export type Directory = {
+    alias: string;
+    path: string;
+};
+export type ReferenceDetails = {
+    url: string;
+    directive?: PolicyDirective;
+    file?: FileInfo;
+};
+/**
+ * An object with methods for parsing URL references
+ * within supporting content types and then embedding
+ * URLs generated for the referenced static assets by
+ * the framework.
+ */
+export interface ReferenceParser {
+    /**
+     * A set of content types supported by this reference parser.
+     */
+    readonly supports: Set<string>;
+    /**
+     * Parses URLs referenced in a file's content and
+     * returns a dependancy map of all references.
+     *
+     * @param content The file content to update.
+     * @param file File info object relating to the file.
+     * @returns A dependancy map of all references.
+     */
+    parse(content: Blob, file: FileInfo, filesByURL: FilesByURL): Promise<ReferenceDetails[]>;
+    /**
+     * Updates a file's embedded hyperlinks to point to
+     * final URLs of other static content.
+     *
+     * @param file File info object.
+     * @returns content with URLs updated.
+     */
+    update(content: Blob, file: FileInfo, filesByURL: FilesByURL): Promise<Blob>;
+}
+export interface ReferencePreprocessor {
+    /**
+     * A set of content types supported by this reference parser.
+     */
+    readonly supports: Set<string>;
+    /**
+     * Content type the pre-processor outputs.
+     */
+    readonly output: string;
+    /**
+     * Parses URLs referenced in a file's content and
+     * returns a dependancy map of all references.
+     *
+     * @param content The file content to update.
+     * @param file File info object relating to the file.
+     * @returns A dependancy map of all references.
+     */
+    parse(content: Blob, file: FileInfo, filesByURL: FilesByURL): Promise<ReferenceDetails[]>;
+    /**
+     * Updates a file's embedded hyperlinks to point to
+     * final URLs of other static content.
+     *
+     * @param file File info object.
+     * @returns content with URLs updated.
+     */
+    process(content: Blob, file: FileInfo, filesByURL: FilesByURL): Promise<Blob>;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/lib/static/css-parser.ts

@@ -0,0 +1,98 @@
+import type {FilesByURL, PolicyDirective, ReferenceDetails, ReferenceParser} from "./types.js";
+import {type FileInfo} from "./file-info.ts";
+
+
+type PolicyDirectiveMap = Record<string, PolicyDirective>;
+
+
+const ruleRe = /(?:(\@[a-z]+)|(?:([a-z][a-z\-]*\s*):))\s*(.*);/gm;
+const urlRe = /url\(\s*(?:(?:\"(.*)\")|(?:\'(.*)\')|(.*))\s*\)/gm;
+const defaultDirectives: PolicyDirectiveMap = {
+  '@import': 'style-src',
+  'background': 'img-src',
+  'background-image': 'img-src',
+  'src': 'font-src',
+} as const;
+
+export type CSSReferenceParserArgs = {
+  contentType?: string | string[];
+};
+
+export class CSSReferenceParser implements ReferenceParser {
+
+  ruleRe: RegExp = ruleRe;
+  urlRe: RegExp = urlRe;
+  supports = new Set(['text/css']);
+  directives: PolicyDirectiveMap = defaultDirectives;
+
+  constructor(args?: CSSReferenceParserArgs) {
+    if (Array.isArray(args?.contentType)) {
+      this.supports = new Set(args.contentType);
+    } else if (args?.contentType != null) {
+      this.supports = new Set([args.contentType]);
+    }
+  }
+
+  /**
+   * Parses all references within a css document.
+   *
+   * @param content Content of a css file.
+   */
+  async parse(
+    content: Blob,
+    file: FileInfo,
+    filesByURL: FilesByURL,
+  ): Promise<ReferenceDetails[]> {
+    let m1: RegExpExecArray | null;
+    let m2: RegExpExecArray | null;
+    let property: string;
+    let url: string;
+    let directive: PolicyDirective;
+    const references: ReferenceDetails[] = [];
+    const text = await content.text();
+
+    this.ruleRe.lastIndex = 0;
+    while ((m1 = this.ruleRe.exec(text))) {
+      property = m1[1] ?? m1[2];
+      directive = this.directives[property];
+
+      if (directive == null) continue;
+
+      this.urlRe.lastIndex = 0;
+      while ((m2 = this.urlRe.exec(m1[3]))) {
+        url = new URL(m2[1] ?? m2[2] ?? m2[3], file.aliasURL).toString();
+        
+        references.push({
+          url,
+          directive,
+          file: filesByURL.get(url),
+        });
+      }
+    }
+
+    return references;
+  }
+
+  async update(
+    content: Blob, 
+    file: FileInfo,
+    filesByURL: FilesByURL,
+  ): Promise<Blob> {
+    const text = await content.text();
+    const updated = text.replace(ruleRe, (match) => {
+      return match.replace(urlRe, (...matches) => {
+        const src = matches[1] ?? matches[2] ?? matches[3];
+        const url = new URL(src, file.aliasURL).toString();
+        const ref = filesByURL.get(url);
+
+        if (ref == null) return matches[0];
+
+        return `url(${ref.url})`;
+      })
+    })
+
+    return new Blob([updated], { type: file.contentType });
+  }
+}
+
+