@johnhenry/sitefile 0.0.0-stage → 0.0.0

package/lib/src/resources/https.d.ts

@@ -0,0 +1,40 @@
+import { type LookupFunction } from 'node:net';
+import { type SourceSpan } from '../errors.js';
+import type { FetchPolicy } from './types.js';
+export interface FetchResult {
+    readonly bytes: Uint8Array;
+    /** URL of the final hop after redirects. */
+    readonly finalUrl: string;
+    /** Media type essence from `content-type` (lowercase, parameters removed). */
+    readonly mediaType?: string;
+}
+/**
+ * What to send. `headers` go to every hop; `originHeaders` (credentials: IMPORT … SECRET) only to hops
+ * on the first URL's origin (scheme, host and port), so a redirect elsewhere never receives them.
+ * A request with a body (POST) never follows redirects (E_REDIRECT).
+ */
+export interface FetchRequest {
+    readonly method?: 'GET' | 'POST';
+    readonly body?: Uint8Array;
+    readonly headers?: Readonly<Record<string, string>>;
+    readonly originHeaders?: Readonly<Record<string, string>>;
+}
+/** A URL rendered without any userinfo, safe for error messages. */
+export declare function redactUrl(url: URL | string): string;
+/**
+ * Validate one hop before connecting: scheme, credentials and IP-literal
+ * hosts. Hostnames are checked at connect time by the guarded lookup.
+ */
+export declare function validateHop(url: URL, policy: FetchPolicy, span?: SourceSpan): void;
+/**
+ * A `lookup` for http(s).request that resolves every address and fails with
+ * E_SSRF_BLOCKED if any is non-public. Runs at connect time, so DNS rebinding
+ * between validation and connection cannot bypass it.
+ */
+export declare function createGuardedLookup(policy: FetchPolicy, span?: SourceSpan): LookupFunction;
+/**
+ * Fetch a remote resource under the given network policy. Redirects are
+ * followed manually and every hop is re-validated; one deadline covers the
+ * whole fetch.
+ */
+export declare function fetchRemote(url: string, policy: FetchPolicy, span?: SourceSpan, request?: FetchRequest): Promise<FetchResult>;

package/lib/src/resources/https.js

@@ -0,0 +1,256 @@
+import { lookup as dnsLookup } from 'node:dns';
+import { request as httpRequest } from 'node:http';
+import { request as httpsRequest } from 'node:https';
+import * as tls from 'node:tls';
+import { isIP } from 'node:net';
+import { SitefileError, isSitefileError } from '../errors.js';
+import { isPublicAddress } from './netpolicy.js';
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const USER_AGENT = 'sitefile/0.1';
+/** A URL rendered without any userinfo, safe for error messages. */
+export function redactUrl(url) {
+    try {
+        const u = new URL(String(url));
+        u.username = '';
+        u.password = '';
+        return u.href;
+    }
+    catch {
+        return '<invalid url>';
+    }
+}
+function stripBrackets(host) {
+    return host.startsWith('[') && host.endsWith(']') ? host.slice(1, -1) : host;
+}
+/**
+ * Validate one hop before connecting: scheme, credentials and IP-literal
+ * hosts. Hostnames are checked at connect time by the guarded lookup.
+ */
+export function validateHop(url, policy, span) {
+    if (url.protocol !== 'https:' && !(url.protocol === 'http:' && policy.allowHttp)) {
+        const hint = url.protocol === 'http:' ? ' (plain http requires --allow-http)' : '';
+        throw new SitefileError('E_PROTOCOL', `unsupported protocol '${url.protocol}' for ${redactUrl(url)}${hint}`, span);
+    }
+    if (url.username !== '' || url.password !== '') {
+        throw new SitefileError('E_PROTOCOL', `URL ${redactUrl(url)} contains credentials; credentials in URLs are not allowed`, span);
+    }
+    const host = stripBrackets(url.hostname);
+    if (host === '')
+        throw new SitefileError('E_PROTOCOL', `URL ${redactUrl(url)} has no host`, span);
+    if (isIP(host) !== 0 && !policy.allowPrivateNetwork && !isPublicAddress(host)) {
+        throw new SitefileError('E_SSRF_BLOCKED', `refusing to connect to non-public address ${host} (${redactUrl(url)}); use --allow-private-network to permit it`, span);
+    }
+}
+/**
+ * A `lookup` for http(s).request that resolves every address and fails with
+ * E_SSRF_BLOCKED if any is non-public. Runs at connect time, so DNS rebinding
+ * between validation and connection cannot bypass it.
+ */
+export function createGuardedLookup(policy, span) {
+    return (hostname, options, callback) => {
+        dnsLookup(hostname, { ...options, all: true }, (err, addresses) => {
+            if (err) {
+                callback(err);
+                return;
+            }
+            const list = addresses;
+            if (list.length === 0) {
+                callback(new SitefileError('E_FETCH', `no addresses found for ${hostname}`, span));
+                return;
+            }
+            if (!policy.allowPrivateNetwork) {
+                const bad = list.find((a) => !isPublicAddress(a.address));
+                if (bad) {
+                    callback(new SitefileError('E_SSRF_BLOCKED', `refusing to connect to ${hostname}: it resolves to non-public address ${bad.address}; use --allow-private-network to permit it`, span));
+                    return;
+                }
+            }
+            if (options.all) {
+                callback(null, list);
+            }
+            else {
+                const first = list[0];
+                callback(null, first.address, first.family);
+            }
+        });
+    };
+}
+/** Find a SitefileError carried by a socket/request error, if any. */
+function findSitefileError(e, depth = 0) {
+    if (depth > 4 || e === null || typeof e !== 'object')
+        return undefined;
+    if (isSitefileError(e))
+        return e;
+    const anyE = e;
+    if (Array.isArray(anyE.errors)) {
+        for (const inner of anyE.errors) {
+            const found = findSitefileError(inner, depth + 1);
+            if (found)
+                return found;
+        }
+    }
+    return findSitefileError(anyE.cause, depth + 1);
+}
+/**
+ * Fetch a remote resource under the given network policy. Redirects are
+ * followed manually and every hop is re-validated; one deadline covers the
+ * whole fetch.
+ */
+export async function fetchRemote(url, policy, span, request = {}) {
+    let current;
+    try {
+        current = new URL(url);
+    }
+    catch (e) {
+        throw new SitefileError('E_PROTOCOL', `invalid URL '${redactUrl(url)}'`, span, { cause: e });
+    }
+    const origin = current.origin;
+    const method = request.method ?? 'GET';
+    const controller = new AbortController();
+    const state = {};
+    const timer = setTimeout(() => {
+        controller.abort();
+        state.req?.destroy();
+        state.res?.destroy();
+    }, policy.timeoutMs);
+    timer.unref?.();
+    const timeoutError = () => new SitefileError('E_FETCH_TIMEOUT', `fetching ${redactUrl(url)} timed out after ${policy.timeoutMs} ms`, span);
+    try {
+        let redirects = 0;
+        for (;;) {
+            validateHop(current, policy, span);
+            if (controller.signal.aborted)
+                throw timeoutError();
+            // Credentials only ever go to the origin they were given for (not after a redirect elsewhere).
+            const headers = current.origin === origin ? { ...request.headers, ...request.originHeaders } : { ...request.headers };
+            const res = await openHop(current, policy, controller.signal, state, span, timeoutError, method, headers, request.body);
+            const status = res.statusCode ?? 0;
+            if (REDIRECT_STATUSES.has(status) && method !== 'GET') {
+                res.destroy();
+                throw new SitefileError('E_REDIRECT', `${method} ${redactUrl(current)} was redirected (HTTP ${status}); redirects are only followed for GET`, span);
+            }
+            if (REDIRECT_STATUSES.has(status)) {
+                const location = res.headers.location;
+                res.destroy();
+                state.req?.destroy();
+                if (!location) {
+                    throw new SitefileError('E_REDIRECT', `redirect ${status} from ${redactUrl(current)} has no Location header`, span);
+                }
+                redirects++;
+                if (redirects > policy.maxRedirects) {
+                    throw new SitefileError('E_REDIRECT', `too many redirects fetching ${redactUrl(url)} (limit ${policy.maxRedirects})`, span);
+                }
+                let next;
+                try {
+                    next = new URL(location, current);
+                }
+                catch (e) {
+                    throw new SitefileError('E_REDIRECT', `invalid redirect Location from ${redactUrl(current)}`, span, { cause: e });
+                }
+                current = next;
+                continue;
+            }
+            if (status !== 200) {
+                res.destroy();
+                throw new SitefileError('E_FETCH_STATUS', `${method} ${redactUrl(current)} returned HTTP ${status}`, span);
+            }
+            const encoding = String(res.headers['content-encoding'] ?? '').trim().toLowerCase();
+            if (encoding !== '' && encoding !== 'identity') {
+                res.destroy();
+                throw new SitefileError('E_FETCH', `${redactUrl(current)} responded with content-encoding '${encoding}'; only identity is accepted`, span);
+            }
+            const lengthHeader = res.headers['content-length'];
+            if (lengthHeader !== undefined) {
+                const declared = Number(lengthHeader);
+                if (Number.isFinite(declared) && declared > policy.maxBytes) {
+                    res.destroy();
+                    throw new SitefileError('E_TOO_LARGE', `${redactUrl(current)} is ${declared} bytes, exceeding the limit of ${policy.maxBytes} bytes`, span);
+                }
+            }
+            const chunks = [];
+            let total = 0;
+            try {
+                for await (const chunk of res) {
+                    total += chunk.length;
+                    if (total > policy.maxBytes) {
+                        res.destroy();
+                        throw new SitefileError('E_TOO_LARGE', `${redactUrl(current)} exceeded the limit of ${policy.maxBytes} bytes`, span);
+                    }
+                    chunks.push(chunk);
+                }
+            }
+            catch (e) {
+                if (isSitefileError(e))
+                    throw e;
+                if (controller.signal.aborted)
+                    throw timeoutError();
+                throw new SitefileError('E_FETCH', `error reading ${redactUrl(current)}: ${e.message}`, span, {
+                    cause: e,
+                });
+            }
+            if (controller.signal.aborted)
+                throw timeoutError();
+            const buf = Buffer.concat(chunks, total);
+            const bytes = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
+            const ct = res.headers['content-type'];
+            const mediaType = ct ? ct.split(';')[0].trim().toLowerCase() || undefined : undefined;
+            return mediaType === undefined ? { bytes, finalUrl: current.href } : { bytes, finalUrl: current.href, mediaType };
+        }
+    }
+    finally {
+        clearTimeout(timer);
+        state.res?.destroy();
+        state.req?.destroy();
+    }
+}
+function openHop(url, policy, signal, state, span, timeoutError, method, extra, body) {
+    return new Promise((resolve, reject) => {
+        const options = {
+            method,
+            headers: { 'accept-encoding': 'identity', 'user-agent': USER_AGENT, accept: '*/*', ...extra, ...(body ? { 'content-length': String(body.byteLength) } : {}) },
+            lookup: createGuardedLookup(policy, span),
+            agent: false,
+            signal,
+        };
+        if (url.protocol === 'https:' && policy.ca !== undefined) {
+            // Additive: setting `ca` replaces Node's whole default store, so start from
+            // that store (bundled roots plus NODE_EXTRA_CA_CERTS / --use-system-ca),
+            // not just the bundled roots, and add the extra CA(s).
+            options.ca = [...defaultCaCertificates(), policy.ca];
+        }
+        const requestFn = url.protocol === 'https:' ? httpsRequest : httpRequest;
+        let req;
+        try {
+            req = requestFn(url, options);
+        }
+        catch (e) {
+            reject(new SitefileError('E_FETCH', `cannot request ${redactUrl(url)}: ${e.message}`, span, { cause: e }));
+            return;
+        }
+        state.req = req;
+        state.res = undefined;
+        req.on('response', (res) => {
+            state.res = res;
+            resolve(res);
+        });
+        req.on('error', (e) => {
+            if (signal.aborted) {
+                reject(timeoutError());
+                return;
+            }
+            const inner = findSitefileError(e);
+            if (inner) {
+                reject(inner);
+                return;
+            }
+            reject(new SitefileError('E_FETCH', `fetching ${redactUrl(url)} failed: ${e.message}`, span, { cause: e }));
+        });
+        req.end(body);
+    });
+}
+/** Node's default CA store; `tls.getCACertificates` exists from Node 22.15/23.5. */
+function defaultCaCertificates() {
+    const get = tls.getCACertificates;
+    return get ? get('default') : tls.rootCertificates;
+}
+//# sourceMappingURL=https.js.map

package/lib/src/resources/ignore.d.ts

@@ -0,0 +1,33 @@
+/**
+ * `.sitefileignore`: a gitignore-style subset, implemented without dependencies.
+ *
+ * - Blank lines and lines starting with `#` are skipped (`\#` escapes a literal `#`).
+ * - `!` negates a rule (`\!` escapes a literal `!`). The last matching rule wins.
+ * - A trailing `/` makes the rule match directories only.
+ * - A leading `/` or a `/` in the middle anchors the rule to the context root;
+ *   otherwise it matches at any depth.
+ * - `*` and `?` match within one path segment; `**` as a whole segment matches
+ *   any number of segments.
+ * - Unescaped trailing spaces are ignored. Character classes are not supported.
+ *
+ * As in gitignore, a file cannot be re-included once a parent directory is
+ * excluded; callers get this by not descending into excluded directories.
+ */
+export declare class IgnoreRules {
+    private readonly rules;
+    private constructor();
+    static readonly empty: IgnoreRules;
+    static parse(text: string): IgnoreRules;
+    /** Rules from `COPY --exclude` patterns, one `.sitefileignore` line each (see `isIgnorePattern`). */
+    static fromPatterns(patterns: readonly string[]): IgnoreRules;
+    /**
+     * Whether `rel` (a POSIX path relative to the rules' root) is excluded, counting excluded parent
+     * directories: what a directory walk that does not descend into excluded directories would skip.
+     */
+    excludes(rel: string): boolean;
+    get size(): number;
+    /** Whether the context-relative POSIX path (no leading `./`) is excluded by the last matching rule. */
+    ignores(rel: string, isDirectory: boolean): boolean;
+}
+/** Whether `text` is a `.sitefileignore` rule (not blank, a `#` comment, or only `!` and `/`). */
+export declare function isIgnorePattern(text: string): boolean;

package/lib/src/resources/ignore.js

@@ -0,0 +1,119 @@
+/**
+ * `.sitefileignore`: a gitignore-style subset, implemented without dependencies.
+ *
+ * - Blank lines and lines starting with `#` are skipped (`\#` escapes a literal `#`).
+ * - `!` negates a rule (`\!` escapes a literal `!`). The last matching rule wins.
+ * - A trailing `/` makes the rule match directories only.
+ * - A leading `/` or a `/` in the middle anchors the rule to the context root;
+ *   otherwise it matches at any depth.
+ * - `*` and `?` match within one path segment; `**` as a whole segment matches
+ *   any number of segments.
+ * - Unescaped trailing spaces are ignored. Character classes are not supported.
+ *
+ * As in gitignore, a file cannot be re-included once a parent directory is
+ * excluded; callers get this by not descending into excluded directories.
+ */
+export class IgnoreRules {
+    rules;
+    constructor(rules) {
+        this.rules = rules;
+    }
+    static empty = new IgnoreRules([]);
+    static parse(text) {
+        const rules = [];
+        for (const raw of text.split(/\r?\n/)) {
+            const rule = parseLine(raw);
+            if (rule)
+                rules.push(rule);
+        }
+        return new IgnoreRules(rules);
+    }
+    /** Rules from `COPY --exclude` patterns, one `.sitefileignore` line each (see `isIgnorePattern`). */
+    static fromPatterns(patterns) {
+        return new IgnoreRules(patterns.map(parseLine).filter((r) => r !== undefined));
+    }
+    /**
+     * Whether `rel` (a POSIX path relative to the rules' root) is excluded, counting excluded parent
+     * directories: what a directory walk that does not descend into excluded directories would skip.
+     */
+    excludes(rel) {
+        const segs = rel.split('/');
+        for (let i = 1; i < segs.length; i++)
+            if (this.ignores(segs.slice(0, i).join('/'), true))
+                return true;
+        return this.ignores(rel, false);
+    }
+    get size() {
+        return this.rules.length;
+    }
+    /** Whether the context-relative POSIX path (no leading `./`) is excluded by the last matching rule. */
+    ignores(rel, isDirectory) {
+        let ignored = false;
+        for (const r of this.rules) {
+            if (r.dirOnly && !isDirectory)
+                continue;
+            if (r.re.test(rel))
+                ignored = !r.negate;
+        }
+        return ignored;
+    }
+}
+/** Whether `text` is a `.sitefileignore` rule (not blank, a `#` comment, or only `!` and `/`). */
+export function isIgnorePattern(text) {
+    return parseLine(text) !== undefined;
+}
+function parseLine(raw) {
+    let line = raw.replace(/(?<!\\)\s+$/, '');
+    if (line === '' || line.startsWith('#'))
+        return undefined;
+    let negate = false;
+    if (line.startsWith('!')) {
+        negate = true;
+        line = line.slice(1);
+    }
+    const dirOnly = line.endsWith('/');
+    if (dirOnly)
+        line = line.replace(/\/+$/, '');
+    const anchored = line.includes('/');
+    line = line.replace(/^\/+/, '');
+    if (line === '')
+        return undefined;
+    const segs = line.split('/');
+    let src = '';
+    segs.forEach((seg, i) => {
+        const last = i === segs.length - 1;
+        if (seg === '**') {
+            // A trailing `**` needs at least one more segment; elsewhere it may match none.
+            src += last ? '.+' : '(?:[^/]+/)*';
+        }
+        else {
+            src += globSegment(seg) + (last ? '' : '/');
+        }
+    });
+    return { negate, dirOnly, re: new RegExp(`^${anchored ? '' : '(?:.*/)?'}${src}$`) };
+}
+function globSegment(seg) {
+    let out = '';
+    for (let i = 0; i < seg.length; i++) {
+        const c = seg[i];
+        if (c === '\\' && i + 1 < seg.length) {
+            out += escapeRe(seg[++i]);
+        }
+        else if (c === '*') {
+            while (seg[i + 1] === '*')
+                i++; // `**` inside a segment behaves like `*`
+            out += '[^/]*';
+        }
+        else if (c === '?') {
+            out += '[^/]';
+        }
+        else {
+            out += escapeRe(c);
+        }
+    }
+    return out;
+}
+function escapeRe(c) {
+    return c.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&');
+}
+//# sourceMappingURL=ignore.js.map

package/lib/src/resources/local.d.ts

@@ -0,0 +1,64 @@
+import { type SourceSpan } from '../errors.js';
+import { IgnoreRules } from './ignore.js';
+import type { FileResource, TreeResource } from './types.js';
+export interface LocalContextOptions {
+    /** Context-relative POSIX paths skipped while walking directories. Default `['.git']`. */
+    readonly ignore?: readonly string[];
+    /** Read `.sitefileignore` from the context root (default true). Applies to directory walks only. */
+    readonly sitefileignore?: boolean;
+    /**
+     * Permit regular files with more than one hard link (default false). realpath cannot
+     * tell whether a hard link points outside the context, so by default such files are
+     * refused with `E_HARDLINK`. Allowing them means you assert they are safe.
+     */
+    readonly allowHardlinks?: boolean;
+}
+export declare const SITEFILEIGNORE = ".sitefileignore";
+/**
+ * The local build context. Every read is confined to `root`: real paths are
+ * resolved and must stay inside it.
+ */
+export declare class LocalContext {
+    readonly root: string;
+    private readonly rules;
+    private readonly allowHardlinks;
+    private readonly ignore;
+    private constructor();
+    static open(dir: string, options?: LocalContextOptions): Promise<LocalContext>;
+    private inside;
+    private isIgnored;
+    /** Hard-linked files may alias files outside the context, which realpath cannot detect. */
+    private checkHardlink;
+    /** Context-relative POSIX path of a real path inside root. */
+    private relOfReal;
+    /** Resolve a context-relative path to its real path and stats, enforcing the sandbox. */
+    /** A context file's real path and bytes, with the usual sandbox, symlink and hard-link rules. */
+    realFile(rel: string, span?: SourceSpan): Promise<{
+        real: string;
+        bytes: Uint8Array;
+    }>;
+    /**
+     * The host path of the context-relative (NFC) path `rel`. A non-ASCII
+     * segment is matched by NFC form against its directory listing, so a file
+     * is found whichever normalization its on-disk name uses (macOS stores what
+     * it was given, often NFD; Linux compares bytes). Two names in one
+     * directory with the same NFC form are E_PATH_COLLISION. A segment with no
+     * match is joined as is, so the caller sees the usual ENOENT.
+     */
+    private hostPath;
+    private resolve;
+    private static kindOf;
+    stat(rel: string, span?: SourceSpan): Promise<'file' | 'directory'>;
+    /** The real path of a context-relative path, with the sandbox checks of every read (an imported Sitefile's context). */
+    realPath(rel: string, span?: SourceSpan): Promise<string>;
+    readFile(rel: string, span?: SourceSpan): Promise<FileResource>;
+    /** `exclude` (COPY --exclude) skips paths relative to `rel`, like `.sitefileignore` rules (excluded directories are not read). */
+    readTree(rel: string, origin: string, span?: SourceSpan, exclude?: IgnoreRules): Promise<TreeResource>;
+    /**
+     * @param realDir   real path of the directory being walked
+     * @param ctxRel    logical context-relative path of that directory
+     * @param treeRel   path of that directory inside the resulting tree ('' = root)
+     * @param ancestors real paths of directories on the current walk path (cycle detection)
+     */
+    private walk;
+}