@johnhenry/sitefile 0.0.0-stage → 0.0.0

package/lib/src/memo.d.ts

@@ -0,0 +1,57 @@
+import { ExportState } from './exporter.js';
+/**
+ * Incremental rebuilds (watch mode): results of build steps, memoized in memory by
+ * content digests of everything each step reads, like Bazel's action cache.
+ *
+ * Keys are SHA-256 digests of a step's identity (the planned operation with its
+ * Sitefile position, the ARG/LET values) and the digests of its inputs (file and
+ * tree digests, import keys, earlier steps' keys). A step whose key is unchanged
+ * returns its previous result without running; anything else runs as in a clean
+ * build. Failures are never memoized. Every entry holds the warnings the step
+ * printed, which a hit prints again, so an incremental build prints and writes
+ * exactly what a clean build does.
+ *
+ * The memo lives only in memory, for one `build --watch` / `serve --watch`
+ * session (see docs/ARCHITECTURE.md, "Incremental rebuilds"). Entries not used by
+ * the latest build (or by the last successful one, while builds fail) are dropped.
+ */
+export interface MemoStats {
+    /** Steps whose result was reused (operations, FOR elements, imports, documents). */
+    hits: number;
+    /** Steps that ran (and were memoized when they succeeded). */
+    misses: number;
+}
+export declare class BuildMemo {
+    private readonly entries;
+    private gen;
+    private goodGen;
+    stats: MemoStats;
+    /** What the session's last directory export wrote, so the next one writes only the changes. */
+    readonly exportState: ExportState;
+    /** Start a build: new statistics, a new generation. */
+    begin(): void;
+    /** End a build: keep what it used (and, after a failure, what the last good build used). */
+    finish(ok: boolean): void;
+    get size(): number;
+    /** Mark an entry and everything inside it as used by this build. */
+    private touch;
+    /** The value memoized under `key`; `parent` (an entry being computed) records it as a child. */
+    get<T>(key: string, parent?: string[]): T | undefined;
+    set(key: string, value: unknown, children?: readonly string[]): void;
+    /** `compute(children)` unless `key` is memoized (a throw is not memoized). */
+    memoize<T>(key: string, compute: (children: string[]) => T, parent?: string[]): T;
+    /** Per-document memo for Markdown and data collections (one entry per file digest, path and label), children of `parent`. */
+    documents(parent: string[]): DocumentMemo;
+}
+export type DocumentMemo = <T>(parts: readonly string[], compute: () => T) => T;
+/** Set to a non-empty value other than `0` to make watch mode rebuild everything every time. */
+export declare const NO_INCREMENTAL_ENV = "SITEFILE_NO_INCREMENTAL";
+/** The memo for a watch session: undefined when $SITEFILE_NO_INCREMENTAL turns it off. */
+export declare function watchMemo(): BuildMemo | undefined;
+/** A memo key: the digest of its parts (strings joined unambiguously). */
+export declare function memoKey(...parts: readonly string[]): string;
+/**
+ * An unambiguous text form of a JSON-like value (imported data, ARG/LET values, plans).
+ * Unlike JSON.stringify alone it tells `undefined`, NaN and ±Infinity apart from null.
+ */
+export declare function valueKey(v: unknown): string;

package/lib/src/memo.js

@@ -0,0 +1,93 @@
+import { sha256 } from './digest.js';
+import { ExportState } from './exporter.js';
+export class BuildMemo {
+    entries = new Map();
+    gen = 0;
+    goodGen = 0;
+    stats = { hits: 0, misses: 0 };
+    /** What the session's last directory export wrote, so the next one writes only the changes. */
+    exportState = new ExportState();
+    /** Start a build: new statistics, a new generation. */
+    begin() {
+        this.gen++;
+        this.stats = { hits: 0, misses: 0 };
+    }
+    /** End a build: keep what it used (and, after a failure, what the last good build used). */
+    finish(ok) {
+        if (ok)
+            this.goodGen = this.gen;
+        const keep = Math.min(this.gen, this.goodGen);
+        for (const [k, e] of this.entries)
+            if (e.gen < keep)
+                this.entries.delete(k);
+    }
+    get size() {
+        return this.entries.size;
+    }
+    /** Mark an entry and everything inside it as used by this build. */
+    touch(e) {
+        if (e.gen === this.gen)
+            return;
+        e.gen = this.gen;
+        for (const c of e.children) {
+            const child = this.entries.get(c);
+            if (child)
+                this.touch(child);
+        }
+    }
+    /** The value memoized under `key`; `parent` (an entry being computed) records it as a child. */
+    get(key, parent) {
+        parent?.push(key);
+        const e = this.entries.get(key);
+        if (!e)
+            return undefined;
+        this.touch(e);
+        this.stats.hits++;
+        return e.value;
+    }
+    set(key, value, children = []) {
+        this.stats.misses++;
+        this.entries.set(key, { gen: this.gen, value, children });
+    }
+    /** `compute(children)` unless `key` is memoized (a throw is not memoized). */
+    memoize(key, compute, parent) {
+        const hit = this.get(key, parent);
+        if (hit !== undefined)
+            return hit;
+        const children = [];
+        const v = compute(children);
+        this.set(key, v, children);
+        return v;
+    }
+    /** Per-document memo for Markdown and data collections (one entry per file digest, path and label), children of `parent`. */
+    documents(parent) {
+        return (parts, compute) => this.memoize(memoKey('document', ...parts), compute, parent);
+    }
+}
+/** Set to a non-empty value other than `0` to make watch mode rebuild everything every time. */
+export const NO_INCREMENTAL_ENV = 'SITEFILE_NO_INCREMENTAL';
+/** The memo for a watch session: undefined when $SITEFILE_NO_INCREMENTAL turns it off. */
+export function watchMemo() {
+    const v = process.env[NO_INCREMENTAL_ENV];
+    return v !== undefined && v !== '' && v !== '0' ? undefined : new BuildMemo();
+}
+/** A memo key: the digest of its parts (strings joined unambiguously). */
+export function memoKey(...parts) {
+    return sha256(parts.map((p) => `${p.length}:${p}`).join(''));
+}
+/**
+ * An unambiguous text form of a JSON-like value (imported data, ARG/LET values, plans).
+ * Unlike JSON.stringify alone it tells `undefined`, NaN and ±Infinity apart from null.
+ */
+export function valueKey(v) {
+    return (JSON.stringify(v, (_k, x) => {
+        if (x === undefined)
+            return { '\u0000': 'undefined' };
+        if (typeof x === 'number' && !Number.isFinite(x))
+            return { '\u0000': String(x) };
+        if (x instanceof Map)
+            return { '\u0000': 'map', entries: [...x] };
+        return x;
+    }) ?? 'undefined');
+}
+//# sourceMappingURL=memo.js.map

package/lib/src/oci.d.ts

@@ -0,0 +1,178 @@
+import type { Referrer } from './attest.js';
+import type { OciCompression } from './language/ast.js';
+import { Tree } from './tree.js';
+/**
+ * OCI image-spec v1.1 writer and reader for Sitefile images.
+ *
+ * A Sitefile image has exactly one layer: a deterministic tar of the site's files placed under a
+ * root directory (default /srv/www), gzip-compressed. It carries files only: it has no entrypoint
+ * and is not runnable by itself (use it with `COPY --from=` in a Dockerfile). Everything that is
+ * hashed (config, manifest, index) is canonical JSON, so the same tree and options always give the
+ * same manifest digest.
+ *
+ * Layout (the on-disk image store is this layout; `save`/`EXPORT AS oci` write it standalone):
+ *   oci-layout            {"imageLayoutVersion":"1.0.0"}
+ *   index.json            image index; entries carry org.opencontainers.image.ref.name
+ *   blobs/sha256/<hex>    manifest, config and layer blobs
+ */
+export declare const DEFAULT_OCI_ROOT = "/srv/www";
+export declare const MEDIA: {
+    readonly manifest: 'application/vnd.oci.image.manifest.v1+json';
+    readonly config: 'application/vnd.oci.image.config.v1+json';
+    readonly layer: 'application/vnd.oci.image.layer.v1.tar+gzip';
+    /** Uncompressed layer (`EXPORT ... COMPRESSION none`): the blob is the tar itself, digest = diff_id. */
+    readonly layerTar: 'application/vnd.oci.image.layer.v1.tar';
+    readonly index: 'application/vnd.oci.image.index.v1+json';
+    readonly dockerManifest: 'application/vnd.docker.distribution.manifest.v2+json';
+    readonly dockerList: 'application/vnd.docker.distribution.manifest.list.v2+json';
+    readonly dockerConfig: 'application/vnd.docker.container.image.v1+json';
+    readonly dockerLayer: 'application/vnd.docker.image.rootfs.diff.tar.gzip';
+};
+export declare const ANN_BASE_NAME = "org.opencontainers.image.base.name";
+export declare const ANN_BASE_DIGEST = "org.opencontainers.image.base.digest";
+export declare const ANN: {
+    readonly refName: 'org.opencontainers.image.ref.name';
+    readonly title: 'org.opencontainers.image.title';
+    readonly source: 'org.opencontainers.image.source';
+    readonly revision: 'org.opencontainers.image.revision';
+    readonly root: 'dev.sitefile.root';
+    readonly files: 'dev.sitefile.files';
+    readonly size: 'dev.sitefile.size';
+    readonly stored: 'dev.sitefile.created';
+};
+export interface OciOptions {
+    /** Where the files live inside the layer. Default /srv/www. */
+    readonly root?: string | undefined;
+    /** `os/arch`, default linux/amd64. Metadata only: the image holds no executables. */
+    readonly platform?: string | undefined;
+    /** `SOURCE_DATE_EPOCH` seconds; default the fixed epoch, so digests are reproducible. */
+    readonly epoch?: number | undefined;
+    readonly title?: string | undefined;
+    readonly source?: string | undefined;
+    readonly revision?: string | undefined;
+    /** Extra config labels (`--label key=value`); they override the defaults above. */
+    readonly labels?: Readonly<Record<string, string>> | undefined;
+    /**
+     * Build on top of this image (`EXPORT ... AS oci BASE <image>`): its layers come first, its config
+     * (Entrypoint, Cmd, Env, ExposedPorts, WorkingDir, ...) is kept, and the site layer is appended.
+     * The base's blobs must already be in the store (see base-image.ts).
+     */
+    readonly base?: BaseImage | undefined;
+    /** Site layer encoding: gzip (default, `tar+gzip`) or none (plain `tar`). Base layers are kept as they are. */
+    readonly compression?: OciCompression | undefined;
+}
+/** A resolved base image: one platform manifest, its config and layers. */
+export interface BaseImage {
+    /** Canonical reference as written (`docker.io/library/nginx:alpine`), without any digest. */
+    readonly name: string;
+    /** Digest of the platform manifest the layers come from (recorded in the image). */
+    readonly manifestDigest: string;
+    readonly layers: readonly Descriptor[];
+    /** The base's parsed config JSON. */
+    readonly config: Readonly<Record<string, unknown>>;
+}
+export interface Descriptor {
+    readonly mediaType: string;
+    readonly digest: string;
+    readonly size: number;
+    readonly annotations?: Readonly<Record<string, string>>;
+}
+export interface OciImage {
+    readonly manifestBytes: Uint8Array;
+    readonly manifestDigest: string;
+    readonly configBytes: Uint8Array;
+    readonly configDigest: string;
+    readonly layerBytes: Uint8Array;
+    readonly layerDigest: string;
+    /** sha256 of the uncompressed tar. */
+    readonly diffId: string;
+    readonly root: string;
+    /** Layers below the site layer (from BASE), already normalized to OCI media types. Empty otherwise. */
+    readonly baseLayers: readonly Descriptor[];
+}
+/** JSON with sorted keys and no whitespace. */
+export declare function canonicalJson(v: unknown): string;
+export declare function parsePlatform(p: string | undefined): {
+    os: string;
+    arch: string;
+};
+/** Validate an in-image root path (absolute, no `..`); returns the normalized form. */
+export declare function normalizeOciRoot(raw: string | undefined): string;
+/** Build the single-layer image for `tree` (the site's files, rooted at `/`). */
+export declare function buildOciImage(tree: Tree, options?: OciOptions): OciImage;
+export interface IndexEntry {
+    readonly digest: string;
+    readonly size: number;
+    readonly refName?: string | undefined;
+    /** When the image was first stored locally (store bookkeeping; not part of any digest). */
+    readonly stored?: string | undefined;
+    /** Extra annotations written verbatim (exports only). */
+    readonly extra?: Readonly<Record<string, string>> | undefined;
+    /** OCI 1.1 artifactType (attestation entries of exports only). */
+    readonly artifactType?: string | undefined;
+}
+export declare const ANN_CONTAINERD_NAME = "io.containerd.image.name";
+/**
+ * Index annotations for an exported image named `refName` (`blog:1`, `localhost:5000/blog:1`), in the
+ * convention `docker save` uses so Docker's containerd store finds the image by name after `docker load`:
+ * `io.containerd.image.name` is the full normalized reference and `org.opencontainers.image.ref.name` the tag.
+ * Single-component names live in docker.io/library/, other names without a registry host in docker.io/.
+ */
+export declare function exportNameAnnotations(refName: string): Record<string, string>;
+export declare function encodeIndex(entries: readonly IndexEntry[]): Uint8Array;
+export declare const OCI_LAYOUT_BYTES: NodeJS.NonSharedUint8Array;
+export declare function decodeIndex(bytes: Uint8Array): IndexEntry[];
+export declare function blobPathOf(digest: string): string;
+/**
+ * A standalone OCI image layout holding one image, as a Tree (write it as a directory or tar it).
+ * `referrers` (attestations, `build --provenance/--sbom`) are listed in index.json after the image,
+ * without a name, and their blobs are included.
+ */
+export declare function layoutTree(img: Pick<OciImage, 'manifestBytes' | 'manifestDigest' | 'configBytes' | 'configDigest' | 'layerBytes' | 'layerDigest'>, refName?: string, 
+/** Blobs of the layers below the site layer (digest -> bytes), for images built on a BASE. */
+baseBlobs?: ReadonlyMap<string, Uint8Array>, referrers?: readonly Referrer[]): Tree;
+/** Deterministic tar of {@link layoutTree}: an "OCI archive" (`.oci.tar`). */
+export declare function ociArchive(img: Parameters<typeof layoutTree>[0], refName: string | undefined, epoch: number | undefined, baseBlobs?: ReadonlyMap<string, Uint8Array>, referrers?: readonly Referrer[]): Uint8Array;
+export interface ParsedManifest {
+    readonly config: Descriptor;
+    readonly layers: readonly Descriptor[];
+    readonly annotations: Readonly<Record<string, string>>;
+}
+export declare function parseManifest(bytes: Uint8Array): ParsedManifest;
+export interface ParsedConfig {
+    readonly os: string;
+    readonly architecture: string;
+    readonly created: string;
+    readonly labels: Readonly<Record<string, string>>;
+    readonly diffIds: readonly string[];
+}
+export declare function parseConfig(bytes: Uint8Array): ParsedConfig;
+/**
+ * Index of the layer holding the site's files: the only layer, or the last one of an image that
+ * carries the `dev.sitefile.root` label (built with BASE). -1 when the image is not a Sitefile image.
+ */
+export declare function siteLayerIndex(m: ParsedManifest, cfg: ParsedConfig): number;
+/**
+ * Decode a site image's files, verifying the layer digest and diff_id.
+ * `layerBytes` are the compressed blob bytes. Returns the tree under the image's root (its
+ * `dev.sitefile.root` label, else `/`), or under `rootOverride` (`FROM image … AT <dir>`).
+ */
+export declare function extractImageFiles(m: ParsedManifest, cfg: ParsedConfig, layerBytes: Uint8Array, origin: string, rootOverride?: string): Promise<Tree>;
+/**
+ * Label values for an image built from `contextDir`: the title (directory name),
+ * extra `--label`s, and, only with `--git-labels`, source/revision from a git
+ * checkout. Git labels are opt-in because they make the image ID change with
+ * every commit even when the files don't, and because finding the checkout
+ * reads outside the build context.
+ */
+export declare function imageMeta(contextDir: string, opts?: {
+    readonly git?: boolean;
+    readonly labels?: Readonly<Record<string, string>>;
+}): Promise<Pick<OciOptions, 'title' | 'source' | 'revision' | 'labels'>>;
+/** Parse `--label key=value` flags. */
+export declare function parseLabels(flags: readonly string[]): Record<string, string>;
+/** Best-effort `org.opencontainers.image.{source,revision}` from a git checkout (never runs git; credentials in URLs are dropped). */
+export declare function gitInfo(contextDir: string): Promise<{
+    source?: string;
+    revision?: string;
+}>;