@ferrule-io/ok-fine 0.0.0-stage → 0.3.6

package/dist/service/principal.js

@@ -0,0 +1,22 @@
+import { OkfError } from "../errors.js";
+import { parseActor } from "../okf/semantics.js";
+export function checkActor(actor, p) {
+    const parsed = parseActor(actor);
+    if (!parsed) {
+        throw new OkfError("invalid_actor", 400, "actor must be <producer>/<version>, human:<id>, or process:<id>");
+    }
+    if (parsed.kind === "human") {
+        if (p.identity === null) {
+            throw new OkfError("forbidden_actor", 403, "this token carries no identity claim; human: actors are not allowed");
+        }
+        // Email identities compare case-insensitively; any other identity must match exactly.
+        const matches = p.identity.includes("@")
+            ? parsed.id.toLowerCase() === p.identity.toLowerCase()
+            : parsed.id === p.identity;
+        if (!matches) {
+            throw new OkfError("forbidden_actor", 403, `this token may only act as human:${p.identity}`);
+        }
+    }
+    return actor;
+}
+//# sourceMappingURL=principal.js.map

package/dist/store/archive.js

@@ -0,0 +1,108 @@
+import { Readable } from "node:stream";
+import * as tar from "tar";
+import { OkfError } from "../errors.js";
+const IGNORED_ENTRY_TYPES = new Set([
+    "GlobalExtendedHeader",
+    "ExtendedHeader",
+    "NextFileHasLongPath",
+    "NextFileHasLongLinkpath",
+    "OldGnuLongPath",
+]);
+const ALLOWED_ENTRY_TYPES = new Set(["File", "OldFile", "Directory"]);
+export async function extractBundleArchive(buffer, destDir, limits) {
+    // Pass 1: Listing and validation
+    const { promise: listPromise, resolve: listResolve, reject: listReject } = Promise.withResolvers();
+    let validationError = null;
+    let entryCount = 0;
+    let totalBytes = 0;
+    const topDirs = new Set();
+    const listStream = tar.t({
+        onentry: (entry) => {
+            if (validationError) {
+                return;
+            }
+            const type = entry.type;
+            if (IGNORED_ENTRY_TYPES.has(type)) {
+                return;
+            }
+            if (!ALLOWED_ENTRY_TYPES.has(type)) {
+                validationError = new OkfError("invalid_archive", 400, `archive contains unsupported entry type: ${type}`);
+                return;
+            }
+            if (entry.path.startsWith("/")) {
+                validationError = new OkfError("invalid_archive", 400, "archive cannot contain absolute paths");
+                return;
+            }
+            const segments = entry.path.split("/").filter((s) => s.length > 0);
+            if (segments.length === 0) {
+                validationError = new OkfError("invalid_archive", 400, "archive contains empty entry path");
+                return;
+            }
+            if (segments.includes("..")) {
+                validationError = new OkfError("invalid_archive", 400, "archive cannot contain path traversal (..) segments");
+                return;
+            }
+            if (type === "File" || type === "OldFile") {
+                if (segments.length < 2) {
+                    validationError = new OkfError("archive_layout", 400, "archive must contain a single top-level directory, as produced by export");
+                    return;
+                }
+            }
+            const topDir = segments[0];
+            if (topDir) {
+                topDirs.add(topDir);
+                if (topDirs.size > 1) {
+                    validationError = new OkfError("archive_layout", 400, "archive must contain a single top-level directory, as produced by export");
+                    return;
+                }
+            }
+            if (entry.size != null && entry.size > limits.maxFileBytes) {
+                validationError = new OkfError("payload_too_large", 413, `archive entry "${entry.path}" exceeds maximum file size (${limits.maxFileBytes} bytes)`);
+                return;
+            }
+            totalBytes += entry.size ?? 0;
+            if (totalBytes > 5 * limits.maxArchiveBytes) {
+                validationError = new OkfError("payload_too_large", 413, `uncompressed archive exceeds 5x max archive bytes limit (${5 * limits.maxArchiveBytes} bytes)`);
+                return;
+            }
+            entryCount++;
+            if (entryCount > 20000) {
+                validationError = new OkfError("payload_too_large", 413, "archive contains more than 20,000 entries");
+                return;
+            }
+        },
+    });
+    listStream.on("end", () => {
+        if (validationError) {
+            listReject(validationError);
+            return;
+        }
+        if (topDirs.size !== 1) {
+            listReject(new OkfError("archive_layout", 400, "archive must contain a single top-level directory, as produced by export"));
+            return;
+        }
+        listResolve();
+    });
+    listStream.on("error", (err) => {
+        listReject(new OkfError("invalid_archive", 400, `failed to read archive: ${err.message}`));
+    });
+    Readable.from(buffer).pipe(listStream);
+    await listPromise;
+    // Pass 2: Extract with strip: 1 and skip dotfiles
+    const { promise: extractPromise, resolve: extractResolve, reject: extractReject } = Promise.withResolvers();
+    const extractStream = tar.x({
+        cwd: destDir,
+        strip: 1,
+        filter: (entryPath) => {
+            const parts = entryPath.split("/");
+            return !parts.some((p) => p.startsWith("."));
+        },
+    });
+    extractStream.on("end", extractResolve);
+    extractStream.on("error", (err) => {
+        extractReject(new OkfError("invalid_archive", 400, `failed to extract archive: ${err.message}`));
+    });
+    Readable.from(buffer).pipe(extractStream);
+    await extractPromise;
+}
+//# sourceMappingURL=archive.js.map

package/dist/store/backend.js

@@ -0,0 +1,4 @@
+export function bundleSource(storage, project, tree) {
+    return { paths: tree.paths, read: (path) => storage.readFile(project, path) };
+}
+//# sourceMappingURL=backend.js.map

package/dist/store/bundle.js

@@ -0,0 +1,250 @@
+import { posix } from "node:path";
+import { renderIndex } from "../okf/index-file.js";
+import { lintConceptFile, lintIndexFile, lintLogFile } from "../okf/lint.js";
+import { isReservedName } from "../okf/paths.js";
+import { displayTitle } from "../okf/semantics.js";
+/** True for non-reserved `.md` files, i.e. files that hold a concept. */
+export function isConceptPath(path) {
+    const name = posix.basename(path);
+    return !isReservedName(name) && name.toLowerCase().endsWith(".md");
+}
+const NO_ENTRIES = Object.freeze({ files: [], dirs: [] });
+const NO_COUNTS = Object.freeze({ conceptCount: 0, fileCount: 0 });
+function parentDir(path) {
+    const parent = posix.dirname(path);
+    return parent === "." ? "" : parent;
+}
+function normalizeDir(dir) {
+    return dir === "." || dir === "/" ? "" : dir.replace(/^\/+|\/+$/g, "");
+}
+/** Immutable file/directory model of one bundle, built from its file paths (including index.md/log.md). */
+export class BundleTree {
+    static EMPTY = new BundleTree([]);
+    paths;
+    conceptFiles;
+    otherFiles;
+    dirs;
+    fileSet;
+    dirSet;
+    dirEntries = new Map();
+    dirCounts = new Map();
+    constructor(paths) {
+        const sorted = Array.from(new Set(paths)).sort();
+        const conceptFiles = [];
+        const otherFiles = [];
+        const dirSet = new Set();
+        for (const path of sorted) {
+            const name = posix.basename(path);
+            const reserved = isReservedName(name);
+            const isConcept = !reserved && name.toLowerCase().endsWith(".md");
+            if (isConcept) {
+                conceptFiles.push(path);
+            }
+            else if (!reserved) {
+                otherFiles.push(path);
+            }
+            let dir = parentDir(path);
+            this.entriesFor(dir).files.push(name);
+            while (true) {
+                if (!reserved) {
+                    const counts = this.dirCounts.get(dir) ?? { conceptCount: 0, fileCount: 0 };
+                    if (isConcept) {
+                        counts.conceptCount++;
+                    }
+                    else {
+                        counts.fileCount++;
+                    }
+                    this.dirCounts.set(dir, counts);
+                }
+                if (dir === "") {
+                    break;
+                }
+                if (!dirSet.has(dir)) {
+                    // First sighting: register dir as a child of its parent.
+                    dirSet.add(dir);
+                    this.entriesFor(parentDir(dir)).dirs.push(posix.basename(dir));
+                }
+                dir = parentDir(dir);
+            }
+        }
+        this.paths = sorted;
+        this.conceptFiles = conceptFiles;
+        this.otherFiles = otherFiles;
+        this.fileSet = new Set(sorted);
+        this.dirSet = dirSet;
+        this.dirs = Array.from(dirSet).sort();
+    }
+    entriesFor(dir) {
+        let entries = this.dirEntries.get(dir);
+        if (!entries) {
+            entries = { files: [], dirs: [] };
+            this.dirEntries.set(dir, entries);
+        }
+        return entries;
+    }
+    get isEmpty() {
+        return this.paths.length === 0;
+    }
+    hasFile(path) {
+        return this.fileSet.has(path);
+    }
+    hasDir(dir) {
+        return dir === "" ? !this.isEmpty : this.dirSet.has(dir);
+    }
+    /** Direct-child basenames of `dir` ("" = root). */
+    entries(dir) {
+        return this.dirEntries.get(dir) ?? NO_ENTRIES;
+    }
+    /** Recursive counts of non-reserved files below `dir`. */
+    counts(dir) {
+        return this.dirCounts.get(dir) ?? NO_COUNTS;
+    }
+    /** True for an existing file or directory addressed by a bundle path that stays inside the bundle. */
+    exists(bundlePath) {
+        let normalized = posix.normalize(bundlePath.replace(/^\/+/, ""));
+        if (normalized === ".." || normalized.startsWith("../"))
+            return false;
+        if (normalized === "." || normalized === "./")
+            return !this.isEmpty;
+        normalized = normalized.replace(/\/+$/, "");
+        return this.fileSet.has(normalized) || this.dirSet.has(normalized);
+    }
+}
+export function buildDirListing(tree, relDir, catalogLookup) {
+    const normRelDir = normalizeDir(relDir);
+    const isRoot = normRelDir === "";
+    const entries = tree.entries(normRelDir);
+    const concepts = [];
+    const files = [];
+    const dirs = [];
+    for (const name of entries.files) {
+        if (isReservedName(name)) {
+            continue;
+        }
+        const itemRel = isRoot ? name : posix.join(normRelDir, name);
+        if (name.toLowerCase().endsWith(".md")) {
+            const conceptId = itemRel.slice(0, -3);
+            const record = catalogLookup(conceptId);
+            concepts.push({
+                file: name,
+                title: record?.title ?? displayTitle(null, conceptId),
+                description: record?.description ?? null,
+                type: record?.type ?? null,
+            });
+        }
+        else {
+            files.push(name);
+        }
+    }
+    for (const name of entries.dirs) {
+        const counts = tree.counts(isRoot ? name : posix.join(normRelDir, name));
+        // Dirs without non-reserved files are pruned by index regeneration, so they are never listed.
+        if (counts.conceptCount + counts.fileCount === 0) {
+            continue;
+        }
+        dirs.push({ name, conceptCount: counts.conceptCount, fileCount: counts.fileCount });
+    }
+    return { isRoot, concepts, files, dirs };
+}
+/**
+ * Plans index.md changes for `dirs` (plus their ancestors) or every dir. `content: null` means delete.
+ * Ordered deepest-first so a child's deletion (which prunes its dir) precedes its parent.
+ */
+export function planIndexes(tree, dirs, lookup) {
+    if (tree.isEmpty) {
+        return [];
+    }
+    let targetDirs;
+    if (dirs === "all") {
+        targetDirs = new Set(["", ...tree.dirs]);
+    }
+    else {
+        targetDirs = new Set([""]);
+        for (const d of dirs) {
+            let current = normalizeDir(d);
+            while (current !== "") {
+                targetDirs.add(current);
+                current = parentDir(current);
+            }
+        }
+    }
+    const sorted = Array.from(targetDirs).sort((a, b) => {
+        const depthA = a === "" ? 0 : a.split("/").length;
+        const depthB = b === "" ? 0 : b.split("/").length;
+        if (depthB !== depthA) {
+            return depthB - depthA;
+        }
+        return b.localeCompare(a);
+    });
+    const plan = [];
+    for (const dir of sorted) {
+        const indexPath = dir === "" ? "index.md" : `${dir}/index.md`;
+        if (dir !== "") {
+            if (!tree.hasDir(dir)) {
+                continue;
+            }
+            const counts = tree.counts(dir);
+            if (counts.conceptCount + counts.fileCount === 0) {
+                if (tree.hasFile(indexPath)) {
+                    plan.push({ path: indexPath, content: null });
+                }
+                continue;
+            }
+        }
+        plan.push({ path: indexPath, content: renderIndex(buildDirListing(tree, dir, lookup)) });
+    }
+    return plan;
+}
+export async function lintBundle(source, now) {
+    const issues = [];
+    const tree = new BundleTree(source.paths);
+    const conceptIdSet = new Set(tree.conceptFiles.map((f) => f.slice(0, -3)));
+    const allBundleFiles = new Set([...tree.conceptFiles, ...tree.otherFiles]);
+    const ctx = {
+        now,
+        conceptExists: (id) => conceptIdSet.has(id),
+        fileExists: (relPath) => allBundleFiles.has(relPath),
+    };
+    const readText = async (path) => {
+        const buf = await source.read(path);
+        return buf === null ? null : buf.toString("utf8");
+    };
+    for (const cFile of tree.conceptFiles) {
+        const text = await readText(cFile);
+        if (text !== null) {
+            issues.push(...lintConceptFile(cFile, text, ctx));
+        }
+    }
+    if (tree.hasFile("index.md")) {
+        const text = await readText("index.md");
+        if (text !== null) {
+            issues.push(...lintIndexFile("index.md", text, true));
+        }
+    }
+    else {
+        issues.push({
+            severity: "info",
+            code: "missing_root_index",
+            path: "index.md",
+            message: "bundle is missing root index.md",
+        });
+    }
+    for (const d of tree.dirs) {
+        const subIndex = `${d}/index.md`;
+        if (tree.hasFile(subIndex)) {
+            const text = await readText(subIndex);
+            if (text !== null) {
+                issues.push(...lintIndexFile(subIndex, text, false));
+            }
+        }
+    }
+    if (tree.hasFile("log.md")) {
+        const text = await readText("log.md");
+        if (text !== null) {
+            issues.push(...lintLogFile("log.md", text));
+        }
+    }
+    const conformant = !issues.some((i) => i.severity === "error");
+    return { conformant, issues };
+}
+//# sourceMappingURL=bundle.js.map

package/dist/store/catalog.js

@@ -0,0 +1,295 @@
+import MiniSearch from "minisearch";
+import { parseConcept } from "../okf/concept.js";
+import { isStale } from "../okf/semantics.js";
+import { isConceptPath } from "./bundle.js";
+function tierRank(tier) {
+    if (tier === "human-reviewed")
+        return 3;
+    if (tier === "machine-confirmed")
+        return 2;
+    return 1;
+}
+function extractSnippet(body, terms, description) {
+    if (terms.length > 0) {
+        const lowerTerms = terms.map((t) => t.toLowerCase()).filter((t) => t.length > 0);
+        const lines = body.split(/\r?\n/);
+        let inCode = false;
+        for (const rawLine of lines) {
+            const trimmed = rawLine.trim();
+            if (trimmed.startsWith("```")) {
+                inCode = !inCode;
+                continue;
+            }
+            if (inCode || trimmed.startsWith("#") || trimmed.length === 0) {
+                continue;
+            }
+            const lowerLine = trimmed.toLowerCase();
+            if (lowerTerms.some((t) => lowerLine.includes(t))) {
+                if (trimmed.length > 240) {
+                    return `${trimmed.slice(0, 240)}…`;
+                }
+                return trimmed;
+            }
+        }
+    }
+    return description ?? null;
+}
+export class Catalog {
+    miniSearch;
+    // project -> (id -> ConceptRecord)
+    projectRecords = new Map();
+    // project -> (targetId -> Set<sourceId>)
+    inboundLinks = new Map();
+    constructor() {
+        this.miniSearch = new MiniSearch({
+            idField: "key",
+            fields: ["title", "description", "tags", "type", "idText", "body"],
+            storeFields: ["project", "id"],
+            searchOptions: {
+                boost: { title: 3, description: 2, tags: 2, idText: 1.5, type: 1, body: 1 },
+                prefix: (t) => t.length >= 3,
+                fuzzy: (t) => (t.length >= 5 ? 0.2 : 0),
+            },
+        });
+    }
+    async rebuildProject(project, source) {
+        this.removeProject(project);
+        for (const file of source.paths) {
+            if (!isConceptPath(file)) {
+                continue;
+            }
+            try {
+                const buf = await source.read(file);
+                if (buf === null) {
+                    continue;
+                }
+                const id = file.endsWith(".md") ? file.slice(0, -3) : file;
+                this.storeRecord(parseConcept(project, id, file, buf));
+            }
+            catch {
+                // ignore unreadable file during rebuild
+            }
+        }
+    }
+    removeProject(project) {
+        const pMap = this.projectRecords.get(project);
+        if (pMap) {
+            for (const id of pMap.keys()) {
+                const key = `${project}/${id}`;
+                if (this.miniSearch.has(key)) {
+                    this.miniSearch.discard(key);
+                }
+            }
+            this.projectRecords.delete(project);
+        }
+        this.inboundLinks.delete(project);
+    }
+    upsert(record) {
+        this.storeRecord(record);
+    }
+    storeRecord(record) {
+        const { project, id } = record;
+        let pMap = this.projectRecords.get(project);
+        if (!pMap) {
+            pMap = new Map();
+            this.projectRecords.set(project, pMap);
+        }
+        let pInbound = this.inboundLinks.get(project);
+        if (!pInbound) {
+            pInbound = new Map();
+            this.inboundLinks.set(project, pInbound);
+        }
+        const prev = pMap.get(id);
+        if (prev) {
+            for (const target of prev.outbound) {
+                pInbound.get(target)?.delete(id);
+            }
+        }
+        pMap.set(id, record);
+        for (const target of record.outbound) {
+            let sources = pInbound.get(target);
+            if (!sources) {
+                sources = new Set();
+                pInbound.set(target, sources);
+            }
+            sources.add(id);
+        }
+        const key = `${project}/${id}`;
+        if (record.parseError) {
+            if (this.miniSearch.has(key)) {
+                this.miniSearch.discard(key);
+            }
+        }
+        else {
+            const doc = {
+                key,
+                project,
+                id,
+                idText: id.replace(/[/\-_]/g, " "),
+                title: record.title,
+                description: record.description ?? "",
+                tags: record.tags.join(" "),
+                type: record.type ?? "",
+                body: record.body,
+            };
+            if (this.miniSearch.has(key)) {
+                this.miniSearch.replace(doc);
+            }
+            else {
+                this.miniSearch.add(doc);
+            }
+        }
+    }
+    remove(project, id) {
+        const pMap = this.projectRecords.get(project);
+        if (pMap) {
+            const prev = pMap.get(id);
+            if (prev) {
+                const pInbound = this.inboundLinks.get(project);
+                if (pInbound) {
+                    for (const target of prev.outbound) {
+                        pInbound.get(target)?.delete(id);
+                    }
+                }
+            }
+            pMap.delete(id);
+        }
+        const key = `${project}/${id}`;
+        if (this.miniSearch.has(key)) {
+            this.miniSearch.discard(key);
+        }
+    }
+    get(project, id) {
+        return this.projectRecords.get(project)?.get(id);
+    }
+    records(project) {
+        const pMap = this.projectRecords.get(project);
+        return pMap ? Array.from(pMap.values()) : [];
+    }
+    projects() {
+        return Array.from(this.projectRecords.keys()).sort((a, b) => a.localeCompare(b, "en"));
+    }
+    inbound(project, id) {
+        const sources = this.inboundLinks.get(project)?.get(id);
+        if (!sources || sources.size === 0) {
+            return [];
+        }
+        return Array.from(sources).sort((a, b) => a.localeCompare(b, "en"));
+    }
+    search(params) {
+        const limit = Math.max(1, Math.min(100, params.limit ?? 20));
+        const now = new Date();
+        const matchesFilter = (record) => {
+            if (record.parseError) {
+                return false;
+            }
+            if (params.project && record.project !== params.project) {
+                return false;
+            }
+            if (params.type && record.type?.toLowerCase() !== params.type.toLowerCase()) {
+                return false;
+            }
+            if (params.tags && params.tags.length > 0) {
+                const lowerRecordTags = record.tags.map((t) => t.toLowerCase());
+                const hasAllTags = params.tags.every((t) => lowerRecordTags.includes(t.toLowerCase()));
+                if (!hasAllTags) {
+                    return false;
+                }
+            }
+            if (params.status != null) {
+                if (record.status !== params.status) {
+                    return false;
+                }
+            }
+            else {
+                // If status is omitted, exclude deprecated
+                if (record.status === "deprecated") {
+                    return false;
+                }
+            }
+            if (params.trustTier != null && record.trustTier !== params.trustTier) {
+                return false;
+            }
+            if (params.stale !== undefined) {
+                const staleState = isStale(record.frontmatter, now);
+                if (staleState !== params.stale) {
+                    return false;
+                }
+            }
+            return true;
+        };
+        const q = params.query?.trim() ?? "";
+        if (q.length > 0) {
+            const rawHits = this.miniSearch.search(q);
+            const results = [];
+            for (const hit of rawHits) {
+                const record = this.get(hit.project, hit.id);
+                if (!record || !matchesFilter(record)) {
+                    continue;
+                }
+                const snippet = extractSnippet(record.body, hit.terms, record.description);
+                results.push({
+                    project: record.project,
+                    id: record.id,
+                    title: record.title,
+                    description: record.description,
+                    type: record.type,
+                    tags: record.tags,
+                    status: record.status,
+                    trustTier: record.trustTier,
+                    stale: isStale(record.frontmatter, now),
+                    score: hit.score,
+                    snippet,
+                });
+            }
+            results.sort((a, b) => {
+                const scoreDiff = (b.score ?? 0) - (a.score ?? 0);
+                if (Math.abs(scoreDiff) > 1e-9) {
+                    return scoreDiff;
+                }
+                const tierDiff = tierRank(b.trustTier) - tierRank(a.trustTier);
+                if (tierDiff !== 0) {
+                    return tierDiff;
+                }
+                const keyA = `${a.project}/${a.id}`;
+                const keyB = `${b.project}/${b.id}`;
+                return keyA.localeCompare(keyB, "en");
+            });
+            return results.slice(0, limit);
+        }
+        else {
+            const results = [];
+            const projectList = params.project ? [params.project] : this.projects();
+            for (const p of projectList) {
+                const pRecords = this.records(p);
+                for (const record of pRecords) {
+                    if (!matchesFilter(record)) {
+                        continue;
+                    }
+                    results.push({
+                        project: record.project,
+                        id: record.id,
+                        title: record.title,
+                        description: record.description,
+                        type: record.type,
+                        tags: record.tags,
+                        status: record.status,
+                        trustTier: record.trustTier,
+                        stale: isStale(record.frontmatter, now),
+                        score: null,
+                        snippet: record.description,
+                    });
+                }
+            }
+            results.sort((a, b) => {
+                const pDiff = a.project.localeCompare(b.project, "en");
+                if (pDiff !== 0) {
+                    return pDiff;
+                }
+                return a.id.localeCompare(b.id, "en");
+            });
+            return results.slice(0, limit);
+        }
+    }
+}
+//# sourceMappingURL=catalog.js.map