@better-fs-tools/read 0.0.0-tegami-trusted-publish-setup → 0.2.0

package/dist/core/pipeline.js

@@ -0,0 +1,175 @@
+import { authorizeRead } from "./authorize.js";
+import { CallScope } from "./call-scope.js";
+import { classificationInfo, classifySample, encodingRefusal } from "./classify.js";
+import { convertFile, selectFileConverter } from "./convert.js";
+import { AbortReadError, ByteCursor } from "./cursor.js";
+import { convertDirectory } from "./directory.js";
+import { formatResult } from "./format.js";
+import { runHooks } from "./hooks.js";
+import { clampedLimit, parseReadInput } from "./input.js";
+import { fileInfo, openFile } from "./open.js";
+import { ReadStop, aborted, invalidInput, ioError, unsupportedBackend, unsupportedOutcome, } from "./outcomes.js";
+import { recordOutcome } from "./record.js";
+import { resolvePath } from "./resolve.js";
+import { takeSample } from "./sample.js";
+import { scanText } from "./scan.js";
+import { missOutcome } from "./suggest.js";
+import { textOutcome } from "./text-outcome.js";
+import { checkSize, verifyHandle } from "./verify.js";
+export async function runRead(deps, input, call) {
+    return formatResult(deps, call, await readOutcome(deps, input, call));
+}
+async function readOutcome(deps, input, call) {
+    let request;
+    try {
+        request = parseReadInput(input, deps.limits);
+    }
+    catch (error) {
+        return invalidInput(deps.messages, input, error);
+    }
+    const scope = new CallScope(deps, request, call);
+    const clampNote = clampedLimitNote(deps, input, request);
+    let resolveNote = null;
+    const finish = (outcome) => withNotes(outcome, [
+        ...(clampNote === null ? [] : [clampNote]),
+        ...scope.allowNotes,
+        ...(resolveNote === null ? [] : [resolveNote]),
+    ]);
+    let outcome;
+    try {
+        outcome = await readStages(deps, request, scope, (note) => {
+            resolveNote = note;
+        });
+    }
+    catch (error) {
+        outcome = outcomeFromError(deps, request, scope, error);
+    }
+    outcome = finish(outcome);
+    if (isAborted(outcome))
+        return outcome;
+    try {
+        const hooked = await runHooks(scope, outcome);
+        await recordOutcome(scope, hooked);
+        return hooked;
+    }
+    catch (error) {
+        return finish(outcomeFromError(deps, request, scope, error));
+    }
+}
+async function readStages(deps, request, scope, onResolveNote) {
+    scope.checkAbort();
+    scope.enter("open");
+    const fs = scope.fileSystem();
+    scope.enter("resolve");
+    const resolved = await resolvePath(deps.resolve, request, fs, scope);
+    onResolveNote(resolved.note);
+    if (resolved.kind === "not-found") {
+        return missOutcome(deps, request, fs, scope, request.path, null);
+    }
+    scope.enter("open");
+    const opened = await openFile(deps, fs, request, resolved.path, scope);
+    if (opened.kind === "directory") {
+        const { error, converter } = opened;
+        return convertDirectory({
+            fs,
+            request,
+            openedPath: resolved.path,
+            error,
+            resolvedFrom: resolved.resolvedFrom,
+            converter,
+            scope,
+        });
+    }
+    const { handle } = opened;
+    try {
+        const file = fileInfo(fs, request, handle.info, resolved.resolvedFrom);
+        await authorizeRead(deps.authorize, request, file, scope);
+        return await readOpenFile(deps, fs, request, file, handle, scope);
+    }
+    finally {
+        await handle.close().catch(() => { });
+    }
+}
+function outcomeFromError(deps, request, scope, error) {
+    if (error instanceof ReadStop)
+        return error.outcome;
+    if (error instanceof AbortReadError)
+        return aborted(deps.messages, request, scope.phase);
+    return ioError(deps.messages, request, scope.phase, error);
+}
+function clampedLimitNote(deps, input, request) {
+    const requested = clampedLimit(input, request);
+    if (requested === null)
+        return null;
+    const max = deps.limits.maxLines;
+    return {
+        code: "clamped",
+        severity: "info",
+        message: deps.messages.limitClamped({ requested, max }),
+        data: { param: "limit", requested, max },
+    };
+}
+function isAborted(outcome) {
+    return outcome.status === "error" && outcome.error.code === "ABORTED";
+}
+function withNotes(outcome, notes) {
+    return notes.length === 0 ? outcome : { ...outcome, notes: [...outcome.notes, ...notes] };
+}
+async function readOpenFile(deps, fs, request, file, handle, scope) {
+    const { limits, messages, classifiers } = deps;
+    const cursor = new ByteCursor(handle.bytes());
+    try {
+        scope.enter("sampling");
+        const sample = await takeSample(cursor, limits, file, scope);
+        if (sample.complete)
+            checkSize(messages, request, file, sample.bytes.byteLength);
+        const decision = classifySample(classifiers, sample);
+        if (decision === null)
+            return unsupportedBackend(messages, request, "sampling", file, null);
+        const converter = selectFileConverter(scope, decision, sample);
+        if (converter !== null) {
+            return await convertFile({
+                deps,
+                fs,
+                request,
+                file,
+                handle,
+                cursor,
+                sample,
+                decision,
+                converter,
+                scope,
+            });
+        }
+        if (decision.classification.kind === "unsupported") {
+            return unsupportedOutcome(request, file, decision.classifier, decision.classification);
+        }
+        scope.enter("scan");
+        const scan = await scanText({ cursor, sample, request, limits, digest: deps.digest, scope });
+        if (scan.kind === "invalid-utf8") {
+            const refusal = encodingRefusal(classifiers, sample);
+            if (refusal === null) {
+                return unsupportedBackend(messages, request, "scan", file, "invalid utf-8 with no encoding refusal");
+            }
+            return unsupportedOutcome(request, file, refusal.classifier, refusal.classification);
+        }
+        scope.enter("verification");
+        if (scan.reachedEof)
+            checkSize(messages, request, file, scan.scannedBytes);
+        await verifyHandle(handle, messages, request, file, scope);
+        return textOutcome({
+            deps,
+            fs,
+            request,
+            file,
+            classification: classificationInfo(decision),
+            conversion: null,
+            scan,
+            contentId: scan.contentId,
+            notes: decision.classification.notes ?? [],
+        });
+    }
+    finally {
+        await cursor.close();
+    }
+}

package/dist/core/record.d.ts

@@ -0,0 +1,3 @@
+import type { ReadReport } from "../contract/result.ts";
+import type { CallScope } from "./call-scope.ts";
+export declare function recordOutcome<THost>(scope: CallScope<THost>, outcome: ReadReport): Promise<void>;

package/dist/core/record.js

@@ -0,0 +1,37 @@
+export async function recordOutcome(scope, outcome) {
+    if (outcome.status !== "ok" && outcome.status !== "media")
+        return;
+    const { observation } = outcome;
+    const { digest } = scope.deps;
+    if (observation === null || digest === null)
+        return;
+    const store = scope.stateStore();
+    if (store === null)
+        return;
+    try {
+        await store.put(outcome.file.resolvedPath, toRecord(outcome, observation, digest.id));
+    }
+    catch {
+        // Session state is a cache: a failing store never fails the read.
+    }
+}
+/** Copies fields one by one so nothing from the host `call` reaches the store. */
+function toRecord(outcome, observation, digest) {
+    const { file, request } = outcome;
+    return {
+        schema: 2,
+        origin: "read",
+        observationId: observation.id,
+        resolvedPath: file.resolvedPath,
+        identity: file.identity,
+        version: file.version,
+        digest,
+        contentId: observation.contentId,
+        viewId: observation.viewId,
+        observedAt: observation.observedAt,
+        wholeFileVisible: observation.wholeFileVisible,
+        // Media is only returned from a whole source, so its source totals are exact.
+        totalsExact: outcome.status === "ok" ? outcome.totals.exact : true,
+        request: { offset: request.offset, limit: request.limit },
+    };
+}

package/dist/core/resolve.d.ts

@@ -0,0 +1,15 @@
+import type { FileSystem } from "@better-fs-tools/fs";
+import type { PathResolver } from "../contract/extensions.ts";
+import type { ReadRequest } from "../contract/input.ts";
+import type { ReadNote } from "../contract/result.ts";
+import type { CallScope } from "./call-scope.ts";
+export type ResolvedPath = {
+    readonly kind: "path";
+    readonly path: string;
+    readonly resolvedFrom: string | null;
+    readonly note: ReadNote | null;
+} | {
+    readonly kind: "not-found";
+    readonly note: ReadNote | null;
+};
+export declare function resolvePath<THost>(resolver: PathResolver<THost> | null, request: ReadRequest, fs: FileSystem, scope: CallScope<THost>): Promise<ResolvedPath>;

package/dist/core/resolve.js

@@ -0,0 +1,43 @@
+import { extensionId } from "./extension-error.js";
+import { isRecord } from "./input.js";
+import { isNote } from "./outcomes.js";
+export async function resolvePath(resolver, request, fs, scope) {
+    if (resolver === null)
+        return { kind: "path", path: request.path, resolvedFrom: null, note: null };
+    const ctx = {
+        ...scope.hookContext(),
+        paths: fs.paths,
+        list: (dir) => scope.list("resolver", dir),
+    };
+    let outcome;
+    try {
+        outcome = await resolver.resolve(request.path, ctx);
+    }
+    catch (error) {
+        scope.checkAbort();
+        scope.throwHeld();
+        throw scope.extensionFailure("resolve", extensionId(resolver, error));
+    }
+    scope.checkAbort();
+    scope.throwHeld();
+    const malformed = () => scope.extensionFailure("resolve", extensionId(resolver));
+    if (!isRecord(outcome))
+        throw malformed();
+    const { note } = outcome;
+    if (note !== undefined && !isNote(note))
+        throw malformed();
+    if (outcome.kind === "not-found")
+        return { kind: "not-found", note: note ?? null };
+    const { path } = outcome;
+    if (outcome.kind !== "path" || !isPath(path))
+        throw malformed();
+    return {
+        kind: "path",
+        path,
+        resolvedFrom: path === request.path ? null : request.path,
+        note: note ?? null,
+    };
+}
+function isPath(value) {
+    return typeof value === "string" && value.trim() !== "" && !value.includes("\0");
+}

package/dist/core/same.d.ts

@@ -0,0 +1,2 @@
+/** Structural equality for plain objects, arrays, byte arrays, and primitives. */
+export declare function same(a: unknown, b: unknown): boolean;

package/dist/core/same.js

@@ -0,0 +1,22 @@
+/** Structural equality for plain objects, arrays, byte arrays, and primitives. */
+export function same(a, b) {
+    if (Object.is(a, b))
+        return true;
+    if (a === null || b === null || typeof a !== "object" || typeof b !== "object")
+        return false;
+    if (a instanceof Uint8Array || b instanceof Uint8Array) {
+        if (!(a instanceof Uint8Array && b instanceof Uint8Array))
+            return false;
+        return a.byteLength === b.byteLength && a.every((byte, index) => byte === b[index]);
+    }
+    if (Array.isArray(a) || Array.isArray(b)) {
+        if (!(Array.isArray(a) && Array.isArray(b)) || a.length !== b.length)
+            return false;
+        return a.every((entry, index) => same(entry, b[index]));
+    }
+    const keys = Object.keys(a);
+    if (keys.length !== Object.keys(b).length)
+        return false;
+    return keys.every((key) => Object.hasOwn(b, key) &&
+        same(a[key], b[key]));
+}

package/dist/core/sample.d.ts

@@ -0,0 +1,6 @@
+import type { ClassificationSample } from "../contract/classify.ts";
+import type { ReadLimits } from "../contract/limits.ts";
+import type { FileInfo } from "../contract/result.ts";
+import type { CallScope } from "./call-scope.ts";
+import type { ByteCursor } from "./cursor.ts";
+export declare function takeSample<THost>(cursor: ByteCursor, limits: Readonly<ReadLimits>, file: FileInfo, scope: CallScope<THost>): Promise<ClassificationSample>;

package/dist/core/sample.js

@@ -0,0 +1,24 @@
+export async function takeSample(cursor, limits, file, scope) {
+    const buffer = new Uint8Array(limits.sampleBytes);
+    let length = 0;
+    let reachedEof = false;
+    while (length < limits.sampleBytes) {
+        scope.checkAbort();
+        const item = await cursor.next(scope.signal);
+        if (item.done) {
+            reachedEof = true;
+            break;
+        }
+        const accepted = Math.min(limits.sampleBytes - length, item.value.byteLength);
+        buffer.set(item.value.subarray(0, accepted), length);
+        length += accepted;
+        if (accepted < item.value.byteLength)
+            cursor.unshift(item.value.subarray(accepted));
+    }
+    return {
+        bytes: buffer.subarray(0, length),
+        complete: reachedEof,
+        path: file.displayPath,
+        mimeType: file.mimeType,
+    };
+}

package/dist/core/scan.d.ts

@@ -0,0 +1,26 @@
+import type { ClassificationSample } from "../contract/classify.ts";
+import type { Digest } from "../contract/digest.ts";
+import type { ReadRequest } from "../contract/input.ts";
+import type { ReadLimits } from "../contract/limits.ts";
+import type { CallScope } from "./call-scope.ts";
+import type { ByteCursor } from "./cursor.ts";
+import { LineScanner } from "./scanner.ts";
+export type ScanOutcome = {
+    readonly kind: "scanned";
+    readonly scanner: LineScanner;
+    readonly scannedBytes: number;
+    readonly scanCapped: boolean;
+    readonly reachedEof: boolean;
+    readonly contentId: string | null;
+} | {
+    readonly kind: "invalid-utf8";
+};
+export interface ScanInput<THost> {
+    readonly cursor: ByteCursor;
+    readonly sample: ClassificationSample;
+    readonly request: ReadRequest;
+    readonly limits: Readonly<ReadLimits>;
+    readonly digest: Digest | null;
+    readonly scope: CallScope<THost>;
+}
+export declare function scanText<THost>(input: ScanInput<THost>): Promise<ScanOutcome>;

package/dist/core/scan.js

@@ -0,0 +1,95 @@
+import { scanBudget } from "./budget.js";
+import { LineScanner } from "./scanner.js";
+const ABORT_YIELD_BYTES = 1024 * 1024;
+class DecoderFailureError extends Error {
+}
+export async function scanText(input) {
+    const { cursor, sample, request, limits, digest, scope } = input;
+    const scanner = new LineScanner(request, limits, scanBudget(scope));
+    const decoder = new TextDecoder("utf-8", { fatal: true });
+    const contentHash = digest === null ? null : digest.create();
+    let scannedBytes = 0;
+    let scanCapped = false;
+    let unscannedBytesKnown = false;
+    let reachedEof = sample.complete;
+    let nextAbortYield = ABORT_YIELD_BYTES;
+    const consume = (chunk) => {
+        const available = limits.maxScanBytes - scannedBytes;
+        if (available <= 0) {
+            scanCapped = chunk.byteLength > 0;
+            unscannedBytesKnown ||= chunk.byteLength > 0;
+            return;
+        }
+        const accepted = chunk.subarray(0, available);
+        if (accepted.byteLength > 0) {
+            contentHash?.update(accepted);
+            scannedBytes += accepted.byteLength;
+            scanner.push(decodeChunk(decoder, accepted));
+        }
+        if (accepted.byteLength < chunk.byteLength) {
+            scanCapped = true;
+            unscannedBytesKnown = true;
+        }
+    };
+    try {
+        consume(sample.bytes);
+        while (!scanCapped && !reachedEof) {
+            scope.checkAbort();
+            if (scope.signal !== undefined && scannedBytes >= nextAbortYield) {
+                await yieldForAbort();
+                nextAbortYield = scannedBytes + ABORT_YIELD_BYTES;
+                scope.checkAbort();
+            }
+            const item = await cursor.next(scope.signal);
+            if (item.done) {
+                reachedEof = true;
+                break;
+            }
+            consume(item.value);
+        }
+        scope.checkAbort();
+        if (reachedEof) {
+            const trailing = finishDecoder(decoder);
+            if (trailing)
+                scanner.push(trailing);
+            scanner.finish();
+        }
+        else {
+            scanner.stopAtScanLimit(unscannedBytesKnown);
+        }
+    }
+    catch (error) {
+        if (error instanceof DecoderFailureError)
+            return { kind: "invalid-utf8" };
+        throw error;
+    }
+    return {
+        kind: "scanned",
+        scanner,
+        scannedBytes,
+        scanCapped,
+        reachedEof,
+        contentId: reachedEof ? (contentHash?.digest() ?? null) : null,
+    };
+}
+function decodeChunk(decoder, chunk) {
+    try {
+        return decoder.decode(chunk, { stream: true });
+    }
+    catch (error) {
+        throw new DecoderFailureError("Invalid UTF-8", { cause: error });
+    }
+}
+function finishDecoder(decoder) {
+    try {
+        return decoder.decode();
+    }
+    catch (error) {
+        throw new DecoderFailureError("Invalid UTF-8", { cause: error });
+    }
+}
+function yieldForAbort() {
+    return new Promise((resolve) => {
+        setTimeout(resolve, 0);
+    });
+}

package/dist/core/scanner.d.ts

@@ -0,0 +1,29 @@
+import type { ReadRequest } from "../contract/input.ts";
+import type { ReadLimits } from "../contract/limits.ts";
+import type { ReadLine } from "../contract/result.ts";
+import type { ScanBudget } from "./budget.ts";
+export type SelectionStop = "lines" | "bytes" | "budget" | null;
+export declare class LineScanner {
+    private readonly request;
+    private readonly limits;
+    private readonly budget;
+    readonly lines: ReadLine[];
+    totalLines: number;
+    viewBytes: number;
+    selectionStop: SelectionStop;
+    firstUnshown: number | null;
+    private budgetUsed;
+    private linePrefix;
+    private lineChars;
+    private lineHasContent;
+    private pendingCr;
+    constructor(request: ReadRequest, limits: Readonly<ReadLimits>, budget?: ScanBudget | null);
+    push(text: string): void;
+    finish(): void;
+    stopAtScanLimit(unscannedBytesKnown: boolean): void;
+    get clampedLines(): readonly number[];
+    get viewText(): string;
+    private retainingCurrentLine;
+    private admitByBudget;
+    private emitLine;
+}

package/dist/core/scanner.js

@@ -0,0 +1,124 @@
+const ENCODER = new TextEncoder();
+export class LineScanner {
+    request;
+    limits;
+    budget;
+    lines = [];
+    totalLines = 0;
+    viewBytes = 0;
+    selectionStop = null;
+    firstUnshown = null;
+    budgetUsed = 0;
+    linePrefix = "";
+    lineChars = 0;
+    lineHasContent = false;
+    pendingCr = false;
+    constructor(request, limits, budget = null) {
+        this.request = request;
+        this.limits = limits;
+        this.budget = budget;
+    }
+    push(text) {
+        for (const character of text) {
+            if (this.pendingCr) {
+                this.emitLine();
+                this.pendingCr = false;
+                if (character === "\n")
+                    continue;
+            }
+            if (character === "\r") {
+                this.pendingCr = true;
+            }
+            else if (character === "\n") {
+                this.emitLine();
+            }
+            else {
+                this.lineHasContent = true;
+                this.lineChars += 1;
+                if (this.lineChars <= this.limits.maxCharsPerLine && this.retainingCurrentLine()) {
+                    this.linePrefix += character;
+                }
+            }
+        }
+    }
+    finish() {
+        if (this.pendingCr) {
+            this.emitLine();
+            this.pendingCr = false;
+        }
+        else if (this.lineHasContent) {
+            this.emitLine();
+        }
+    }
+    stopAtScanLimit(unscannedBytesKnown) {
+        if (this.pendingCr) {
+            this.emitLine();
+            this.pendingCr = false;
+            if (unscannedBytesKnown && this.firstUnshown === null) {
+                const nextLine = this.totalLines + 1;
+                if (nextLine >= this.request.offset)
+                    this.firstUnshown = nextLine;
+            }
+            return;
+        }
+        if ((this.lineHasContent || unscannedBytesKnown) && this.firstUnshown === null) {
+            const currentLine = this.totalLines + 1;
+            if (currentLine >= this.request.offset)
+                this.firstUnshown = currentLine;
+        }
+    }
+    get clampedLines() {
+        return this.lines.filter((line) => line.clamped).map((line) => line.number);
+    }
+    get viewText() {
+        return this.lines.map((line) => line.text).join("\n");
+    }
+    retainingCurrentLine() {
+        return this.totalLines + 1 >= this.request.offset && this.selectionStop === null;
+    }
+    admitByBudget() {
+        if (this.budget === null)
+            return true;
+        const cost = this.budget.measure(this.linePrefix);
+        // The first line is always admitted, so a budget alone never gives an empty view.
+        if (this.lines.length > 0 && this.budgetUsed + cost > this.budget.max)
+            return false;
+        this.budgetUsed += cost;
+        return true;
+    }
+    emitLine() {
+        this.totalLines += 1;
+        const number = this.totalLines;
+        if (number >= this.request.offset && this.selectionStop === null) {
+            if (this.lines.length >= this.request.limit) {
+                this.selectionStop = "lines";
+                this.firstUnshown = number;
+            }
+            else {
+                const clamped = this.lineChars > this.limits.maxCharsPerLine;
+                const separatorBytes = this.lines.length === 0 ? 0 : 1;
+                const candidateBytes = ENCODER.encode(this.linePrefix).byteLength + separatorBytes;
+                if (this.viewBytes + candidateBytes > this.limits.maxViewBytes) {
+                    this.selectionStop = "bytes";
+                    this.firstUnshown = number;
+                }
+                else if (!this.admitByBudget()) {
+                    this.selectionStop = "budget";
+                    this.firstUnshown = number;
+                }
+                else {
+                    this.lines.push({
+                        number,
+                        text: this.linePrefix,
+                        clamped,
+                        sourceChars: clamped ? this.lineChars : null,
+                    });
+                    this.viewBytes += candidateBytes;
+                }
+            }
+        }
+        this.linePrefix = "";
+        this.lineChars = 0;
+        this.lineHasContent = false;
+    }
+}

package/dist/core/suggest.d.ts

@@ -0,0 +1,6 @@
+import type { FileSystem, FileSystemError } from "@better-fs-tools/fs";
+import type { ReadDependencies } from "../contract/deps.ts";
+import type { ReadRequest } from "../contract/input.ts";
+import type { ReadFailure } from "../contract/result.ts";
+import type { CallScope } from "./call-scope.ts";
+export declare function missOutcome<THost>(deps: ReadDependencies<THost>, request: ReadRequest, fs: FileSystem, scope: CallScope<THost>, missed: string, error: FileSystemError | null): Promise<ReadFailure>;

package/dist/core/suggest.js

@@ -0,0 +1,41 @@
+import { errorData, notFound } from "./outcomes.js";
+export async function missOutcome(deps, request, fs, scope, missed, error) {
+    const data = error === null ? {} : errorData(error);
+    const found = deps.suggest === null
+        ? null
+        : await suggestNames(deps.suggest, fs, scope, missed, deps.limits.maxSuggestions);
+    scope.checkAbort();
+    if (found === null)
+        return notFound(deps.messages, request, [], data);
+    const { suggestions, entriesTruncated } = found;
+    return notFound(deps.messages, request, suggestions, {
+        ...data,
+        ...(suggestions.length === 0 ? {} : { suggestions: [...suggestions] }),
+        ...(entriesTruncated ? { entriesTruncated } : {}),
+    });
+}
+async function suggestNames(suggest, fs, scope, missed, max) {
+    const listed = await scope.list("open", fs.paths.dirname(missed));
+    scope.throwHeld();
+    if (!listed.ok)
+        return null;
+    let names;
+    try {
+        names = suggest({
+            path: missed,
+            name: fs.paths.basename(missed),
+            entries: listed.entries,
+            entriesTruncated: listed.truncated,
+            max,
+            call: scope.call,
+        });
+    }
+    catch {
+        scope.checkAbort();
+        throw scope.extensionFailure("suggest");
+    }
+    if (!Array.isArray(names) || !names.every((name) => typeof name === "string")) {
+        throw scope.extensionFailure("suggest");
+    }
+    return { suggestions: names.slice(0, max), entriesTruncated: listed.truncated };
+}