@better-fs-tools/read 0.0.0-tegami-trusted-publish-setup → 0.2.0

package/dist/core/call-scope.js

@@ -0,0 +1,112 @@
+import { authorizeList } from "./authorize.js";
+import { AbortReadError } from "./cursor.js";
+import { isFileSystem, isStateStore } from "./deps.js";
+import { ReadStop, extensionFailed, messageOf } from "./outcomes.js";
+export class CallScope {
+    deps;
+    request;
+    call;
+    phase = "input";
+    allowNotes = [];
+    heldFailure = null;
+    resolvedFs = null;
+    resolvedState = undefined;
+    hook = null;
+    listed = new Set();
+    constructor(deps, request, call) {
+        this.deps = deps;
+        this.request = request;
+        this.call = call;
+    }
+    get signal() {
+        return this.call.signal;
+    }
+    enter(phase) {
+        this.phase = phase;
+    }
+    checkAbort() {
+        if (this.call.signal?.aborted)
+            throw new AbortReadError("aborted");
+    }
+    fileSystem() {
+        if (this.resolvedFs !== null)
+            return this.resolvedFs;
+        const { fs } = this.deps;
+        if (typeof fs !== "function")
+            return (this.resolvedFs = fs);
+        let produced;
+        try {
+            produced = fs(this.call);
+        }
+        catch {
+            throw this.extensionFailure("fs");
+        }
+        if (!isFileSystem(produced))
+            throw this.extensionFailure("fs");
+        return (this.resolvedFs = produced);
+    }
+    stateStore() {
+        if (this.resolvedState !== undefined)
+            return this.resolvedState;
+        const { state } = this.deps;
+        if (typeof state !== "function")
+            return (this.resolvedState = state);
+        let produced;
+        try {
+            produced = state(this.call);
+        }
+        catch {
+            throw this.extensionFailure("state");
+        }
+        if (produced !== null && !isStateStore(produced))
+            throw this.extensionFailure("state");
+        return (this.resolvedState = produced);
+    }
+    hookContext() {
+        if (this.hook !== null)
+            return this.hook;
+        const { limits, messages, digest, clock } = this.deps;
+        return (this.hook = Object.freeze({
+            tool: "read",
+            request: this.request,
+            limits,
+            messages,
+            digest,
+            clock,
+            call: this.call,
+        }));
+    }
+    /** One bounded, authorized listing per slot. Never throws: every failure is an error outcome. */
+    async list(slot, dir, displayPath = dir) {
+        if (this.listed.has(slot)) {
+            return { ok: false, error: { reason: "denied", detail: "listing budget spent" } };
+        }
+        this.listed.add(slot);
+        const fs = this.fileSystem();
+        if (typeof fs.list !== "function") {
+            return { ok: false, error: { reason: "unsupported", detail: "the backend cannot list" } };
+        }
+        const refused = await authorizeList(this.deps.authorize, this.request, dir, displayPath, this);
+        if (refused !== null)
+            return refused;
+        const signal = this.signal;
+        const limit = this.deps.limits.maxDirectoryEntries;
+        try {
+            return await fs.list(dir, signal === undefined ? { limit } : { limit, signal });
+        }
+        catch (error) {
+            return { ok: false, error: { reason: "io", detail: messageOf(error) } };
+        }
+    }
+    /** Keeps a listing failure out of reach of host code that might catch and ignore it. */
+    hold(stop) {
+        this.heldFailure ??= stop;
+    }
+    throwHeld() {
+        if (this.heldFailure !== null)
+            throw this.heldFailure;
+    }
+    extensionFailure(extension, id = null) {
+        return new ReadStop(extensionFailed(this.deps.messages, this.request, extension, this.phase, id));
+    }
+}

package/dist/core/classify.d.ts

@@ -0,0 +1,9 @@
+import type { Classification, ClassificationSample, Classifier, UnsupportedClassification } from "../contract/classify.ts";
+import type { ClassificationInfo } from "../contract/result.ts";
+export interface Decision<T> {
+    readonly classifier: string;
+    readonly classification: T;
+}
+export declare function classifySample(classifiers: readonly Classifier[], sample: ClassificationSample): Decision<Classification> | null;
+export declare function encodingRefusal(classifiers: readonly Classifier[], sample: ClassificationSample): Decision<UnsupportedClassification> | null;
+export declare function classificationInfo(decision: Decision<Classification>): ClassificationInfo;

package/dist/core/classify.js

@@ -0,0 +1,27 @@
+export function classifySample(classifiers, sample) {
+    for (const candidate of classifiers) {
+        const classification = candidate.classify(sample);
+        if (classification !== null)
+            return { classifier: candidate.id, classification };
+    }
+    return null;
+}
+export function encodingRefusal(classifiers, sample) {
+    for (const candidate of classifiers) {
+        const classification = candidate.encodingFailure?.(sample) ?? null;
+        if (classification !== null)
+            return { classifier: candidate.id, classification };
+    }
+    return null;
+}
+export function classificationInfo(decision) {
+    const { classifier, classification } = decision;
+    return {
+        kind: classification.kind,
+        classifier,
+        code: classification.kind === "unsupported" ? classification.code : null,
+        mimeType: classification.mimeType,
+        confidence: classification.confidence,
+        reasons: classification.reasons,
+    };
+}

package/dist/core/convert-source.d.ts

@@ -0,0 +1,29 @@
+import type { ClassificationSample } from "../contract/classify.ts";
+import type { Digest } from "../contract/digest.ts";
+import type { CallScope } from "./call-scope.ts";
+import type { ByteCursor } from "./cursor.ts";
+export declare class ConvertLimitError extends Error {
+}
+export declare class ConvertSource<THost> {
+    private readonly cursor;
+    private readonly sample;
+    private readonly limit;
+    private readonly scope;
+    bytesRead: number;
+    /** Stays set even when the converter catches the ConvertLimitError. */
+    exceeded: boolean;
+    reachedEof: boolean;
+    /** Stays set even when the converter catches the backend error. */
+    backendFailure: unknown;
+    private readonly hash;
+    private cachedContentId;
+    private sampleSent;
+    private started;
+    constructor(cursor: ByteCursor, sample: ClassificationSample, limit: number, digest: Digest | null, scope: CallScope<THost>);
+    bytes(): AsyncIterable<Uint8Array>;
+    drain(): Promise<void>;
+    contentId(): string | null;
+    private iterate;
+    private pull;
+    private nextChunk;
+}

package/dist/core/convert-source.js

@@ -0,0 +1,85 @@
+import { AbortReadError } from "./cursor.js";
+export class ConvertLimitError extends Error {
+}
+export class ConvertSource {
+    cursor;
+    sample;
+    limit;
+    scope;
+    bytesRead = 0;
+    /** Stays set even when the converter catches the ConvertLimitError. */
+    exceeded = false;
+    reachedEof = false;
+    /** Stays set even when the converter catches the backend error. */
+    backendFailure = null;
+    hash;
+    cachedContentId;
+    sampleSent = false;
+    started = false;
+    constructor(cursor, sample, limit, digest, scope) {
+        this.cursor = cursor;
+        this.sample = sample;
+        this.limit = limit;
+        this.scope = scope;
+        this.hash = digest === null ? null : digest.create();
+    }
+    bytes() {
+        if (this.started)
+            throw new TypeError("the converter source is single use");
+        this.started = true;
+        return this.iterate();
+    }
+    async drain() {
+        this.started = true;
+        while ((await this.pull()) !== null) { }
+    }
+    contentId() {
+        if (!this.reachedEof)
+            return null;
+        this.cachedContentId ??= this.hash?.digest() ?? null;
+        return this.cachedContentId;
+    }
+    async *iterate() {
+        for (;;) {
+            const chunk = await this.pull();
+            if (chunk === null)
+                return;
+            yield chunk;
+        }
+    }
+    async pull() {
+        if (this.exceeded)
+            throw new ConvertLimitError("conversion limit exceeded");
+        if (this.reachedEof)
+            return null;
+        this.scope.checkAbort();
+        const chunk = await this.nextChunk();
+        if (chunk === null) {
+            this.reachedEof = true;
+            return null;
+        }
+        if (this.bytesRead + chunk.byteLength > this.limit) {
+            this.exceeded = true;
+            throw new ConvertLimitError("conversion limit exceeded");
+        }
+        this.bytesRead += chunk.byteLength;
+        this.hash?.update(chunk);
+        return chunk;
+    }
+    async nextChunk() {
+        if (!this.sampleSent) {
+            this.sampleSent = true;
+            if (this.sample.bytes.byteLength > 0)
+                return this.sample.bytes;
+        }
+        try {
+            const item = await this.cursor.next(this.scope.signal);
+            return item.done ? null : item.value;
+        }
+        catch (error) {
+            if (!(error instanceof AbortReadError))
+                this.backendFailure ??= error;
+            throw error;
+        }
+    }
+}

package/dist/core/convert.d.ts

@@ -0,0 +1,23 @@
+import type { FileSystem, OpenFile } from "@better-fs-tools/fs";
+import type { Classification, ClassificationSample } from "../contract/classify.ts";
+import type { ReadDependencies } from "../contract/deps.ts";
+import type { FileConverter } from "../contract/extensions.ts";
+import type { ReadRequest } from "../contract/input.ts";
+import type { FileInfo, ReadReport } from "../contract/result.ts";
+import type { CallScope } from "./call-scope.ts";
+import type { Decision } from "./classify.ts";
+import type { ByteCursor } from "./cursor.ts";
+export declare function selectFileConverter<THost>(scope: CallScope<THost>, decision: Decision<Classification>, sample: ClassificationSample): FileConverter<THost> | null;
+export interface ConvertFileInput<THost> {
+    readonly deps: ReadDependencies<THost>;
+    readonly fs: FileSystem;
+    readonly request: ReadRequest;
+    readonly file: FileInfo;
+    readonly handle: OpenFile;
+    readonly cursor: ByteCursor;
+    readonly sample: ClassificationSample;
+    readonly decision: Decision<Classification>;
+    readonly converter: FileConverter<THost>;
+    readonly scope: CallScope<THost>;
+}
+export declare function convertFile<THost>(input: ConvertFileInput<THost>): Promise<ReadReport>;

package/dist/core/convert.js

@@ -0,0 +1,132 @@
+import { classificationInfo } from "./classify.js";
+import { ConvertSource } from "./convert-source.js";
+import { checkConvertOutcome, firstMediaType, mediaBytes, refusedOutcome, scanConvertedText, tooLarge, } from "./converted.js";
+import { AbortReadError, raceAbort } from "./cursor.js";
+import { extensionId } from "./extension-error.js";
+import { buildObservation } from "./observation.js";
+import { ReadStop } from "./outcomes.js";
+import { textOutcome } from "./text-outcome.js";
+import { checkSize, verifyHandle } from "./verify.js";
+export function selectFileConverter(scope, decision, sample) {
+    const match = Object.freeze({
+        classification: decision.classification,
+        classifier: decision.classifier,
+        sample,
+    });
+    for (const converter of scope.deps.converters) {
+        if (converter.target !== "file")
+            continue;
+        scope.enter("conversion");
+        let accepted;
+        try {
+            accepted = converter.accepts(match);
+        }
+        catch (error) {
+            throw scope.extensionFailure("converters", extensionId(converter, error));
+        }
+        if (accepted === true)
+            return converter;
+    }
+    return null;
+}
+export async function convertFile(input) {
+    const { deps, request, file, handle, sample, decision, converter, scope } = input;
+    const { limits, messages } = deps;
+    const classification = classificationInfo(decision);
+    const source = new ConvertSource(input.cursor, sample, limits.maxConvertBytes, deps.digest, scope);
+    const throwIfSourceFailed = () => {
+        scope.checkAbort();
+        if (source.exceeded) {
+            throw new ReadStop(tooLarge(messages, request, file, classification, "convert", limits.maxConvertBytes));
+        }
+        if (source.backendFailure !== null)
+            throw source.backendFailure;
+    };
+    const failed = (error) => {
+        throwIfSourceFailed();
+        return scope.extensionFailure("converters", extensionId(converter, error));
+    };
+    scope.enter("conversion");
+    const convertInput = Object.freeze({
+        info: handle.info,
+        classification: decision.classification,
+        sample,
+        bytes: () => source.bytes(),
+    });
+    let produced;
+    try {
+        produced = await raceAbort(() => converter.convert(convertInput, scope.hookContext()), scope.signal);
+    }
+    catch (error) {
+        throw failed(error);
+    }
+    throwIfSourceFailed();
+    const outcome = checkConvertOutcome(produced);
+    if (outcome === null)
+        throw failed();
+    if (outcome.kind === "refuse") {
+        return refusedOutcome(request, file, classification, outcome.code, outcome.note);
+    }
+    if (outcome.kind === "media") {
+        if (mediaBytes(outcome.parts) > limits.maxMediaBytes) {
+            return tooLarge(messages, request, file, classification, "media", limits.maxMediaBytes);
+        }
+        await drainAndVerify(source, handle, input, throwIfSourceFailed);
+        return mediaOutcome(input, classification, outcome.parts, outcome.notes ?? [], source);
+    }
+    let scan;
+    try {
+        scan = await scanConvertedText(outcome.text, request, file, scope);
+    }
+    catch (error) {
+        if (error instanceof ReadStop || error instanceof AbortReadError)
+            throw error;
+        throw failed(error);
+    }
+    await drainAndVerify(source, handle, input, throwIfSourceFailed);
+    return textOutcome({
+        deps,
+        fs: input.fs,
+        request,
+        file,
+        classification,
+        conversion: { converter: converter.id, mimeType: outcome.mimeType },
+        scan,
+        contentId: source.contentId(),
+        notes: outcome.notes ?? [],
+    });
+}
+async function drainAndVerify(source, handle, input, throwIfSourceFailed) {
+    const { deps, request, file, scope } = input;
+    try {
+        await source.drain();
+    }
+    catch (error) {
+        throwIfSourceFailed();
+        throw error;
+    }
+    scope.enter("verification");
+    checkSize(deps.messages, request, file, source.bytesRead);
+    await verifyHandle(handle, deps.messages, request, file, scope);
+}
+function mediaOutcome(input, classification, parts, notes, source) {
+    const { deps, request, file, converter } = input;
+    return {
+        tool: "read",
+        status: "media",
+        request,
+        file,
+        classification,
+        conversion: { converter: converter.id, mimeType: firstMediaType(parts) },
+        parts,
+        observation: buildObservation(deps.digest, {
+            file,
+            contentId: source.contentId(),
+            view: parts,
+            observedAt: deps.clock().toISOString(),
+            // Media is not source text, so it cannot back a text edit.
+            wholeFileVisible: false,
+        }),
+        notes,
+    };
+}

package/dist/core/converted.d.ts

@@ -0,0 +1,18 @@
+import type { ConvertOutcome } from "../contract/extensions.ts";
+import type { ReadRequest } from "../contract/input.ts";
+import type { ReadMessageCatalog } from "../contract/messages.ts";
+import type { ClassificationInfo, ContentPart, FileInfo, ReadNote, ReadUnsupported } from "../contract/result.ts";
+import type { CallScope } from "./call-scope.ts";
+import type { ScanOutcome } from "./scan.ts";
+export declare class ConverterOutputError extends Error {
+}
+export type ScannedText = Extract<ScanOutcome, {
+    kind: "scanned";
+}>;
+export declare function checkConvertOutcome(value: unknown): ConvertOutcome | null;
+export declare function scanConvertedText<THost>(text: string | AsyncIterable<string>, request: ReadRequest, file: FileInfo, scope: CallScope<THost>): Promise<ScannedText>;
+export declare function mediaBytes(parts: readonly ContentPart[]): number;
+export declare function firstMediaType(parts: readonly ContentPart[]): string | null;
+export declare function refusedOutcome(request: ReadRequest, file: FileInfo, classification: ClassificationInfo, code: string, note: ReadNote): ReadUnsupported;
+export declare function tooLarge(messages: Readonly<ReadMessageCatalog>, request: ReadRequest, file: FileInfo, classification: ClassificationInfo, stage: "convert" | "media", limit: number): ReadUnsupported;
+export declare function isContentPart(value: unknown): value is ContentPart;

package/dist/core/converted.js

@@ -0,0 +1,111 @@
+import { ByteCursor } from "./cursor.js";
+import { isRecord } from "./input.js";
+import { isNote } from "./outcomes.js";
+import { scanText } from "./scan.js";
+const ENCODER = new TextEncoder();
+export class ConverterOutputError extends Error {
+}
+export function checkConvertOutcome(value) {
+    if (!isRecord(value))
+        return null;
+    switch (value.kind) {
+        case "text": {
+            const { text, mimeType } = value;
+            const textOk = typeof text === "string" || isAsyncIterable(text);
+            const mimeOk = mimeType === null || typeof mimeType === "string";
+            return textOk && mimeOk && notesOk(value.notes) ? value : null;
+        }
+        case "media": {
+            const { parts } = value;
+            const partsOk = Array.isArray(parts) && parts.every(isContentPart);
+            return partsOk && notesOk(value.notes) ? value : null;
+        }
+        case "refuse":
+            return typeof value.code === "string" && value.code !== "" && isNote(value.note)
+                ? value
+                : null;
+        default:
+            return null;
+    }
+}
+export async function scanConvertedText(text, request, file, scope) {
+    const cursor = new ByteCursor(encoded(text));
+    try {
+        const scan = await scanText({
+            cursor,
+            sample: { bytes: new Uint8Array(0), complete: false, path: file.displayPath, mimeType: null },
+            request,
+            limits: scope.deps.limits,
+            digest: null,
+            scope,
+        });
+        // Encoded JS strings are valid UTF-8.
+        if (scan.kind !== "scanned")
+            throw new ConverterOutputError("converted text is not UTF-8");
+        return scan;
+    }
+    finally {
+        await cursor.close();
+    }
+}
+export function mediaBytes(parts) {
+    let total = 0;
+    for (const part of parts)
+        if (part.type === "media")
+            total += part.data.byteLength;
+    return total;
+}
+export function firstMediaType(parts) {
+    for (const part of parts)
+        if (part.type === "media")
+            return part.mediaType;
+    return null;
+}
+export function refusedOutcome(request, file, classification, code, note) {
+    return {
+        tool: "read",
+        status: "unsupported",
+        code,
+        request,
+        file,
+        classification,
+        notes: [note],
+    };
+}
+export function tooLarge(messages, request, file, classification, stage, limit) {
+    return refusedOutcome(request, file, classification, "TOO_LARGE", {
+        code: "too-large",
+        severity: "warning",
+        message: messages.tooLarge({ request, stage, limit }),
+        data: { stage, limit },
+    });
+}
+async function* encoded(text) {
+    if (typeof text === "string") {
+        yield ENCODER.encode(text);
+        return;
+    }
+    for await (const chunk of text) {
+        if (typeof chunk !== "string")
+            throw new ConverterOutputError("text chunks must be strings");
+        yield ENCODER.encode(chunk);
+    }
+}
+function isAsyncIterable(value) {
+    return (value !== null &&
+        typeof value === "object" &&
+        typeof value[Symbol.asyncIterator] === "function");
+}
+function notesOk(notes) {
+    return notes === undefined || (Array.isArray(notes) && notes.every(isNote));
+}
+export function isContentPart(value) {
+    if (!isRecord(value))
+        return false;
+    if (value.type === "text")
+        return typeof value.text === "string";
+    return (value.type === "media" &&
+        typeof value.mediaType === "string" &&
+        value.data instanceof Uint8Array &&
+        (value.name === undefined || typeof value.name === "string"));
+}

package/dist/core/create-read-tool.d.ts

@@ -0,0 +1,7 @@
+import type { ReadTool } from "../contract/context.ts";
+import type { ReadToolDeps, StateNeedsDigest } from "../contract/deps.ts";
+/**
+ * Creates the read tool. Throws TypeError on an unknown or malformed dependency,
+ * a missing `fs`, an empty `classifiers` list, or a `state` without a `digest`.
+ */
+export declare function createReadTool<THost = undefined>(deps: ReadToolDeps<THost> & StateNeedsDigest): ReadTool<THost>;

package/dist/core/create-read-tool.js

@@ -0,0 +1,18 @@
+import { resolveDependencies } from "./deps.js";
+import { isRecord } from "./input.js";
+import { runRead } from "./pipeline.js";
+/**
+ * Creates the read tool. Throws TypeError on an unknown or malformed dependency,
+ * a missing `fs`, an empty `classifiers` list, or a `state` without a `digest`.
+ */
+export function createReadTool(deps) {
+    const resolved = resolveDependencies(deps);
+    const read = async (input, ctx) => {
+        if (ctx !== undefined && !isRecord(ctx)) {
+            throw new TypeError("read context must be an object");
+        }
+        const call = ctx ?? {};
+        return runRead(resolved, input, call);
+    };
+    return read;
+}

package/dist/core/cursor.d.ts

@@ -0,0 +1,14 @@
+export declare class AbortReadError extends Error {
+}
+export declare class ByteCursor {
+    private iterator;
+    private chunk;
+    private pushed;
+    private offset;
+    constructor(source: AsyncIterable<Uint8Array>);
+    unshift(piece: Uint8Array): void;
+    next(signal: AbortSignal | undefined): Promise<IteratorResult<Uint8Array>>;
+    close(): Promise<void>;
+    private pull;
+}
+export declare function raceAbort<T>(start: () => T | Promise<T>, signal: AbortSignal | undefined): Promise<T>;