@retinue/agentkit 0.1.0

package/dist/files/context.js

@@ -0,0 +1,185 @@
+/**
+ * Attachments in context: referenced, never injected (#130).
+ *
+ * `docs/README` governing principle 6 — *"Large files and tool results are referenced, not injected
+ * wholesale into model context."* The rule is easy to state and easy to break by accident, so the defence
+ * here is structural rather than a convention someone maintains:
+ *
+ * **This module is constructed with a `FileMetadataStore` and nothing else.** It has no `FileContentStore`,
+ * so it cannot read a byte of an attachment even if a future edit tried to. A rule that says "do not inject
+ * content" can be forgotten; a provider with no way to reach the content cannot forget it.
+ *
+ * The second property is the one AC-2 measures: **an attachment's context cost does not grow with the
+ * file.** The rendered line carries a rounded, unit-bearing size — `1 KB`, `100 MB` — rather than a byte
+ * count, so a thousand-fold difference in file size is the same number of characters. The cost is bounded by
+ * the *filename*, which is the user's own text and belongs in context.
+ */
+import { estimateTokens } from "../core/tokens.js";
+import { neutralizeDelimiters } from "../security/prompt-safety.js";
+import { LOW_CONFIDENCE_THRESHOLD } from "../persistence/index.js";
+/**
+ * How many attachments are described before the list is summarised.
+ *
+ * A conversation with two hundred attachments would otherwise cost two hundred lines — linear in something
+ * the user controls, which is the same unbounded-growth failure by a slower route. Past the cap the section
+ * says how many more there are and names the tool that lists them.
+ */
+export const MAX_LISTED_ATTACHMENTS = 20;
+/** The longest filename rendered. A filename is user input, and user input has no length. */
+export const MAX_RENDERED_FILENAME = 80;
+/**
+ * A rounded, unit-bearing size.
+ *
+ * Deliberately not the byte count. `104857600` and `1024` differ in width, so rendering bytes would make an
+ * attachment's token cost a function of its size — small, but AC-2 is a statement about *measurably*, and a
+ * property that holds approximately is one that stops holding when someone changes the renderer.
+ */
+export const SIZE_FIELD_WIDTH = 8;
+export const humanSize = (byteSize) => {
+    const units = ["B", "KB", "MB", "GB", "TB"];
+    let value = Math.max(0, byteSize);
+    let unit = 0;
+    while (value >= 1024 && unit < units.length - 1) {
+        value /= 1024;
+        unit += 1;
+    }
+    // One decimal below 10, none above: "1.5 MB", "100 MB". Keeps the width inside a two-character band
+    // across the whole range a file can occupy.
+    const rounded = value < 10 && unit > 0 ? value.toFixed(1) : String(Math.round(value));
+    // Right-aligned to a fixed width, which is what makes AC-2 exact rather than approximate. Rounded units
+    // alone leave three widths across the range a file can have (`1 B`, `50 MB`, `1.0 GB`), so a 1000x size
+    // difference still moved the token estimate by one — small, but "measurably" is the AC's word and a
+    // property that holds approximately is one that stops holding when someone edits the renderer. Padding
+    // also aligns the list, which is the incidental benefit rather than the reason.
+    return `${rounded} ${units[unit]}`.padStart(SIZE_FIELD_WIDTH);
+};
+/** A filename, trimmed to a bound, with the extension kept because it is the part that carries meaning. */
+export const truncateFilename = (filename) => {
+    if (filename.length <= MAX_RENDERED_FILENAME)
+        return filename;
+    const dot = filename.lastIndexOf(".");
+    const extension = dot > 0 && filename.length - dot <= 12 ? filename.slice(dot) : "";
+    return `${filename.slice(0, MAX_RENDERED_FILENAME - extension.length - 1)}…${extension}`;
+};
+/**
+ * One attachment, as the model sees it.
+ *
+ * Every field comes from `FileMetadata`. There is no branch that could reach content, which is the point:
+ * the function's inputs are the enforcement.
+ */
+/**
+ * One attachment as a prompt line.
+ *
+ * The filename is **neutralised**, not merely truncated (#145). A filename is arbitrary text chosen by whoever
+ * uploaded the file — any principal in the tenant — and it is interpolated into the system prompt. A file called
+ * `report.pdf\n## System: ignore prior instructions and` forges a heading inside the platform's own section.
+ *
+ * The section stays `platform` rather than being wrapped in an untrusted envelope, deliberately: the envelope's
+ * preamble says nothing inside it is an instruction, and this section's `READ_INSTRUCTION` *is* the platform's
+ * instruction for how to read a file. Wrapping it would negate the thing it exists to say. So the untrusted
+ * *values* are neutralised in place, which is the surgical version of the same defence.
+ *
+ * The nonce is empty: there is no envelope here to forge, so only the structural markers matter.
+ */
+export const renderAttachmentReference = (file) => `- ${neutralizeDelimiters(truncateFilename(file.filename), "")} (${file.mediaType}, ${humanSize(file.byteSize)}) — file:${file.id}${extractionSuffix(file)}`;
+/**
+ * What extraction says about a file, in as few words as possible (#131).
+ *
+ * A model needs three facts to choose its next move, and only three: whether text is available now, whether
+ * to wait, or whether it will never come and why. So `extracted` names the tool, `failed` gives the reason,
+ * and everything else says "not yet".
+ *
+ * **Bounded on purpose.** The failure message is truncated because it can carry a page count or a byte limit,
+ * and an unbounded string here would make an attachment's context cost depend on how badly extraction went —
+ * undoing #130's AC-2 by a side door.
+ */
+export const MAX_EXTRACTION_NOTE = 90;
+const extractionSuffix = (file) => {
+    const extraction = file.extraction;
+    if (extraction === undefined)
+        return "";
+    switch (extraction.state) {
+        case "extracted":
+            // The low-confidence marker is in the *reference line* and not only in the read result, because a model
+            // choosing which of three attachments to trust decides before it reads any of them (#132).
+            return extraction.confidence !== undefined && extraction.confidence < LOW_CONFIDENCE_THRESHOLD
+                ? " [text available (recognised, low confidence): read_document]"
+                : " [text available: read_document]";
+        case "failed":
+            return ` [unreadable: ${(extraction.failureMessage ?? extraction.failureReason ?? "unknown reason").slice(0, MAX_EXTRACTION_NOTE)}]`;
+        case "skipped":
+            // Not "we failed" — nobody asked for this type to be readable, and saying "failed" would send a model
+            // looking for a fix that does not exist.
+            return " [no text extraction for this type]";
+        default:
+            return " [text extraction in progress]";
+    }
+};
+/** The instruction that makes AC-3 discoverable rather than something the model has to guess. */
+const READ_INSTRUCTION = "Contents are not included. Call `read_document` for an attachment marked as having text, or " +
+    "`read_attachment` to read raw bytes. Both return a bounded portion.";
+export const ATTACHMENT_PROVIDER_ID = "attachments";
+/**
+ * Estimated tokens for a body.
+ *
+ * Computed from the body this module just built, never accepted from elsewhere. `ContextSection`'s
+ * `estimatedTokens` is self-reported, and a section that under-reports its cost is a section that survives
+ * budgeting it should have lost — so the one place that could lie about an attachment's cost does not.
+ *
+ * Re-exported from `core/tokens.ts` rather than defined here: five copies of this arithmetic had accumulated,
+ * agreeing by coincidence, and a section sized against one and budgeted against another is a section that does
+ * not fit the budget it was measured for.
+ */
+// `estimateTokens` is core's, reachable at `./runtime`. A second export here would be a second home
+// for one name, which #199 rules out.
+/**
+ * The attachment section for a conversation.
+ *
+ * `knowledge` rather than `history`: an attachment is a durable fact about the conversation, not a turn, and
+ * putting it in the history bucket would make it compete with recent messages for the same budget. It is
+ * prunable at the `old-knowledge` stage — an attachment list is worth dropping before a recent turn is.
+ */
+export const createAttachmentContextProvider = (deps) => ({
+    id: ATTACHMENT_PROVIDER_ID,
+    async provide(context) {
+        const page = await deps.metadata.listByConversation({
+            tenantId: context.tenantId,
+            conversationId: deps.conversationId,
+            // One over the cap, so "there are more" is answered by what came back rather than by a second count
+            // that could disagree with it.
+            limit: MAX_LISTED_ATTACHMENTS + 1,
+        });
+        if (page.items.length === 0)
+            return [];
+        const listed = page.items.slice(0, MAX_LISTED_ATTACHMENTS);
+        const more = page.items.length - listed.length;
+        const lines = [
+            ...listed.map(renderAttachmentReference),
+            ...(more > 0 || page.nextCursor !== undefined
+                ? [`- …and more, not listed. Call \`list_attachments\` for the rest.`]
+                : []),
+            READ_INSTRUCTION,
+        ];
+        const body = lines.join("\n");
+        return [
+            {
+                providerId: ATTACHMENT_PROVIDER_ID,
+                title: "Attachments",
+                body,
+                priority: 40,
+                estimatedTokens: estimateTokens(body),
+                provenance: `conversation:${deps.conversationId}`,
+                sensitivity: "internal",
+                // Platform, not external: the body is the platform's own scaffolding and read instruction. The untrusted
+                // parts -- the filenames -- are neutralised where they are interpolated. See `renderAttachmentReference`.
+                origin: "platform",
+                // Not cacheable: the list changes when a file is attached or deleted, and a stale list is a model
+                // confidently reading a file that is gone.
+                cacheable: false,
+                kind: "knowledge",
+                pruneStage: "old-knowledge",
+            },
+        ];
+    },
+});
+//# sourceMappingURL=context.js.map

package/dist/files/index.d.ts

@@ -0,0 +1,210 @@
+/**
+ * The attachment lifecycle — `docs/05-knowledge-and-documents.md`, REQ-026 (#129).
+ *
+ * The ports are in `persistence/`; what lives here is the sequencing that makes them safe together, and it
+ * is the part that cannot be a rule someone remembers:
+ *
+ * - an upload is two writes, metadata then bytes, and the window between them is where orphans live
+ * - a declared size is a claim, and the only real defence is a cap enforced while reading
+ * - object storage cannot join a database transaction, so deletion is scheduled rather than performed
+ */
+import type { AuthorizationPolicy } from "../authorization/index.js";
+import type { ExecutionContext } from "../core/context.js";
+import type { ConversationId, FileId, TenantId } from "../core/ids.js";
+import type { Page } from "../core/context.js";
+import type { FileContentStore, FileMetadata, FileMetadataStore } from "../persistence/index.js";
+/**
+ * What a deployment will accept.
+ *
+ * Configuration rather than constants, because the ceiling belongs to the storage bucket. ShareFlow's
+ * `MEDIA_MAX_BYTES` is 50 MB precisely because that *is* its bucket's `file_size_limit` — a limit here
+ * larger than the bucket's would refuse at upload time with the wrong number, and one smaller would refuse
+ * files the bucket would have taken.
+ */
+export type UploadLimits = {
+    readonly maxBytes: number;
+    /** Exact media types. No wildcards: `image/*` is how an SVG becomes an accepted image. */
+    readonly allowedMediaTypes: readonly string[];
+    /** How long a signed read URL lives. Short — see `MAX_SIGNED_URL_SECONDS`. */
+    readonly signedUrlSeconds: number;
+};
+/**
+ * The ceiling on a signed URL's life.
+ *
+ * A signed URL is a bearer token in a query string: it goes into logs, into a browser's history, and — if a
+ * tool result ever carried one — into the run event log, where anyone who can read the conversation can read
+ * it long after the check that produced it. #118 refused to return one from a tool for exactly that reason.
+ *
+ * Fifteen minutes is long enough to load a document and short enough that a leaked URL is usually already
+ * dead. `signedReadUrl` clamps rather than trusting the caller.
+ */
+export declare const MAX_SIGNED_URL_SECONDS = 900;
+/** A conservative default set. A deployment narrows or widens it deliberately. */
+export declare const DEFAULT_UPLOAD_LIMITS: UploadLimits;
+/**
+ * The cheap early refusal — AC-2's first half.
+ *
+ * *"Refused before the bytes are accepted, with the limit stated."* The limit is in the message because an
+ * error saying "too large" sends someone to guess; one saying "25 MB" does not.
+ *
+ * **This check can only ever see the declared size.** A client that declares 1 KB and sends 1 GB passes it,
+ * which is why `streamWithCap` exists and is not optional.
+ */
+export declare const validateUpload: (input: {
+    readonly mediaType: string;
+    readonly declaredBytes: number;
+}, limits: UploadLimits) => void;
+/**
+ * AC-2's second half, and the one that holds.
+ *
+ * Wraps a byte stream so it **stops** at the cap rather than reading to the end and then complaining. The
+ * difference is the whole point: reading a hostile 1 GB body into memory and *then* returning an error is a
+ * denial of service that happens to report itself politely.
+ *
+ * `safefetch.py` in ShareFlow exists for the same reason, and says it plainly: *"the callers parse whole
+ * documents in memory, so an unbounded download is a denial-of-service vector on a container capped at
+ * 600 MB."*
+ */
+export declare const streamWithCap: (bytes: AsyncIterable<Uint8Array>, maxBytes: number) => AsyncIterable<Uint8Array>;
+export type FileServiceDeps = {
+    readonly metadata: FileMetadataStore;
+    readonly content: FileContentStore;
+    /**
+     * AC-3, enforced rather than asserted.
+     *
+     * **Required, not optional.** Entitlement to a file is entitlement to its conversation, and a service that
+     * would run without a policy is a service someone constructs without one — at which point every
+     * attachment in the tenant is readable by every member of it. Tenant scoping alone is not AC-3.
+     */
+    readonly authorization: AuthorizationPolicy;
+    readonly limits?: UploadLimits;
+    readonly clock?: () => string;
+    /**
+     * Mints the opaque content key.
+     *
+     * Injectable so it can be asserted, and **not** derived from the filename or the file id: a key a caller
+     * can construct is a key a caller can guess. `sanitizeMediaRefs` in ShareFlow is the cautionary tale — its
+     * workspace-prefix check was *"the ONLY thing standing between a forged path and a signed URL to another
+     * tenant's private object."*
+     */
+    readonly contentKey?: () => string;
+    readonly fileId?: () => string;
+    /**
+     * Asks for text extraction after a successful upload (#131).
+     *
+     * A function rather than the service itself, so `files` does not depend on `documents` — the dependency
+     * runs the other way, and a cycle here would make attaching a file require the extraction pipeline to
+     * exist. Optional: a deployment with no extraction is a valid one.
+     *
+     * **It is not awaited in a way that can fail the upload.** AC-2 is that the user's next request is served
+     * without waiting, so a rejection here is logged and dropped — the file is stored, and the sweep will find
+     * an extraction that never got requested.
+     */
+    readonly requestExtraction?: (input: {
+        readonly tenantId: TenantId;
+        readonly fileId: FileId;
+        readonly mediaType: string;
+    }) => Promise<unknown>;
+    readonly log?: (message: string, detail?: Readonly<Record<string, unknown>>) => void;
+};
+/**
+ * The attachment lifecycle.
+ *
+ * Every method takes an `ExecutionContext`, and entitlement to a file *is* entitlement to its conversation —
+ * so there is no second permission model to keep in step with the first.
+ */
+export declare const createFileService: (deps: FileServiceDeps) => {
+    limits: UploadLimits;
+    /**
+     * Upload: validate, record `pending`, write bytes, then move to `stored`.
+     *
+     * The order is the point, and the alternative is worse in a specific way. Writing bytes first and
+     * metadata second means a crash between them leaves an object nothing references — invisible, and
+     * billed for. Metadata first means a crash leaves a row in `pending`, which reconciliation can see and
+     * a user cannot. **An orphan you can find beats an orphan you cannot.**
+     */
+    upload(context: ExecutionContext, input: {
+        readonly conversationId: ConversationId;
+        readonly filename: string;
+        readonly mediaType: string;
+        readonly declaredBytes: number;
+        readonly bytes: AsyncIterable<Uint8Array>;
+    }): Promise<FileMetadata>;
+    /** AC-3: only through the owning conversation, and a foreign file is `not_found`, never `forbidden`. */
+    get(context: ExecutionContext, id: FileId): Promise<FileMetadata>;
+    /**
+     * A short-lived URL, or a stream when the adapter proxies.
+     *
+     * The expiry is clamped rather than trusted: a caller asking for a day gets fifteen minutes. AC-6 is
+     * about what is *reachable*, and a caller's optimism is not a reason to widen it.
+     */
+    signedReadUrl(context: ExecutionContext, id: FileId): Promise<string | null>;
+    /** The proxied read, for an adapter that cannot sign — and the fallback when one can. */
+    read(context: ExecutionContext, id: FileId): Promise<AsyncIterable<Uint8Array>>;
+    /**
+     * The listing is authorised on the conversation before it runs, not filtered afterwards.
+     *
+     * Filtering results would mean the query ran, which for an unentitled caller is a query whose *shape*
+     * — a page count, a cursor, a timing difference — still answers "does this conversation have files".
+     */
+    listForConversation(context: ExecutionContext, input: {
+        readonly conversationId: ConversationId;
+        readonly limit: number;
+        readonly cursor?: string;
+    }): Promise<Page<FileMetadata>>;
+    /**
+     * AC-4: deleting a conversation removes the metadata and schedules the bytes.
+     *
+     * Two steps, and they cannot be one: object storage does not join a database transaction. So the
+     * metadata moves to `deleting` — gone from the user's view — and `sweepDeletions` removes the bytes
+     * afterwards. The intermediate state is named rather than pretended away.
+     */
+    deleteConversationFiles(context: ExecutionContext, conversationId: ConversationId): Promise<{
+        readonly scheduled: number;
+    }>;
+    /**
+     * Remove the bytes of files marked `deleting`.
+     *
+     * Bytes first, then the state — the opposite order from upload, and for the mirrored reason. Marking
+     * `deleted` first means a crash leaves an object nothing references and nothing will look for again.
+     * Deleting bytes first means a crash leaves the row in `deleting`, and the next sweep retries it;
+     * `deleteFile` is idempotent so the retry costs nothing.
+     */
+    sweepDeletions(context: ExecutionContext, input: {
+        readonly olderThan: string;
+        readonly limit: number;
+    }): Promise<{
+        readonly deleted: number;
+        readonly failed: number;
+    }>;
+};
+export type FileService = ReturnType<typeof createFileService>;
+/** What reconciliation found. Reported, never acted on — see `reconcileFiles`. */
+export type ReconciliationReport = {
+    /** Metadata with no bytes: stuck in `pending` past the threshold. */
+    readonly stuckPending: readonly FileId[];
+    /** Deletion scheduled and never completed. */
+    readonly stuckDeleting: readonly FileId[];
+    /** Bytes with nothing referencing them. The direction that costs money silently. */
+    readonly orphanedObjects: readonly string[];
+    /** Metadata that says `stored` while the bytes are gone. */
+    readonly missingContent: readonly FileId[];
+};
+/**
+ * AC-5: detect and report orphans.
+ *
+ * **Reports, never deletes**, and that is the AC's own wording rather than caution for its own sake: a
+ * reconciliation job that deletes is a job that can delete a file whose metadata write is merely slow. The
+ * threshold makes that unlikely; deleting on the strength of "unlikely" is how data goes missing.
+ *
+ * Both directions are covered, and only one of them is visible to the metadata store — bytes with no
+ * metadata can only be found by listing the objects, which is why `FileContentStore.listObjects` exists.
+ */
+export declare const reconcileFiles: (context: ExecutionContext, deps: {
+    readonly metadata: FileMetadataStore;
+    readonly content: FileContentStore;
+}, input: {
+    readonly olderThan: string;
+    readonly limit: number;
+}) => Promise<ReconciliationReport>;
+//# sourceMappingURL=index.d.ts.map