@ultimat3/storage 30.9.0 → 31.0.0

package/src/upload-scan.ts

@@ -0,0 +1,91 @@
+// Single responsibility: the scan step of `promoteAttachment` — the installed `UploadScanner` asked
+// about a per-ATTEMPT copy of the upload, and its four outcomes: clean (the attempt is what gets
+// attached), infected (attempt and upload deleted, refused), hold (attempt deleted, upload kept,
+// refused until cleared) and no verdict (as hold, unless the policy accepts unscanned).
+
+import { assert, renderThrowable } from '@ultimat3/core';
+import type { StorageDriver, StorageObject } from './driver';
+import { scanUnavailable, uploadHeld, uploadInfected } from './errors-scan';
+import type { UploadPolicy } from './upload';
+import { uploadScanner } from './upload-scanner';
+
+/**
+ * `pending/scan/<random>-<name>`: a key of THIS confirm's own (`attachment.ts` builds it).
+ * Server-named, so no presigned URL reaches it and one scan of it is final; never the row's key, so
+ * a refusal deletes nothing a doubled confirm attached; inside `pending/`, so an attempt a crash
+ * left behind is an orphan `sweepOrphans` collects.
+ */
+export const SCAN_SEGMENT = 'scan';
+
+export interface ScanAttemptInput {
+  readonly disk: StorageDriver;
+  /** The upload as it was confirmed: what a refusal names and a retry promotes again. */
+  readonly pendingKey: string;
+  /** The attempt copy under `SCAN_SEGMENT`, already sniffed. */
+  readonly attempt: StorageObject;
+  readonly policy: UploadPolicy;
+}
+
+/** Deleted, or reported as not: a cleanup that fails never replaces the refusal it follows. */
+async function removeAll(disk: StorageDriver, keys: readonly string[]): Promise<boolean> {
+  let removed = true;
+  for (const key of keys) {
+    try {
+      await disk.delete(key);
+    } catch {
+      removed = false;
+    }
+  }
+  return removed;
+}
+
+/** True when a scanner is installed — `promoteAttachment` stages an attempt copy only then. */
+export const scanning = (): boolean => uploadScanner() !== undefined;
+
+/**
+ * Stream the attempt — pinned to its etag where it has one — to the installed scanner. Returns on
+ * clean (or no verdict under `onScanUnavailable: 'accept'`); every refusal deletes the attempt, and
+ * infected deletes the upload too; hold keeps it pending. The stream is cancelled however the scanner returns: one that
+ * answers off the first chunk would otherwise hold the disk's connection open.
+ */
+export async function scanAttempt(input: ScanAttemptInput): Promise<void> {
+  const scanner = uploadScanner();
+  if (scanner === undefined) return;
+  const { disk, attempt } = input;
+  let body: ReadableStream<Uint8Array> | undefined;
+  let verdict: Awaited<ReturnType<typeof scanner.scan>>;
+  try {
+    // Inside the `try`: an attempt that cannot be READ for the scanner (an S3 GET reset, an
+    // etag-pinned read refused) is no verdict too, never a raw error past the cleanup.
+    body = await disk.stream(attempt.key, attempt.etag === '' ? undefined : { etag: attempt.etag });
+    verdict = await scanner.scan({
+      key: attempt.key,
+      pendingKey: input.pendingKey,
+      size: attempt.size,
+      contentType: attempt.contentType,
+      body,
+    });
+  } catch (error) {
+    // A holding scanner's outage is never a pass: it may be the store that knows what is held.
+    if (input.policy.onScanUnavailable === 'accept' && scanner.holds !== true) return;
+    // The attempt goes, the upload stays: a retry once the scanner is back is a fresh confirm.
+    await removeAll(disk, [attempt.key]);
+    throw scanUnavailable(input.pendingKey, scanner.name, renderThrowable(error));
+  } finally {
+    await body?.cancel().catch(() => undefined);
+  }
+  if (verdict.verdict === 'clean') return;
+  if (verdict.verdict === 'hold') {
+    // A decision, not an outage: no `onScanUnavailable` reaches here. The upload is what a later
+    // confirm promotes; the attempt was this call's.
+    await removeAll(disk, [attempt.key]);
+    assert(
+      scanner.holds === true,
+      `the ${scanner.name} upload scanner answered hold without declaring holds: true, so its outages would have been promoted unscanned under onScanUnavailable: 'accept'`,
+      'uploadScanner: { ...yourScanner, holds: true }   # in apps/<app>/runtime.ts',
+    );
+    throw uploadHeld(input.pendingKey, scanner.name, verdict.reason);
+  }
+  const removed = await removeAll(disk, [attempt.key, input.pendingKey]);
+  throw uploadInfected(input.pendingKey, scanner.name, verdict.signature, removed);
+}

package/src/upload-scanner-memory.ts

@@ -0,0 +1,62 @@
+// The reference `UploadScanner`: finds the EICAR test file, the one byte string every antivirus
+// agrees is a virus and no file legitimately contains. A test's scanner and the shape an app's
+// ClamAV adapter copies — streamed, matched across chunk boundaries, no state between calls.
+
+import type { ScanInput, ScanVerdict, UploadScanner } from './upload-scanner';
+
+/**
+ * The 68-byte EICAR anti-malware test string. Assembled from two halves so this source file is not
+ * itself a detection on a developer's machine or in a registry scan.
+ */
+export const EICAR_TEST_STRING = `${'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR'}${'-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'}`;
+
+const SIGNATURE = 'EICAR-Test-File';
+const NEEDLE = new TextEncoder().encode(EICAR_TEST_STRING);
+
+function indexOf(haystack: Uint8Array, needle: Uint8Array): number {
+  outer: for (let at = 0; at + needle.byteLength <= haystack.byteLength; at += 1) {
+    for (let i = 0; i < needle.byteLength; i += 1) {
+      if (haystack[at + i] !== needle[i]) continue outer;
+    }
+    return at;
+  }
+  return -1;
+}
+
+/**
+ * Reads the body to the end (or to the first match, then cancels it), keeping only the last
+ * `NEEDLE.length - 1` bytes between chunks: a match split by the stream is still a match, and the
+ * heap never holds more than one chunk plus that tail.
+ */
+async function containsEicar(body: ReadableStream<Uint8Array>): Promise<boolean> {
+  const reader = body.getReader();
+  let tail = new Uint8Array();
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) return false;
+      const window = new Uint8Array(tail.byteLength + value.byteLength);
+      window.set(tail);
+      window.set(value, tail.byteLength);
+      if (indexOf(window, NEEDLE) !== -1) {
+        await reader.cancel();
+        return true;
+      }
+      tail = window.slice(Math.max(0, window.byteLength - (NEEDLE.byteLength - 1)));
+    }
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+/** Infected exactly when the bytes contain the EICAR string, anywhere. Never unavailable. */
+export function memoryUploadScanner(): UploadScanner {
+  return {
+    name: 'memory',
+    async scan(input: ScanInput): Promise<ScanVerdict> {
+      return (await containsEicar(input.body))
+        ? { verdict: 'infected', signature: SIGNATURE }
+        : { verdict: 'clean' };
+    },
+  };
+}

package/src/upload-scanner.ts

@@ -0,0 +1,71 @@
+// The `uploadScanner` port (`ADAPTER_PORTS`): what a content scanner is asked, what it answers, and
+// the one slot the boot installs `runtime.ts`'s scanner into. `promoteAttachment` asks it; the
+// framework ships the seam and a memory reference, never a vendor (axiom 8 — ClamAV is the app's).
+
+import { adapterSlot } from '@ultimat3/core/adapter';
+
+/** One stored object, handed over as a stream: a scanner never forces the whole file into a heap. */
+export interface ScanInput {
+  /** Where the scanned bytes live: this confirm's own copy under `pending/scan/`, no URL rewrites. */
+  readonly key: string;
+  /**
+   * The upload being promoted — the `pending/` key `grantUpload` minted, stable across retries. What
+   * an out-of-band decision (a review queue, a vendor's async job) is keyed by; `key` changes per
+   * attempt.
+   */
+  readonly pendingKey: string;
+  readonly size: number;
+  /** The type the object is stored under, already sniffed against its bytes. */
+  readonly contentType: string;
+  /** Every byte of the object. Read it to the end or cancel it; it is not reused. */
+  readonly body: ReadableStream<Uint8Array>;
+}
+
+/**
+ * A verdict about the bytes, never about the scanner. A scanner that cannot decide — its daemon is
+ * down, it timed out, the file is past its own limit — REJECTS instead: `promoteAttachment` turns
+ * that into `X_STORAGE_SCAN_UNAVAILABLE` (or a pass, under `onScanUnavailable: 'accept'`).
+ *
+ * `hold` is a decision too: not yet cleared, by something outside this call — a human review, a
+ * vendor's asynchronous sandbox. The upload stays pending and promotion answers
+ * `X_STORAGE_QUARANTINED` until the scanner answers `clean` for that `pendingKey`. Neither `hold`
+ * nor `infected` is something a policy can wave through.
+ */
+export type ScanVerdict =
+  | { readonly verdict: 'clean' }
+  | { readonly verdict: 'infected'; readonly signature: string }
+  | { readonly verdict: 'hold'; readonly reason: string };
+
+export interface UploadScanner {
+  /** Named in every refusal's `meta.scanner` — `clamd`, `memory`. */
+  readonly name: string;
+  /**
+   * `true` for a scanner that answers `hold` (a review queue, an async vendor scan). Its
+   * no-verdict then ALWAYS refuses — `onScanUnavailable: 'accept'` never applies — because a
+   * scanner that cannot reach its own record of holds cannot say an upload was cleared. A `hold`
+   * from a scanner that did not declare this is refused as a bug (`X_INVARIANT`), so the
+   * declaration cannot be forgotten past the first held upload.
+   */
+  readonly holds?: boolean | undefined;
+  scan(input: ScanInput): Promise<ScanVerdict>;
+}
+
+const ambient = /* @__PURE__ */ adapterSlot<UploadScanner | undefined>(() => undefined);
+
+/**
+ * The boot's install, from `uploadScanner` in `apps/<app>/runtime.ts` — exported from
+ * `@ultimat3/storage/adapter` only, so that file is the one place an app chooses a scanner. Answers
+ * the release that removes THIS install and no other.
+ */
+export function installUploadScanner(scanner: UploadScanner): () => void {
+  return ambient.install(scanner);
+}
+
+/** The installed scanner, or `undefined`: no scanner installed means promotion scans nothing. */
+export function uploadScanner(): UploadScanner | undefined {
+  return ambient.current();
+}
+
+export function resetUploadScanner(): void {
+  ambient.reset();
+}

package/src/upload.ts

@@ -7,7 +7,7 @@
 // confirmed. A type no rule can ever confirm (`text/csv`, `application/json`) is still accepted
 // on the client's word: there is nothing to compare it against.
 
-import { finiteCount } from '@ultimat3/core';
+import { assert, finiteCount } from '@ultimat3/core';
 import { sha256Base64 } from './driver';
 import {
   checksumMismatch,
@@ -55,12 +55,33 @@ export interface UploadPolicy {
   readonly allowedContentTypes: readonly string[];
   /** When true a candidate without a `checksum` is rejected, not silently trusted. */
   readonly requireChecksum: boolean;
+  /**
+   * What `promoteAttachment` does when the installed `uploadScanner` gives no verdict (its daemon is
+   * down, it timed out). `'refuse'` — fail closed, `X_STORAGE_SCAN_UNAVAILABLE`, the upload stays
+   * pending for a retry — is the default and what an absent field means. `'accept'` promotes the
+   * file unscanned. An INFECTED verdict is refused under either: this decides availability only.
+   */
+  readonly onScanUnavailable?: ScanUnavailablePolicy | undefined;
 }
 
+export type ScanUnavailablePolicy = 'refuse' | 'accept';
+
 export interface UploadPolicyInit {
   readonly maxBytes?: number | undefined;
   readonly allowedContentTypes?: readonly string[] | undefined;
   readonly requireChecksum?: boolean | undefined;
+  readonly onScanUnavailable?: ScanUnavailablePolicy | undefined;
+}
+
+/** Screened where it is declared: a typo (`'allow'`) must not read as the fail-open answer. */
+function scanUnavailablePolicy(value: ScanUnavailablePolicy | undefined): ScanUnavailablePolicy {
+  const policy = value === undefined ? 'refuse' : value;
+  assert(
+    policy === 'refuse' || policy === 'accept',
+    `uploadPolicy was given onScanUnavailable ${JSON.stringify(policy)}, which is neither 'refuse' nor 'accept'`,
+    "uploadPolicy({ onScanUnavailable: 'refuse' }) — or 'accept' to promote unscanned files while the scanner is down",
+  );
+  return policy;
 }
 
 export function uploadPolicy(init: UploadPolicyInit = {}): UploadPolicy {
@@ -84,6 +105,7 @@ export function uploadPolicy(init: UploadPolicyInit = {}): UploadPolicy {
       normalizeContentType,
     ),
     requireChecksum: init.requireChecksum ?? false,
+    onScanUnavailable: scanUnavailablePolicy(init.onScanUnavailable),
   };
 }
 
@@ -296,8 +318,19 @@ function hasSignature(declared: string): boolean {
   );
 }
 
+/**
+ * Whether a head opens with markup (after whitespace): then `sniffText` asks `activeMarkupType` of
+ * the WHOLE body, so a streamed accept must read it whole too — or text with XHTML past the head
+ * passes as plain text. A binary body never opens with `<`, so it keeps streaming.
+ */
+export const opensWithMarkup = (head: Uint8Array): boolean =>
+  new TextDecoder('utf-8')
+    .decode(head.subarray(0, MAX_LEADING_WHITESPACE + 1))
+    .trimStart()
+    .startsWith('<');
+
 /** `application/xml`, `text/xml` and every `+xml` suffix type (RFC 6839): what a browser parses as XML. */
-const isXmlType = (type: string): boolean =>
+export const isXmlType = (type: string): boolean =>
   type === 'application/xml' || type === 'text/xml' || type.endsWith('+xml');
 
 /**