@volter/world-core 3.0.164 → 3.0.165

package/src/head.ts

@@ -34,6 +34,9 @@ export type PushOutcome = {
   actionId?: string;
   /** the group's other entries the answer named an id for (a resource's `companions`) */
   also?: Array<{ actionId: string; externalId: string; data?: Record<string, unknown> }>;
+  /** the group's entries the vendor refused while it took the rest (a git push to a root without `atomic`, each ref its
+   *  own outcome: ADR 0031); each settles refused with its reason */
+  refused?: Array<{ actionId: string; reason: string }>;
 };
 /** What a perform is handed beside the executor: a resolver from a local id to the vendor's (an
  *  entry authored against `twin-1` crosses against `REAL-42` once that subject's landing adopted it). */
@@ -269,6 +272,8 @@ export type DeployReport = {
   refused?: { actionId: string; check: string; reason: string };
   /** the first failure, with the vendor's words; nothing after it was attempted */
   failed?: { actionId: string; error: string };
+  /** a group the vendor took in part: its entries it refused, each with its reason; nothing after it was attempted */
+  partial?: Array<{ actionId: string; reason: string }>;
   /** entries after the refusal or failure, not attempted */
   skipped: string[];
   /** entries whose landed copy already said `deployed` — not performed again */
@@ -287,12 +292,13 @@ function receiptsByEntry(parent: Entry[]): Map<string, Receipt> {
 /** Whether an entry records a call of the vendor's surface (an operation or a GraphQL mutation): what a perform can send
  *  again (the real-system adapters, "What the derived core records with a write"). */
 function surfaceWrite(e: { input?: Record<string, unknown> }): boolean {
-  return typeof e.input?.operationId === 'string' || (e.input?.graphql !== undefined && e.input.graphql !== null);
+  // a push to a World repository is the vendor's too: a deploy sends it as a git push (derived-core.ts pushedInput)
+  return typeof e.input?.operationId === 'string' || (e.input?.graphql !== undefined && e.input.graphql !== null) || (e.input?.git !== undefined && e.input.git !== null);
 }
 
 /** The branch entries a root twin still has to settle: `set` rows, not reverted, whose landed copy is absent or says
  *  `failed`/`skipped` (a `deployed` or `refused` copy settles the entry), split into what is offered to the vendor (a
- *  surface write on a subject the vendor can have) and what is the World's own: a write a door, a screen, the gap, the
+ *  surface write or a git push on a subject the vendor can have) and what is the World's own: a write a door, a screen, the gap, the
  *  clock or a seed made, and any write on a subject only the World's own writes made. */
 export function settlingEntries(service: string, root?: string): { offered: TwinAction[]; worldOnly: TwinAction[] } {
   const parent = parentEntries(service, root);
@@ -436,7 +442,10 @@ async function performNow(opts: Parameters<typeof performEntries>[0]): Promise<D
       const answered = outcome.data !== null && typeof outcome.data === 'object' && !Array.isArray(outcome.data) ? (outcome.data as Record<string, unknown>) : {};
       const primary = resolved.find((a) => a.id === outcome.actionId) ?? resolved[0]!;
       const also = new Map((outcome.also ?? []).map((a) => [a.actionId, a]));
+      const refusedHere = new Map((outcome.refused ?? []).map((r) => [r.actionId, r.reason]));
       for (const action of resolved) {
+        const refusal = refusedHere.get(action.id);
+        if (refusal !== undefined) { settle(action, at, { status: 'refused', reason: refusal }); refusedIds.add(action.subject.id); continue; }
         const answeredId = action === primary ? outcome.externalId : also.get(action.id)?.externalId;
         const externalId = answeredId || aliases.get(`${action.subject.type}:${action.subject.id}`);
         settle(action, at, { status: 'deployed', ...(action === primary && outcome.url ? { url: outcome.url } : {}) }, { ...action.fields, ...(action === primary ? answered : also.get(action.id)?.data ?? {}) }, externalId || undefined);
@@ -444,6 +453,7 @@ async function performNow(opts: Parameters<typeof performEntries>[0]): Promise<D
         report.pushed += 1;
         report.deployed.push({ actionId: action.id, at, ...(externalId ? { externalId } : {}), ...(data !== undefined ? { data } : {}), ...(action === primary && outcome.response ? { response: outcome.response } : {}) });
       }
+      if (refusedHere.size) { report.partial = [...refusedHere].map(([actionId, reason]) => ({ actionId, reason })); stopped = true; }
     } catch (error) {
       // a write the vendor refuses (a 4xx: it can never be sent as it is) settles the group refused, as a check's
       // refusal does, so the next deploy moves on to the group after it; anything else failed and stays offered
@@ -515,7 +525,7 @@ export async function deferPerform<T>(request: string, fn: () => Promise<T>): Pr
 /** Under `auto`, perform the entries one request made as one call (the real-system adapters, "Perform"): the ids the
  *  vendor gave them by local id, or the vendor's refusal or failure, the request's entries reverted as performAtHead
  *  reverts one. Nothing when the head is not real and auto, or the request wrote nothing the vendor is offered. */
-export async function performRequestGroup(service: string, request: string, root?: string): Promise<{ ids: Map<string, string>; response?: RemoteExecuteResponse } | RefusedWriteError | VendorWriteError> {
+export async function performRequestGroup(service: string, request: string, root?: string): Promise<{ ids: Map<string, string>; response?: RemoteExecuteResponse; refused?: Array<{ actionId: string; reason: string }> } | RefusedWriteError | VendorWriteError> {
   const head = headOf(service, root);
   const ids = new Map<string, string>();
   if (head.kind !== 'real' || head.deploy !== 'auto') return { ids };
@@ -525,21 +535,42 @@ export async function performRequestGroup(service: string, request: string, root
   try { report = await performEntries({ service, actionIds: group.map((e) => e.id), ...(root !== undefined ? { root } : {}) }); }
   catch (error) {
     if (error instanceof DeploymentOrderError) {
-      for (const e of [...group].reverse()) revertAction({ service, actionId: e.id, occurredAt: worldNow(), ...(root !== undefined ? { root } : {}) });
+      revertRequest(service, request, group, root);
       return new VendorWriteError(`An earlier ${service} write is still pending; this request was rolled back before reaching the vendor. Retry after that write finishes.`, group[0]!.id);
     }
     throw error instanceof HeadError ? error : new HeadError(error instanceof Error ? error.message : String(error));
   }
   if (report.refused || report.failed) {
-    for (const e of [...group].reverse()) revertAction({ service, actionId: e.id, occurredAt: worldNow(), ...(root !== undefined ? { root } : {}) });
+    revertRequest(service, request, group, root);
     return report.refused ? new RefusedWriteError(report.refused.check, report.refused.reason, report.refused.actionId) : new VendorWriteError(report.failed!.error, report.failed!.actionId);
   }
+  if (report.partial) revertRequest(service, request, group.filter((e) => report.partial!.some((r) => r.actionId === e.id)), root);
   for (const d of report.deployed) {
     const e = group.find((g) => g.id === d.actionId);
     if (e && d.externalId && d.externalId !== e.subject.id) ids.set(e.subject.id, d.externalId);
   }
   const response = report.deployed.find((entry) => entry.response)?.response;
-  return { ids, ...(response ? { response } : {}) };
+  return { ids, ...(response ? { response } : {}), ...(report.partial ? { refused: report.partial } : {}) };
+}
+
+/** Revert what a request's refused entries made, latest first. An entry that is a declared push a screen recorded
+ *  (`input.git`, ADR 0031) owns the request's writes after it up to the next such push, its push's effects (the
+ *  reactions and vendor moves the pack runs on it), which go with it; any other refused entry goes alone, as the group
+ *  it is in. */
+function revertRequest(service: string, request: string, refused: TwinAction[], root?: string): void {
+  const pushed = (e: { input?: Record<string, unknown> }): boolean => e.input?.git !== undefined && e.input.git !== null;
+  const undo = new Set<string>();
+  if (refused.some(pushed)) {
+    const made = (branchEntries(service, root) as TwinAction[]).filter((e) => e.correlationId === request && e.op === 'set');
+    for (const r of refused) {
+      if (!pushed(r)) { undo.add(r.id); continue; }
+      const at = made.findIndex((e) => e.id === r.id);
+      if (at < 0) { undo.add(r.id); continue; }
+      undo.add(r.id);
+      for (const e of made.slice(at + 1)) { if (pushed(e)) break; undo.add(e.id); }
+    }
+  } else for (const r of refused) undo.add(r.id);
+  for (const e of branchEntries(service, root).filter((x) => undo.has(x.id)).reverse()) revertAction({ service, actionId: e.id, occurredAt: worldNow(), ...(root !== undefined ? { root } : {}) });
 }
 
 /** Wrap a pack's fetch: a `RefusedWriteError` or `VendorWriteError` the head raised becomes the

package/src/html.ts

@@ -0,0 +1,72 @@
+// The WHATWG HTML grammar is shared mechanics; what a pack keeps of a document is the pack's own policy.
+// The exact optional dependency belongs to this kernel, so Node and browser bundles resolve one parser.
+// Source: parse5 7.3.0 (parseFragment, serialize, defaultTreeAdapter).
+//
+// A fragment is parsed in an HTML `<body>` context. Elements keep their namespace (HTML, SVG or MathML) and attributes
+// their prefix and namespace, and serialization reproduces what the tree holds: a foreign subtree survives parsing, and
+// a pack's policy may drop it.
+export type HtmlAttribute = { name: string; value: string; prefix?: string; namespace?: string };
+export type HtmlElement = { type: 'element'; name: string; namespace: string; attributes: HtmlAttribute[]; children: HtmlNode[] };
+export type HtmlText = { type: 'text'; value: string };
+export type HtmlNode = HtmlElement | HtmlText;
+export class HtmlError extends Error {
+  constructor(readonly code: 'parser-unavailable') {
+    super('HTML parser parse5 7.3.0 is unavailable');
+    this.name = 'HtmlError';
+  }
+}
+type Grammar = typeof import('parse5');
+type Tree = import('parse5').DefaultTreeAdapterMap;
+let grammar: Promise<Grammar> | undefined;
+function loadGrammar(): Promise<Grammar> {
+  return grammar ??= import('parse5').catch(() => { grammar = undefined; throw new HtmlError('parser-unavailable'); });
+}
+
+/**
+ * A fragment parsed as WHATWG fragment parsing does in an HTML `<body>` context: elements and text, comments dropped,
+ * an HTML `<template>`'s content carried as that element's children. Importing this module does not load the grammar.
+ */
+export async function parseFragment(source: string): Promise<HtmlNode[]> {
+  const { parseFragment: parse, defaultTreeAdapter: tree, html } = await loadGrammar();
+  const convert = (nodes: Tree['node'][]): HtmlNode[] => {
+    const out: HtmlNode[] = [];
+    for (const node of nodes) {
+      if (node.nodeName === '#text') out.push({ type: 'text', value: (node as Tree['textNode']).value });
+      else if ('tagName' in node) {
+        const element = node as Tree['element'];
+        // only an HTML template has content; an SVG or MathML `template` is a foreign element with children
+        const children = element.tagName === 'template' && element.namespaceURI === html.NS.HTML
+          ? (element as Tree['template']).content.childNodes
+          : element.childNodes;
+        out.push({
+          type: 'element', name: element.tagName, namespace: element.namespaceURI,
+          attributes: element.attrs.map(attr => ({ name: attr.name, value: attr.value, ...(attr.prefix ? { prefix: attr.prefix } : {}), ...(attr.namespace ? { namespace: attr.namespace } : {}) })),
+          children: convert(children),
+        });
+      }
+    }
+    return out;
+  };
+  return convert(parse(tree.createElement('body', html.NS.HTML, []), source, {}).childNodes);
+}
+
+/** Nodes serialized by the WHATWG HTML fragment serialization: its escaping, its void elements, its foreign names. */
+export async function serialize(nodes: HtmlNode[]): Promise<string> {
+  const { serialize: write, defaultTreeAdapter: tree, html } = await loadGrammar();
+  const append = (parent: Tree['parentNode'], list: HtmlNode[]): void => {
+    for (const node of list) {
+      if (node.type === 'text') { tree.insertText(parent, node.value); continue; }
+      const element = tree.createElement(node.name, node.namespace as import('parse5').html.NS,
+        node.attributes.map(({ name, value, prefix, namespace }) => ({ name, value, ...(prefix ? { prefix } : {}), ...(namespace ? { namespace } : {}) })));
+      tree.appendChild(parent, element);
+      if (node.name === 'template' && node.namespace === html.NS.HTML) {
+        const content = tree.createDocumentFragment();
+        tree.setTemplateContent(element as Tree['template'], content);
+        append(content, node.children);
+      } else append(element, node.children);
+    }
+  };
+  const fragment = tree.createDocumentFragment();
+  append(fragment, nodes);
+  return write(fragment);
+}

package/src/index.ts

@@ -76,11 +76,13 @@ export { passwordHash, personOf, personWith, recordPerson, shownPersonFields, ty
 // the World's managed Postgres a handler reaches as `ctx.engine` (managed-database.ts), and a mail's route (`ctx.mail`)
 export { EngineDatabaseError, EngineUnavailableError, quoteIdent } from './managed-database.ts';
 export type { Batch as EngineBatch, ManagedDatabase, Statement as EngineStatement, StatementResult as EngineStatementResult } from './managed-database.ts';
-export type { Mail, SmtpRoute } from './smtp.ts';
+export type { Mail, MailReply } from './smtp.ts';
 // S3's wire (contract "S3's wire is a kernel library too") and the git plane (contract "The git plane is a kernel library")
 export * as s3 from './s3/wire.ts';
 // Generic strict XML grammar; vendor interpretation remains in its consumer.
 export * as xml from './xml.ts';
+// Generic WHATWG HTML grammar; what a pack keeps of received HTML remains the pack's policy.
+export * as html from './html.ts';
 export * as clickhouse from './clickhouse/index.ts';
 export { sampleFromSchema } from './schema-sample.ts';
 export * as openaiWire from './openai-wire.ts';
@@ -92,6 +94,7 @@ export { placeholderPng } from './placeholder-image.ts';
 export { imageDimensions, readImageHeader, readMp4Header } from './media-header.ts';
 export { presignSigV4, signSigV4, verifySigV4, type SigV4Verdict } from './sigv4.ts';
 export * as redis from './redis/index.ts';
+export * as smtpServer from './smtp-server.ts';
 export * as mysql from './mysql/index.ts';
 export * as live from './live/index.ts';
 export * as mongodb from './mongodb/index.ts';

package/src/pack-fetch.ts

@@ -19,7 +19,9 @@ import { isReadOnlyRequest, READ_ONLY_REQUEST_HEADER, runAsVendorMove } from './
 import { parseScenarioDocument, ScenarioEngine, ScenarioError, statefulTwinManifest, twinManifest, type PackScenarioAdapter } from './scenario.ts';
 import { getActiveWorldStore } from './world-store.ts';
 import { runWithCorrelationId } from './actions.ts';
-import { deferPerform, performRequestGroup, RefusedWriteError, VendorWriteError } from './head.ts';
+import { deferPerform, deployableEntries, performRequestGroup, RefusedWriteError, VendorWriteError } from './head.ts';
+import { FLUSH, ZERO_SHA, pktLine, readPktLines } from './git/smart-http.ts';
+import { concat } from './git/objects.ts';
 import { withCors } from './cors.ts';
 import type { ApplicationStandIn } from './events.ts';
 import { contentAddressOf } from './content-route.ts';
@@ -32,7 +34,7 @@ import { expireSocketWaits, socketUpgrade, type SocketEngine } from './sockets.t
 import { liveIngest } from './live/ingest.ts';
 import type { LiveEngine } from './live/types.ts';
 import { readResourceBlob, removeResourceBlob, writeResourceBlob } from './resource-blob.ts';
-import { resourcesOfType } from './derived-core.ts';
+import { gitRepositoryOf, resourcesOfType } from './derived-core.ts';
 import { worldNow } from './world-clock.ts';
 import type { WebSocketUpgrade, WebSocketPeer } from './serve-http.ts';
 import { MACHINE_POOL, POOL_KINDS, resetFakePool } from './machines.ts';
@@ -643,7 +645,38 @@ export function createPackFetch(parts: PackParts, options: PackFetchOptions = {}
     const id = `req_${globalThis.crypto.randomUUID()}`;
     const { value: answer, appended } = await runWithCorrelationId(id, () => deferPerform(id, () => fetch(request)));
     if (!appended) return answer;
+    // a receive-pack's pushes (derived-core.ts pushedInput): what the World's refs held before and after each
+    const pushes = m.git ? deployableEntries(m.service, options.root).filter((e) => e.correlationId === id && e.input?.git).map((e) => ({ actionId: e.id, ...(e.input!.git as { name: string; ref: string; before: string; after: string }) })) : [];
     const performed = await runWithCorrelationId(id, () => performRequestGroup(m.service, id, options.root));
+    // under `auto`, what the root did is what the World keeps (ADR 0031): a ref it refused goes back to its value before
+    // the push, by compare-and-set under the ref lock (a later push built on it keeps its value), its push and effects
+    // are reverted (head.ts), and the client hears git's own report, `ok` or `ng` for each ref
+    if (pushes.length) {
+      const whole = performed instanceof RefusedWriteError ? performed.reason : performed instanceof VendorWriteError ? performed.message : undefined;
+      const refused = new Map<string, string>();
+      for (const p of pushes) {
+        const reason = whole ?? (performed as { refused?: Array<{ actionId: string; reason: string }> }).refused?.find((r) => r.actionId === p.actionId)?.reason;
+        if (reason !== undefined) refused.set(p.actionId, reason);
+      }
+      if (refused.size) {
+        for (const p of [...pushes].reverse()) {
+          if (!refused.has(p.actionId)) continue;
+          const { refs } = gitRepositoryOf(m, p.name, options.root);
+          refs.withLock(() => {
+            if ((refs.get(p.ref) ?? ZERO_SHA) !== p.after) return;
+            if (p.before === ZERO_SHA) refs.delete(p.ref); else refs.set(p.ref, p.before);
+          });
+        }
+        const ng = new Map(pushes.filter((p) => refused.has(p.actionId)).map((p) => [p.ref, refused.get(p.actionId)!.replace(/\s+/g, ' ')]));
+        const report = concat(...readPktLines(new Uint8Array(await answer.arrayBuffer())).lines.map((line) => {
+          const ref = line.startsWith('ok ') ? line.slice(3).trim() : undefined;
+          return pktLine(ref !== undefined && ng.has(ref) ? `ng ${ref} ${ng.get(ref)}\n` : line);
+        }), FLUSH);
+        const headers = new Headers(answer.headers);
+        headers.delete('content-length');
+        return new Response(report as unknown as BodyInit, { status: answer.status, headers });
+      }
+    }
     // the vendor's refusal, as its status and words (perform names them `<status>: <message>`), in the vendor's error body
     if (performed instanceof RefusedWriteError) { const said = /^(\d{3}): ([\s\S]*)$/.exec(performed.reason); return vendorError(m, { status: said ? Number(said[1]) : 400, message: said ? said[2]! : performed.reason }); }
     if (performed instanceof VendorWriteError) return vendorError(m, { status: 502, message: performed.message });

package/src/pack-paths.ts

@@ -102,20 +102,12 @@ export function installedPackageDir(from: string, name: string): string | undefi
   }
 }
 
-/** A pack repository itself, or a platform checkout's P3 repository: the one `VOLTER_PACKS` names (a worktree's own
- *  pinned packs, as the repository's scripts read it), else the one beside it. Applications do not acquire a
- *  sibling checkout merely because one happens to be beside them. */
+/** The pack repository a World is rooted in, when it is one (ADR 0034: a platform checkout finds no pack repository
+ *  beside it; a pack reaches any other World as an installed or linked package). */
 export function packRepository(from: string): string | undefined {
   let dir = resolve(from);
   for (;;) {
     if (packageName(dir) === 'twin-packs-p3') return dir;
-    if (packageName(join(dir, 'packages', 'world-core')) === '@volter/world-core') {
-      const env = globalThis.process?.env?.VOLTER_PACKS;
-      const named = env ? resolve(env) : undefined;
-      if (named && packageName(named) === 'twin-packs-p3') return named;
-      const sibling = join(dirname(dir), 'twin-packs-p3');
-      if (packageName(sibling) === 'twin-packs-p3') return sibling;
-    }
     const up = dirname(dir);
     if (up === dir) return undefined;
     dir = up;
@@ -125,7 +117,7 @@ export function packRepository(from: string): string | undefined {
 /** Resolve one vendor's package directory for host-side reads (assets, infrastructure launchers). */
 export function vendorPackageDir(from: string, vendor: string): string | undefined {
   if (!/^[a-z][a-z0-9-]*$/.test(vendor)) return undefined;
-  const installed = installedVendorPacks(from).get(vendor);
+  const installed = installedVendorPacks(from, { vendors: [vendor] }).get(vendor);
   const repo = packRepository(from);
   const checkout = repo === undefined ? undefined : join(repo, vendor);
   if (!checkout || packageName(checkout) !== `@volter/twin-${vendor}` || !isDerivedPack(checkout)) return installed;

package/src/packRegistry.ts

@@ -166,13 +166,16 @@ export async function pullOnSchedule<T>(vendor: PullVendor, pull: () => Promise<
  * matches and no exclude does. Validated by scripts/pack-facts.ts: keys are `[a-z0-9-]+`, a
  * foreign key is never another pack's vendor id, belongs to one pack, and must appear in the
  * pack's `adoption.worldIds` (the world wires `<KEY>_TWIN_URL` from there); a pack must claim its
- * own vendor id too; no key is exclude-only; `exclude` is `true` or absent.
+ * own vendor id too; no key is exclude-only; `exclude` is `true` or absent. `basic` is a vendor fact for the executor: on
+ * a URL the rule names, the host takes the sealed token as HTTP Basic's password under this user (a vendor's git over
+ * HTTP whose API takes the same token as a bearer; ADR 0031).
  */
 export type HostRule = ({ host: string } | { suffix: string } | { hostPattern: string }) & {
   pathPattern?: string;
   demandPathPattern?: string | false;
   key?: string;
   exclude?: true;
+  basic?: string;
 };
 
 export type RoundTripWrite = { method: string; path: string; body?: unknown; headers?: Record<string, string> };
@@ -234,6 +237,11 @@ export type TwinPack = {
   managedService?: { kind: string; scheme: string; note: string; configurationArg?: '--configuration'; originalInputs?: { version: 1; manifestArg: '--original-inputs'; rootArg: '--original-root'; digestArg: '--original-inputs-sha256'; paths: string[] } };
   managedDatabase?: { kind: 'postgres'; arg: string; user: string; database: string; clock: 'wall'; credentialField: string; note: string };
   sidePorts?: Array<{ name: string; option: string; flag: string; path?: string; scheme?: string; note: string }>;
+  /** The vendor's own requests a World's generated seed makes for what its world.json declares (ADR 0033 §3), each a URL
+   *  at the vendor's own host, defined by the twin's own screen form for that act: its sign-up, and its domain screen's
+   *  add and verify with `record`, the one `/_twin/` stand-in, for a record at the domain's DNS host; `settles` is the
+   *  World-clock time the vendor takes to settle a verified domain (`10m`). */
+  seed?: { signUp?: string; domain?: { add: string; record: string; verify: string; settles: string } };
   stateSystem?: StateSystemAdapters;
   /** The pack has no vendor-backed half, and why (copied from the manifest by packOf). */
   vendorBacked?: { none: string };

package/src/runtime.ts

@@ -307,6 +307,8 @@ export * as git from './git/index.ts';
 export * as s3 from './s3/wire.ts';
 // Generic strict XML grammar; vendor interpretation remains in its consumer.
 export * as xml from './xml.ts';
+// Generic WHATWG HTML grammar; what a pack keeps of received HTML remains the pack's policy.
+export * as html from './html.ts';
 
 // The push arm: the kernel's transaction over a pack's one `perform<Name>Action` (contract section of that name).
 
@@ -343,6 +345,7 @@ export { brandTokensResponse } from './brand-tokens.ts';
 export { multipartBoundary, multipartParts, multipartEnvelope, inferMultipartBoundary, multipartDelimiterPattern, type MultipartPart, type MultipartFramedPart, type MultipartEnvelope } from './multipart.ts';
 export { corsHeaders, withCors, type CorsDecl } from './cors.ts';
 export { setInProcessTwins, vendorFetch, VendorUnreachableError, type InProcessTwin } from './vendor-call.ts';
+export { setInProcessMailServers } from './smtp.ts';
 export { closeManagedDatabases, databaseSession, EngineDatabaseError, EngineUnavailableError, managedDatabase, quoteIdent, useDatabaseDriver, type DatabaseDriver, type DatabaseSession } from './managed-database.ts';
 export type { Batch as EngineBatch, ManagedDatabase, Statement as EngineStatement, StatementResult as EngineStatementResult } from './managed-database.ts';
 export { readResourceBlob, readResourceBlobRange, removeResourceBlob, resourceBlobFile, resourceBlobKey, resourceBlobSize, resourceChain, writeResourceBlob, writeResourceBlobStream } from './resource-blob.ts';

package/src/serve-http.ts

@@ -185,17 +185,19 @@ const STREAM_HIGH_WATER = 4 * 1024 * 1024;
 /** A raw-TCP twin's listener (the seam's byte-stream half): its `create<Name>TwinStream` served on a TCP port, each
  *  socket one connection, on loopback unless the caller names a hostname. Runtime-neutral, as `serveHttp` is: Node's
  *  `net` reached at call time, never imported. A stream that declares `maxConnections` (a live ingest) is held to it. */
-export async function serveStream(options: { port?: number; hostname?: string; stream: TwinStream; maxConnections?: number }): Promise<{ port: number; stop: () => void }> {
+export async function serveStream(options: { port?: number; hostname?: string; stream: TwinStream; maxConnections?: number }): Promise<{ port: number; stop: () => Promise<void> }> {
   const get = (process as { getBuiltinModule?: (id: string) => unknown }).getBuiltinModule;
   const net = typeof get === 'function' ? get('node:net') as typeof import('node:net') | undefined : undefined;
   if (!net || typeof net.createServer !== 'function') throw new StreamListenerUnavailable('this runtime has no TCP listener (node:net)');
   const cap = options.maxConnections ?? (options.stream as { maxConnections?: number }).maxConnections;
   let open = 0;
+  const accepted = new Set<import('node:net').Socket>();
   const server = net.createServer((socket) => {
     // past its cap a connection is closed before the stream sees it, so connections that never get anywhere cannot pile up
     if (cap !== undefined && open >= cap) { socket.destroy(); return; }
     open += 1;
-    socket.once('close', () => { open -= 1; });
+    accepted.add(socket);
+    socket.once('close', () => { open -= 1; accepted.delete(socket); });
     const conn = options.stream({ write: (bytes) => { socket.write(bytes); }, end: () => { socket.end(); } }, socket.remoteAddress ?? '127.0.0.1');
     // a connection that says what waits (`backlog`: whole messages its work has not reached) is read no further past the
     // high-water mark until that work has settled, so a peer faster than the twin (a live ingest pushed faster than real
@@ -212,7 +214,13 @@ export async function serveStream(options: { port?: number; hostname?: string; s
   });
   await new Promise<void>((resolve, reject) => { server.once('error', reject); server.listen(options.port ?? 0, options.hostname ?? '127.0.0.1', () => resolve()); });
   const address = server.address();
-  return { port: typeof address === 'object' && address ? address.port : options.port ?? 0, stop: () => { server.close(); } };
+  // stop is the listener's real retirement: it settles once the listener is closed and every connection it accepted has
+  // ended (each is closed here, so none holds the close open)
+  const stop = (): Promise<void> => new Promise<void>((resolve) => {
+    server.close(() => resolve());
+    for (const socket of accepted) socket.destroy();
+  });
+  return { port: typeof address === 'object' && address ? address.port : options.port ?? 0, stop };
 }
 function builtin<T>(name: string): T {
   const get = (process as { getBuiltinModule?: (id: string) => unknown }).getBuiltinModule;

package/src/smtp-server.ts

@@ -0,0 +1,166 @@
+// THE KERNEL'S SMTP SERVER MACHINE (`smtpServer`, from `@volter/world-core`): RFC 5321's mechanics every pack that serves an
+// SMTP server shares (a relay's submission server, a mailbox provider's receiving server: ADR 0032 §4), behind the kernel's
+// stream boundary (TwinStream). It frames a connection's bytes into commands, a message's data and an AUTH response,
+// carries them to the pack's front over the pack's own fetch (a private bridge the pack's key signs), and parses the
+// RFC's command arguments. What a server answers, which senders and recipients it takes and what it keeps stay in each
+// pack: its replies, its policy and its writes. The kernel names no vendor. docs/contributing/architecture.md, "Other
+// wires: lanes, headers and line protocols".
+// source: RFC 5321 §4.1.1.4 "In particular, the sequence \"<LF>.<LF>\" (bare line feeds, without carriage returns) MUST NOT be treated as equivalent to <CRLF>.<CRLF> as the end of mail data indication."
+// source: RFC 5321 §4.5.2 "If the first character is a period and there are other characters on the line, the first character is deleted."
+import type { HandlerContext } from './derived-core.ts';
+import type { TwinStream } from './twin-fetch.ts';
+
+/** What a server's front answers the stream: its reply lines, the moved session, and whether the stream reads a
+ *  message's data next (`data`), an AUTH response (`auth`) or commands, or closes. */
+export type SmtpAnswer<S> = { replies: string[]; session: S; mode: 'command' | 'data' | 'auth'; close?: boolean };
+
+/** The bridge between a connection and the pack's front: the header that carries the pack's key, the key, a fresh
+ *  session, and the pack's answer when its front cannot answer (its words for a local error). */
+export type SmtpBridge<S> = { header: string; key: () => string; fresh: () => S; failure: (session: S) => SmtpAnswer<S> };
+
+/** A connection's bytes framed as SMTP frames them, each piece posted to the pack's front (`smtpFront`) through its
+ *  fetch with the session the connection carries: commands are lines ended by CRLF; after a 354, the message's lines up
+ *  to the lone dot, their leading dot removed; after a 334, one line of AUTH response. Pipelined commands are answered
+ *  in order. */
+export function smtpStream<S>(fetch: (request: Request) => Response | Promise<Response>, bridge: SmtpBridge<S>): TwinStream {
+  return (sink, peer) => {
+    const encoder = new TextEncoder();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    let session = bridge.fresh();
+    let mode: SmtpAnswer<S>['mode'] = 'command';
+    let data: string[] = [];
+    let closed = false;
+    let chain: Promise<void> = Promise.resolve();
+
+    const post = async (body: Record<string, unknown>): Promise<void> => {
+      if (closed) return;
+      let a: SmtpAnswer<S>;
+      try {
+        const res = await fetch(new Request('http://smtp.wire/', { method: 'POST', headers: { 'content-type': 'application/json', [bridge.header]: bridge.key(), 'x-forwarded-for': peer }, body: JSON.stringify({ session, ...body }) }));
+        a = res.ok ? (await res.json()) as SmtpAnswer<S> : bridge.failure(session);
+        if (!Array.isArray(a.replies) || !a.session || !['command', 'data', 'auth'].includes(a.mode)) a = bridge.failure(session);
+      } catch { a = bridge.failure(session); }
+      session = a.session;
+      mode = a.mode;
+      sink.write(encoder.encode(a.replies.map((r) => `${r}\r\n`).join('')));
+      if (a.close) { closed = true; sink.end(); }
+    };
+
+    const line = async (text: string): Promise<void> => {
+      if (mode === 'data') {
+        if (text === '.') { const message = data.join('\r\n') + (data.length ? '\r\n' : ''); data = []; mode = 'command'; await post({ data: message }); return; }
+        data.push(text.startsWith('.') ? text.slice(1) : text);
+        return;
+      }
+      if (mode === 'auth') { await post({ response: text }); return; }
+      await post({ line: decoder.decode(Uint8Array.from(text, (c) => c.charCodeAt(0))) });
+    };
+
+    const pump = (): void => {
+      let at: number;
+      while ((at = buffer.indexOf('\r\n')) >= 0) {
+        const text = buffer.slice(0, at);
+        buffer = buffer.slice(at + 2);
+        chain = chain.then(() => line(text));
+      }
+    };
+
+    chain = chain.then(() => post({ open: true }));
+    return {
+      data(chunk) { if (closed) return; buffer += Array.from(chunk, (b) => String.fromCharCode(b)).join(''); pump(); },
+      settled: () => chain,
+      close() { closed = true; },
+    };
+  };
+}
+
+/** What a pack's front does with each piece the stream carries: the greeting, a command line, a message's data, an AUTH
+ *  response (a server that takes none leaves it out). */
+type SmtpServer<S> = {
+  open(): SmtpAnswer<S> | Promise<SmtpAnswer<S>>;
+  command(session: S, line: string): SmtpAnswer<S> | Promise<SmtpAnswer<S>>;
+  data(session: S, data: string): SmtpAnswer<S> | Promise<SmtpAnswer<S>>;
+  response?(session: S, response: string): SmtpAnswer<S> | Promise<SmtpAnswer<S>>;
+};
+
+/** The pack's front (its `around`): a request the stream signed with the pack's key is a piece of a connection, answered
+ *  by the pack's server; any other request is the pack's own (its doors and screens). */
+export async function smtpFront<S>(ctx: HandlerContext, next: (request?: Request) => Promise<Response>, bridge: SmtpBridge<S>, server: SmtpServer<S>): Promise<Response> {
+  const request = ctx.call.request;
+  if (!request.headers.has(bridge.header)) return next();
+  if (request.headers.get(bridge.header) !== bridge.key()) return new Response('not the server\'s wire', { status: 403 });
+  const w = (ctx.body ?? {}) as { session?: S; open?: boolean; line?: string; data?: string; response?: string };
+  const s = w.session ?? bridge.fresh();
+  if (w.open) return Response.json(await server.open());
+  if (w.data !== undefined) return Response.json(await server.data(s, w.data));
+  if (w.response !== undefined && server.response) return Response.json(await server.response(s, w.response));
+  return Response.json(await server.command(s, String(w.line ?? '')));
+}
+
+/** A command's verb (upper-cased) and its argument. */
+export function smtpCommand(line: string): { verb: string; arg: string } {
+  const [verb = '', ...rest] = line.split(' ');
+  return { verb: verb.toUpperCase(), arg: rest.join(' ') };
+}
+
+/** The session machine's word on a command from where a session stands (the manifest's states): a refusal's reply, or
+ *  undefined to go on. */
+export function smtpOrder(ctx: Pick<HandlerContext, 'legal'>, machine: string, stage: string, op: string, to?: string): string | undefined {
+  const refusal = ctx.legal(machine, 'stage', op, stage, to);
+  return refusal ? `${refusal.status} ${String(refusal.code)} ${refusal.message}` : undefined;
+}
+
+/** EHLO's multi-line 250 reply: the server's name, then each keyword. */
+// source: RFC 5321 §4.1.1.1 "ehlo-ok-rsp"
+export const smtpEhlo = (keywords: string[]): string[] => keywords.map((k, i) => `250${i === keywords.length - 1 ? ' ' : '-'}${k}`);
+
+/** An address in angle brackets (`<a@b>`, `<>` for the null sender), or undefined when it is not one. */
+function smtpAddress(arg: string): string | undefined {
+  const m = /^<([^<>\s]*)>$/.exec(arg.trim());
+  if (!m) return undefined;
+  const a = m[1]!;
+  if (a === '') return '';
+  return /^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(a) ? a : undefined;
+}
+
+/** An address's domain, lower-cased. */
+export const smtpDomain = (address: string): string => address.slice(address.lastIndexOf('@') + 1).toLowerCase();
+
+/** MAIL's argument: the sender and its SIZE, BODY and SMTPUTF8 parameters, or why it is refused (the pack words it). */
+// source: RFC 5321 §4.1.1.2 "MAIL FROM:<reverse-path> [SP <mail-parameters> ] <CRLF>"
+export function smtpMailFrom(arg: string): { from: string; size?: number; smtputf8: boolean } | { refused: 'syntax' | 'sender' | 'size' } | { refused: 'parameter'; parameter: string } {
+  const m = /^FROM:\s*(<[^>]*>)((?:\s+\S+)*)\s*$/i.exec(arg);
+  if (!m) return { refused: 'syntax' };
+  const from = smtpAddress(m[1]!);
+  if (from === undefined) return { refused: 'sender' };
+  let size: number | undefined;
+  let smtputf8 = false;
+  for (const parameter of m[2]!.trim().split(/\s+/).filter(Boolean)) {
+    const [k = '', v] = parameter.split('=');
+    if (k.toUpperCase() === 'SIZE') { if (!/^\d+$/.test(v ?? '')) return { refused: 'size' }; size = Number(v); }
+    else if (k.toUpperCase() === 'BODY' && ['7BIT', '8BITMIME'].includes((v ?? '').toUpperCase())) continue;
+    else if (k.toUpperCase() === 'SMTPUTF8' && v === undefined) smtputf8 = true;
+    else return { refused: 'parameter', parameter };
+  }
+  return { from, ...(size !== undefined ? { size } : {}), smtputf8 };
+}
+
+/** RCPT's argument: the recipient, or why it is refused (the pack words it). */
+// source: RFC 5321 §4.1.1.3 "RCPT TO:<forward-path> [ SP <rcpt-parameters> ] <CRLF>"
+export function smtpRcptTo(arg: string): { to: string } | { refused: 'syntax' | 'recipient' } | { refused: 'parameter'; parameter: string } {
+  const m = /^TO:\s*(<[^>]*>)(?:\s+\S+)*\s*$/i.exec(arg);
+  if (!m) return { refused: 'syntax' };
+  // source: RFC 5321 §4.1.1.11 "command, it will return code 555."
+  if (/^TO:\s*<[^>]*>\s+\S/i.test(arg)) return { refused: 'parameter', parameter: arg.slice(arg.indexOf('>') + 1).trim() };
+  const to = smtpAddress(m[1]!);
+  return to ? { to } : { refused: 'recipient' };
+}
+
+/** A transaction's state cleared, the greeting kept (RSET, a new EHLO, the end of DATA): the session's `stage`,
+ *  `greeted` and an empty `rcpt`, with whatever of the session the pack keeps across transactions (`kept`). */
+// source: RFC 5321 §4.1.1.5 "This command specifies that the current mail transaction will be aborted."
+export function smtpCleared<S extends { stage: string; rcpt: string[]; greeted?: 'helo' | 'ehlo' }>(s: S, stage: string, kept: Omit<S, 'stage' | 'rcpt' | 'greeted' | 'from'>): Omit<S, 'stage' | 'rcpt' | 'greeted' | 'from'> & { stage: string; rcpt: string[]; greeted?: 'helo' | 'ehlo' } {
+  const greeted = stage === 'helo' || stage === 'ehlo' ? stage : s.greeted;
+  return { ...kept, stage, rcpt: [], ...(greeted ? { greeted } : {}) };
+}