@stigmer/temporal-codecs 3.12.9

package/src/claimcheck/payload-codec.ts

@@ -0,0 +1,144 @@
+/**
+ * Temporal PayloadCodec that transparently offloads large payloads to
+ * external storage ({@link ClaimcheckStorage}). Payloads below the
+ * threshold pass through unchanged. Payloads at or above the threshold
+ * are compressed (optional), uploaded, and replaced with a small
+ * reference marker.
+ *
+ * On decode, markers are detected, the original payload is downloaded
+ * and decompressed, and the original bytes are restored — transparent
+ * to workflow/activity code.
+ *
+ * Moved from backend/services/runner/src/claimcheck/payload-codec.ts when
+ * the codecs became @stigmer/temporal-codecs (one home for the
+ * cross-language envelope contract; the TS server is the second consumer).
+ */
+
+import { randomUUID } from "node:crypto";
+import type { Payload, PayloadCodec } from "@temporalio/common";
+import type { ClaimcheckStorage } from "./storage.js";
+import type { ClaimcheckConfig } from "./config.js";
+import { compress, decompress } from "./compressor.js";
+
+const MARKER_METADATA_KEY = "encoding";
+const MARKER_ENCODING_VALUE = "binary/claimcheck";
+
+interface ClaimcheckMarker {
+  key: string;
+  size: number;
+  compressed: boolean;
+  /**
+   * The relocated payload's original metadata, base64-encoded per value.
+   * Without it the restored payload would carry the marker's own
+   * "binary/claimcheck" encoding and no payload converter could interpret
+   * it. Absent on markers written before this field existed; those decode
+   * with the legacy (metadata-less) behavior.
+   */
+  metadata?: Record<string, string>;
+}
+
+export class ClaimcheckPayloadCodec implements PayloadCodec {
+  constructor(
+    private readonly storage: ClaimcheckStorage,
+    private readonly config: ClaimcheckConfig,
+  ) {}
+
+  async encode(payloads: Payload[]): Promise<Payload[]> {
+    return Promise.all(payloads.map((p) => this.encodePayload(p)));
+  }
+
+  async decode(payloads: Payload[]): Promise<Payload[]> {
+    return Promise.all(payloads.map((p) => this.decodePayload(p)));
+  }
+
+  private async encodePayload(payload: Payload): Promise<Payload> {
+    const data = payload.data;
+    if (!data || data.length < this.config.thresholdBytes) {
+      return payload;
+    }
+
+    const originalBuf = Buffer.from(data);
+    let uploadBuf = originalBuf;
+    let compressed = false;
+
+    if (this.config.compressionEnabled) {
+      const compressedBuf = compress(originalBuf);
+      if (compressedBuf.length < originalBuf.length) {
+        uploadBuf = compressedBuf;
+        compressed = true;
+      }
+    }
+
+    const key = `${this.config.keyPrefix}${randomUUID()}`;
+    await this.storage.upload(key, uploadBuf, "application/octet-stream");
+
+    const marker: ClaimcheckMarker = {
+      key,
+      size: data.length,
+      compressed,
+      metadata: serializeMetadata(payload.metadata),
+    };
+
+    return {
+      metadata: {
+        [MARKER_METADATA_KEY]: Buffer.from(MARKER_ENCODING_VALUE),
+      },
+      data: Buffer.from(JSON.stringify(marker)),
+    };
+  }
+
+  private async decodePayload(payload: Payload): Promise<Payload> {
+    if (!this.isClaimcheckPayload(payload)) {
+      return payload;
+    }
+
+    const marker: ClaimcheckMarker = JSON.parse(
+      Buffer.from(payload.data!).toString("utf-8"),
+    );
+
+    let rawBuf: Buffer;
+    try {
+      rawBuf = await this.storage.download(marker.key);
+    } catch (err) {
+      // Preserve the claimcheck-scoped error contract; the shared download error
+      // already carries the HTTP status (proxy) or the miss (local) as the cause.
+      const cause = err instanceof Error ? err.message : String(err);
+      throw new Error(`Claimcheck retrieve failed for key ${marker.key}: ${cause}`);
+    }
+    const dataBuf = marker.compressed ? decompress(rawBuf) : rawBuf;
+
+    return {
+      metadata: marker.metadata
+        ? deserializeMetadata(marker.metadata)
+        : payload.metadata,
+      data: dataBuf,
+    };
+  }
+
+  private isClaimcheckPayload(payload: Payload): boolean {
+    const encoding = payload.metadata?.[MARKER_METADATA_KEY];
+    if (!encoding) return false;
+    return Buffer.from(encoding).toString("utf-8") === MARKER_ENCODING_VALUE;
+  }
+}
+
+function serializeMetadata(
+  metadata: Payload["metadata"],
+): Record<string, string> | undefined {
+  if (!metadata) return undefined;
+  const out: Record<string, string> = {};
+  for (const [key, value] of Object.entries(metadata)) {
+    if (value) out[key] = Buffer.from(value).toString("base64");
+  }
+  return out;
+}
+
+function deserializeMetadata(
+  metadata: Record<string, string>,
+): Record<string, Uint8Array> {
+  const out: Record<string, Uint8Array> = {};
+  for (const [key, value] of Object.entries(metadata)) {
+    out[key] = Buffer.from(value, "base64");
+  }
+  return out;
+}

package/src/claimcheck/storage.ts

@@ -0,0 +1,19 @@
+/**
+ * The blob-store contract the claim-check codec needs — nothing more.
+ *
+ * Deliberately minimal (upload + download by key) so any consumer's
+ * storage client satisfies it structurally: the runner's ArtifactStorage
+ * (a richer interface with presigned-URL minting) matches as-is, and the
+ * TS server brings its own. Widening this port widens the wire contract's
+ * dependency surface for every consumer — do not add methods the codec
+ * itself does not call.
+ */
+export interface ClaimcheckStorage {
+  /** Store `content` under `key`; resolves once the blob is durable. */
+  upload(key: string, content: Buffer, contentType?: string): Promise<string>;
+  /**
+   * Return the exact bytes stored under `key`, or throw a descriptive,
+   * key-scoped `Error` when the object is missing or the transport fails.
+   */
+  download(key: string): Promise<Buffer>;
+}

package/src/encryption/config.ts

@@ -0,0 +1,174 @@
+/**
+ * Payload-encryption configuration (stigmer-cloud#227, stigmer#398).
+ *
+ * Two key sources, strict precedence:
+ *
+ *   1. Environment (STIGMER_PAYLOAD_ENCRYPTION_KEY(_ID)) — the operator's
+ *      explicit choice: self-hosted deployments sharing one key with their
+ *      server, and cloud sandboxes injected with the platform key. When the
+ *      env key is set, bootstrap-delivered material is ignored entirely.
+ *   2. Bootstrap-delivered — server-managed per-identity keys handed to
+ *      desktop-class runners by getRunnerBootstrapConfig. Held in memory
+ *      only; persistence lives server-side, which is what makes Temporal
+ *      replay work across runner restarts (every boot re-fetches the SAME
+ *      key).
+ *
+ * Encryption is enabled iff a key is present from either source — the same
+ * enabled-iff-configured pattern as the claim-check codec. A malformed key
+ * fails the boot rather than silently running plaintext: an operator who set
+ * the key (or a server that minted one) intended history to be encrypted.
+ *
+ * Key rotation: payloads carry the id of the key that encrypted them.
+ * During a rotation window the previous key stays readable via the
+ * secondary pair while new payloads are written under the primary key.
+ * Workers capture keys at construction, so a rotated bootstrap key lands
+ * on the next runner boot — there is no live re-key.
+ *
+ * Moved from backend/services/runner/src/encryption/config.ts when the
+ * codecs became @stigmer/temporal-codecs (one home for the cross-language
+ * envelope contract; the TS server is the second consumer). Key VALUES are
+ * read through an injected {@link SecretReader} because secret custody is
+ * consumer policy, not codec policy: the runner routes reads through its
+ * boot-capture credential store (stigmer#508 — secrets must not live in
+ * process.env where agent shells could read them), while other consumers
+ * read their own stores. No process.env default is provided for the
+ * VALUES: a forgotten injection must fail to compile, not silently bypass
+ * a consumer's secret custody.
+ */
+
+export interface EncryptionKey {
+  readonly keyId: string;
+  /** 32-byte AES-256 key. */
+  readonly key: Buffer;
+}
+
+export interface PayloadEncryptionConfig {
+  /** Key used to encrypt outgoing payloads (and decrypt its own). */
+  readonly primary: EncryptionKey;
+  /** Decrypt-only key accepted during rotation windows. */
+  readonly secondary?: EncryptionKey;
+}
+
+/**
+ * Reads one secret VALUE by its env-var name. Consumers inject their own
+ * custody policy (see the module doc); the *_KEY_ID companions are
+ * rotation bookkeeping, not secrets, and stay plain `process.env` reads.
+ */
+export type SecretReader = (name: string) => string | undefined;
+
+/**
+ * Server-managed key material delivered by the runner's
+ * getRunnerBootstrapConfig. Structurally mirrors
+ * {@link BootstrapPayloadEncryptionKeys} in the runner's stigmer-client.ts
+ * — declared here so the library stays free of client imports.
+ */
+export interface BootstrapKeyMaterial {
+  readonly key: string;
+  readonly keyId?: string;
+  readonly secondaryKey?: string;
+  readonly secondaryKeyId?: string;
+}
+
+const KEY_ENV = "STIGMER_PAYLOAD_ENCRYPTION_KEY";
+const KEY_ID_ENV = "STIGMER_PAYLOAD_ENCRYPTION_KEY_ID";
+const SECONDARY_KEY_ENV = "STIGMER_PAYLOAD_ENCRYPTION_SECONDARY_KEY";
+const SECONDARY_KEY_ID_ENV = "STIGMER_PAYLOAD_ENCRYPTION_SECONDARY_KEY_ID";
+
+const AES_256_KEY_BYTES = 32;
+
+/**
+ * Returns the encryption config, or undefined when encryption is not
+ * configured (the codec is then simply not installed).
+ *
+ * Source precedence: the env key wins outright; bootstrap-delivered
+ * material applies only when no env key is set (see the module doc).
+ *
+ * @throws when a key is present but malformed, or a key id is missing —
+ *   key misconfiguration must stop the boot, not degrade to plaintext.
+ *   This applies equally to bootstrap material: a server that hands out
+ *   a bad key or omits its id has broken the protocol contract, and
+ *   running plaintext against a server that manages keys would silently
+ *   defeat the feature.
+ */
+export function loadPayloadEncryptionConfig(
+  readSecret: SecretReader,
+  bootstrap?: BootstrapKeyMaterial,
+): PayloadEncryptionConfig | undefined {
+  const rawKey = readSecret(KEY_ENV);
+  if (rawKey) {
+    const primary: EncryptionKey = {
+      keyId: requireKeyId(KEY_ID_ENV),
+      key: parseKey(rawKey, KEY_ENV),
+    };
+
+    const rawSecondary = readSecret(SECONDARY_KEY_ENV);
+    const secondary: EncryptionKey | undefined = rawSecondary
+      ? {
+          keyId: requireKeyId(SECONDARY_KEY_ID_ENV),
+          key: parseKey(rawSecondary, SECONDARY_KEY_ENV),
+        }
+      : undefined;
+
+    return { primary, secondary };
+  }
+
+  if (bootstrap?.key) {
+    const primary: EncryptionKey = {
+      keyId: requireBootstrapKeyId(bootstrap.keyId, "payload_encryption_key_id"),
+      key: parseKey(bootstrap.key, "bootstrap payload_encryption_key"),
+    };
+
+    const secondary: EncryptionKey | undefined = bootstrap.secondaryKey
+      ? {
+          keyId: requireBootstrapKeyId(
+            bootstrap.secondaryKeyId,
+            "payload_encryption_secondary_key_id",
+          ),
+          key: parseKey(bootstrap.secondaryKey, "bootstrap payload_encryption_secondary_key"),
+        }
+      : undefined;
+
+    return { primary, secondary };
+  }
+
+  return undefined;
+}
+
+function requireKeyId(envName: string): string {
+  const keyId = process.env[envName];
+  // An explicit id is required (no default): during rotation two keys
+  // coexist, and payloads must name which one encrypted them.
+  if (!keyId) {
+    throw new Error(
+      `Payload encryption misconfigured: ${envName} is required when the ` +
+        `corresponding key is set`,
+    );
+  }
+  return keyId;
+}
+
+function requireBootstrapKeyId(keyId: string | undefined, fieldName: string): string {
+  if (!keyId) {
+    throw new Error(
+      `Runner bootstrap returned a payload encryption key without its ${fieldName} — ` +
+        `refusing to encrypt under an unidentified key (server contract violation)`,
+    );
+  }
+  return keyId;
+}
+
+function parseKey(rawBase64: string, envName: string): Buffer {
+  let key: Buffer;
+  try {
+    key = Buffer.from(rawBase64, "base64");
+  } catch {
+    throw new Error(`Payload encryption misconfigured: ${envName} is not valid base64`);
+  }
+  if (key.length !== AES_256_KEY_BYTES) {
+    throw new Error(
+      `Payload encryption misconfigured: ${envName} must decode to ` +
+        `${AES_256_KEY_BYTES} bytes (AES-256), got ${key.length}`,
+    );
+  }
+  return key;
+}

package/src/encryption/payload-codec.ts

@@ -0,0 +1,156 @@
+/**
+ * Temporal PayloadCodec that encrypts payloads at rest in workflow
+ * history (stigmer-cloud#227).
+ *
+ * Why: the workflow engine runs inside the Temporal deterministic
+ * sandbox, so decrypted execution-context values cross the history
+ * boundary in many places — the hydrate activity result, the runtime
+ * env passed as an input to every per-task activity, and expression
+ * results recorded as local-activity markers. Encrypting at the payload
+ * codec layer closes the entire class with one mechanism instead of
+ * chasing each crossing.
+ *
+ * Envelope (cross-SDK contract — the Java decode-only codec in
+ * stigmer-cloud's temporal-starter must match it byte-for-byte, pinned
+ * by the conformance fixture in __tests__/fixtures/):
+ *
+ *   metadata: encoding            = "binary/encrypted"
+ *             encryption-key-id   = <key id that encrypted this payload>
+ *   data:     iv (12 bytes) ‖ AES-256-GCM(ciphertext ‖ tag (16 bytes))
+ *
+ * The plaintext is the serialized ORIGINAL Payload proto (metadata AND
+ * data), so decode restores the payload exactly — including its
+ * original encoding — with no side channel.
+ *
+ * Decode passes through payloads it did not encode. This is what keeps
+ * plaintext signals from the Java/Go orchestrators and pre-rollout
+ * in-flight histories working with zero migration. Everything else
+ * fails closed: unknown key id, missing key id, and ciphertext tampering
+ * all throw rather than surfacing bogus payloads.
+ *
+ * Moved from backend/services/runner/src/encryption/payload-codec.ts when
+ * the codecs became @stigmer/temporal-codecs (one home for the
+ * cross-language envelope contract; the TS server is the second consumer).
+ */
+
+import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
+import type { Payload, PayloadCodec } from "@temporalio/common";
+// Default-import + destructure, NOT `import { temporal } from …`:
+// @temporalio/proto is CommonJS and its `temporal` export defeats Node's
+// cjs-module-lexer named-export detection, so the named form loads under
+// tsx/vitest (their interop is looser) but crashes plain `node dist/main.js`
+// at startup with "Named export 'temporal' not found" (stigmer/stigmer#399
+// boot regression). Pinned by scripts/verify-dist-boot.mjs in CI.
+import proto from "@temporalio/proto";
+import type { EncryptionKey, PayloadEncryptionConfig } from "./config.js";
+
+const { temporal } = proto;
+
+const ENCODING_METADATA_KEY = "encoding";
+const ENCRYPTED_ENCODING_VALUE = "binary/encrypted";
+const KEY_ID_METADATA_KEY = "encryption-key-id";
+
+/** AES-GCM parameters shared with the Java implementation. */
+const IV_BYTES = 12;
+const AUTH_TAG_BYTES = 16;
+
+export class EncryptionPayloadCodec implements PayloadCodec {
+  private readonly decryptKeysById: Map<string, Buffer>;
+
+  constructor(private readonly config: PayloadEncryptionConfig) {
+    this.decryptKeysById = new Map([[config.primary.keyId, config.primary.key]]);
+    if (config.secondary) {
+      this.decryptKeysById.set(config.secondary.keyId, config.secondary.key);
+    }
+  }
+
+  async encode(payloads: Payload[]): Promise<Payload[]> {
+    return payloads.map((p) => this.encodePayload(p));
+  }
+
+  async decode(payloads: Payload[]): Promise<Payload[]> {
+    return payloads.map((p) => this.decodePayload(p));
+  }
+
+  private encodePayload(payload: Payload): Payload {
+    // Data-less payloads (binary/null from void results) stay as-is:
+    // there is nothing to protect, and the cross-language parents that
+    // await our workflows as void must be able to read them without a key.
+    if (!payload.data || payload.data.length === 0) {
+      return payload;
+    }
+
+    const plaintext = temporal.api.common.v1.Payload.encode(payload).finish();
+    return {
+      metadata: {
+        [ENCODING_METADATA_KEY]: Buffer.from(ENCRYPTED_ENCODING_VALUE),
+        [KEY_ID_METADATA_KEY]: Buffer.from(this.config.primary.keyId),
+      },
+      data: encrypt(plaintext, this.config.primary),
+    };
+  }
+
+  private decodePayload(payload: Payload): Payload {
+    if (!isEncryptedPayload(payload)) {
+      return payload;
+    }
+
+    const keyIdBytes = payload.metadata?.[KEY_ID_METADATA_KEY];
+    if (!keyIdBytes) {
+      throw new Error(
+        "Encrypted payload is missing its encryption-key-id metadata — refusing to decode",
+      );
+    }
+    const keyId = Buffer.from(keyIdBytes).toString("utf-8");
+    const key = this.decryptKeysById.get(keyId);
+    if (!key) {
+      throw new Error(
+        `Encrypted payload uses unknown key id '${keyId}' — configure it as the ` +
+          `primary or secondary payload encryption key (rotation window?)`,
+      );
+    }
+
+    const plaintext = decrypt(payload.data!, key, keyId);
+    return temporal.api.common.v1.Payload.decode(plaintext);
+  }
+}
+
+function isEncryptedPayload(payload: Payload): boolean {
+  const encoding = payload.metadata?.[ENCODING_METADATA_KEY];
+  if (!encoding) return false;
+  return Buffer.from(encoding).toString("utf-8") === ENCRYPTED_ENCODING_VALUE;
+}
+
+function encrypt(plaintext: Uint8Array, key: EncryptionKey): Buffer {
+  const iv = randomBytes(IV_BYTES);
+  const cipher = createCipheriv("aes-256-gcm", key.key, iv);
+  const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
+  // Layout iv ‖ ciphertext ‖ tag matches Java's AES/GCM/NoPadding, whose
+  // doFinal() output is ciphertext ‖ tag.
+  return Buffer.concat([iv, ciphertext, cipher.getAuthTag()]);
+}
+
+function decrypt(data: Uint8Array, key: Buffer, keyId: string): Buffer {
+  if (data.length < IV_BYTES + AUTH_TAG_BYTES) {
+    throw new Error(
+      `Encrypted payload under key id '${keyId}' is truncated (${data.length} bytes)`,
+    );
+  }
+  const buf = Buffer.from(data);
+  const iv = buf.subarray(0, IV_BYTES);
+  const ciphertext = buf.subarray(IV_BYTES, buf.length - AUTH_TAG_BYTES);
+  const tag = buf.subarray(buf.length - AUTH_TAG_BYTES);
+
+  const decipher = createDecipheriv("aes-256-gcm", key, iv);
+  decipher.setAuthTag(tag);
+  try {
+    return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+  } catch {
+    // GCM auth failure: tampered ciphertext or a key that does not match
+    // its advertised id. Never surface partially decrypted bytes.
+    throw new Error(
+      `Failed to decrypt payload under key id '${keyId}' — ciphertext is ` +
+        `corrupt or the configured key does not match`,
+    );
+  }
+}

package/src/index.ts

@@ -0,0 +1,34 @@
+/**
+ * @stigmer/temporal-codecs — Temporal payload codecs shared by Stigmer's
+ * TypeScript Temporal processes (the runner today; the TS server at its
+ * cutover). The encryption envelope is a cross-language wire contract
+ * (the Java decode-only codec in stigmer-cloud's temporal-starter must
+ * match it byte-for-byte, pinned by the fixture in
+ * src/__tests__/fixtures/), which is why the codecs live in one library
+ * instead of per-consumer copies that could fork.
+ *
+ * This is the package's ONLY public boundary. Codec order at the consumer
+ * is a correctness property: install [encryption, claimcheck] so encode
+ * encrypts before relocating (object storage only ever sees ciphertext)
+ * and decode restores the blob before decrypting.
+ *
+ * Dependency policy: @temporalio/common and @temporalio/proto are pinned
+ * exact and identical to the runner's pins, bumped in lockstep with it —
+ * the runner's consumer-install gate (stigmer#786) fails any version mix
+ * because a split @temporalio/proto tree registers the core-sdk protobuf
+ * namespace twice and crashes worker init.
+ */
+
+export { EncryptionPayloadCodec } from "./encryption/payload-codec.js";
+export { loadPayloadEncryptionConfig } from "./encryption/config.js";
+export type {
+  BootstrapKeyMaterial,
+  EncryptionKey,
+  PayloadEncryptionConfig,
+  SecretReader,
+} from "./encryption/config.js";
+
+export { ClaimcheckPayloadCodec } from "./claimcheck/payload-codec.js";
+export { loadClaimcheckConfig } from "./claimcheck/config.js";
+export type { ClaimcheckConfig } from "./claimcheck/config.js";
+export type { ClaimcheckStorage } from "./claimcheck/storage.js";