@shivaedev/platform 0.18.2 → 0.20.0

package/src/server/rpc-server/tracing.ts

@@ -2,9 +2,10 @@ import { Cause, Effect, Layer, Option, Predicate, Random } from "effect";
 import { Headers } from "effect/http";
 import { RequestId } from "#rpc/identity.ts";
 import { RequestTracing } from "#rpc/middleware.ts";
+import { commandChanges } from "#server/commandChanges.ts";
 import { redact } from "./redact.ts";
 import { redactCause } from "./redactCause.ts";
-import type { SensitiveKey } from "./sensitive.ts";
+import { isSensitiveKey, REDACTED, redactText, type SensitiveKey } from "./sensitive.ts";
 export interface RequestTracingOptions {
 	readonly header?: string;
 	readonly sensitive?: SensitiveKey;
@@ -23,10 +24,15 @@ function requestIdOf(headers: Headers.Headers, header: string): Effect.Effect<st
 		},
 	);
 }
-function failureTag(cause: Cause.Cause<unknown>): string {
+function failureTag(cause: Cause.Cause<unknown>, sensitive: SensitiveKey = isSensitiveKey): string {
 	return Option.match(Cause.findErrorOption(cause), {
 		onNone: () => "unknown",
-		onSome: (error) => (Predicate.hasProperty(error, "_tag") && Predicate.isString(error._tag) ? error._tag : "unknown"),
+		onSome: (error) => {
+			if (!(Predicate.hasProperty(error, "_tag") && Predicate.isString(error._tag))) {
+				return "unknown";
+			}
+			return sensitive("_tag") ? REDACTED : redactText(error._tag, sensitive);
+		},
 	});
 }
 function logFailure(cause: Cause.Cause<unknown>, payload: unknown, sensitive: SensitiveKey | undefined): Effect.Effect<void> {
@@ -35,7 +41,7 @@ function logFailure(cause: Cause.Cause<unknown>, payload: unknown, sensitive: Se
 	}
 	const annotations = { "rpc.payload": redact(payload, sensitive) };
 	if (!Cause.hasDies(cause)) {
-		return Effect.annotateLogs(Effect.logInfo("RPC failure"), { ...annotations, "rpc.failure": failureTag(cause) });
+		return Effect.annotateLogs(Effect.logInfo("RPC failure"), { ...annotations, "rpc.failure": failureTag(cause, sensitive) });
 	}
 	const defects = cause.reasons.filter(Cause.isDieReason).map((reason) => redact(reason.defect, sensitive));
 	return Effect.annotateLogs(Effect.logError("RPC defect"), { ...annotations, "rpc.defect": defects });
@@ -45,7 +51,7 @@ export function requestTracingLayer(options: RequestTracingOptions = {}): Layer.
 		Effect.gen(function* () {
 			const requestId = yield* requestIdOf(headers, options.header ?? "x-request-id");
 			yield* Effect.annotateCurrentSpan({ "request.id": requestId, "rpc.method": rpc._tag });
-			return yield* effect.pipe(
+			return yield* commandChanges(effect, rpc, payload).pipe(
 				Effect.catchCauseIf(Cause.hasDies, (cause) => Effect.failCause(redactCause(cause, options.sensitive))),
 				Effect.tapCause((cause) => logFailure(cause, payload, options.sensitive)),
 				Effect.provideService(RequestId, requestId),

package/src/server/safeOtlpSerialization.ts

@@ -1,27 +1,28 @@
 import { Effect, Layer } from "effect";
-import type { OtlpResource } from "effect/observability";
+import type { OtlpResource, OtlpTracer } from "effect/observability";
 import { OtlpSerialization } from "effect/observability";
+import { RedactionPolicy } from "#server/RedactionPolicy.ts";
 import { redactOtlpUrl } from "#server/redactOtlpUrl.ts";
-import { isSensitiveKey, REDACTED, redactText } from "#server/rpc-server/sensitive.ts";
+import { isSensitiveKey, REDACTED, redactText, type SensitiveKey } from "#server/rpc-server/sensitive.ts";
 
-function sensitive(key: string) {
-	return isSensitiveKey(key) || ["db.user", "db.user.name", "db.connection_string"].includes(key.toLowerCase());
+function otlpSensitive(sensitive: SensitiveKey): SensitiveKey {
+	return (key) => sensitive(key) || ["db.user", "db.user.name", "db.connection_string"].includes(key.toLowerCase());
 }
-export function redactOtlpText(value: string, key = "") {
-	return redactText(redactOtlpUrl(value, key), sensitive);
+export function redactOtlpText(value: string, key = "", sensitive: SensitiveKey = isSensitiveKey) {
+	return redactText(redactOtlpUrl(value, key), otlpSensitive(sensitive));
 }
-function attributeValue(value: OtlpResource.AnyValue, key: string): OtlpResource.AnyValue {
+function attributeValue(value: OtlpResource.AnyValue, key: string, sensitive: SensitiveKey): OtlpResource.AnyValue {
 	return {
 		...value,
-		...(typeof value.stringValue === "string" ? { stringValue: redactOtlpText(value.stringValue, key) } : {}),
-		...(value.arrayValue ? { arrayValue: { values: value.arrayValue.values.map((item) => attributeValue(item, key)) } } : {}),
-		...(value.kvlistValue ? { kvlistValue: { values: attributes(value.kvlistValue.values) } } : {}),
+		...(typeof value.stringValue === "string" ? { stringValue: redactOtlpText(value.stringValue, key, sensitive) } : {}),
+		...(value.arrayValue ? { arrayValue: { values: value.arrayValue.values.map((item) => attributeValue(item, key, sensitive)) } } : {}),
+		...(value.kvlistValue ? { kvlistValue: { values: attributes(value.kvlistValue.values, sensitive) } } : {}),
 	};
 }
-function attributes(values: OtlpResource.KeyValue[]): OtlpResource.KeyValue[] {
+function attributes(values: OtlpResource.KeyValue[], sensitive: SensitiveKey): OtlpResource.KeyValue[] {
 	return values.map(({ key, value }) => ({
 		key,
-		value: sensitive(key) ? { stringValue: REDACTED } : attributeValue(value, key),
+		value: sensitive(key) ? { stringValue: REDACTED } : attributeValue(value, key, sensitive),
 	}));
 }
 
@@ -36,28 +37,37 @@ function spanName(span: { readonly name: string; readonly kind: number; readonly
 	return span.name;
 }
 
+function redactTraces(data: OtlpTracer.TraceData, sensitive: SensitiveKey): OtlpTracer.TraceData {
+	function text(value: string) {
+		return redactOtlpText(value, "", sensitive);
+	}
+	return {
+		...data,
+		resourceSpans: data.resourceSpans.map((resource) => ({
+			...resource,
+			resource: { ...resource.resource, attributes: attributes(resource.resource.attributes, sensitive) },
+			scopeSpans: resource.scopeSpans.map((scope) => ({
+				...scope,
+				spans: scope.spans.map((span) => ({
+					...span,
+					attributes: attributes(span.attributes, sensitive),
+					events: span.events.map((event) => ({ ...event, attributes: attributes(event.attributes, sensitive), name: text(event.name) })),
+					links: span.links.map((link) => ({ ...link, attributes: attributes(link.attributes, sensitive) })),
+					name: text(spanName(span)),
+					status: { ...span.status, ...(span.status.message === undefined ? {} : { message: text(span.status.message) }) },
+				})),
+			})),
+		})),
+	};
+}
 export const safeOtlpSerialization = Layer.effect(
 	OtlpSerialization.OtlpSerialization,
-	Effect.map(OtlpSerialization.OtlpSerialization, (serialization) => ({
-		...serialization,
-		traces: (data) =>
-			serialization.traces({
-				...data,
-				resourceSpans: data.resourceSpans.map((resource) => ({
-					...resource,
-					resource: { ...resource.resource, attributes: attributes(resource.resource.attributes) },
-					scopeSpans: resource.scopeSpans.map((scope) => ({
-						...scope,
-						spans: scope.spans.map((span) => ({
-							...span,
-							attributes: attributes(span.attributes),
-							events: span.events.map((event) => ({ ...event, attributes: attributes(event.attributes), name: redactOtlpText(event.name) })),
-							links: span.links.map((link) => ({ ...link, attributes: attributes(link.attributes) })),
-							name: redactOtlpText(spanName(span)),
-							status: { ...span.status, ...(span.status.message === undefined ? {} : { message: redactOtlpText(span.status.message) }) },
-						})),
-					})),
-				})),
-			}),
-	})),
+	Effect.gen(function* () {
+		const serialization = yield* OtlpSerialization.OtlpSerialization;
+		const sensitive = otlpSensitive(yield* RedactionPolicy);
+		return OtlpSerialization.OtlpSerialization.of({
+			...serialization,
+			traces: (data) => serialization.traces(redactTraces(data, sensitive)),
+		});
+	}),
 );

package/src/server/schemaSensitiveKeys.ts

@@ -0,0 +1,148 @@
+import { Predicate, type Schema, type SchemaAST } from "effect";
+import { Rpc, type RpcGroup } from "effect/rpc";
+import { isSensitiveKey } from "#server/rpc-server/sensitive.ts";
+
+function children(ast: SchemaAST.AST): readonly SchemaAST.AST[] {
+	switch (ast._tag) {
+		case "Objects":
+			return [...ast.propertySignatures.map((property) => property.type), ...ast.indexSignatures.flatMap((index) => [index.parameter, index.type])];
+		case "Arrays":
+			return [...ast.elements, ...ast.rest];
+		case "Union":
+			return ast.types;
+		case "Declaration":
+			return ast.typeParameters;
+		case "Suspend":
+			return [ast.thunk()];
+		default:
+			return [];
+	}
+}
+function sensitiveCheck(check: SchemaAST.Check<never>): boolean {
+	return check.annotations?.sensitive === true || (check._tag === "FilterGroup" && check.checks.some(sensitiveCheck));
+}
+function marked(ast: SchemaAST.AST, seen = new Set<SchemaAST.AST>()): boolean {
+	if (seen.has(ast)) {
+		return false;
+	}
+	seen.add(ast);
+	if (
+		ast.annotations?.sensitive === true
+		|| ast.context?.annotations?.sensitive === true
+		|| ast.checks?.some(sensitiveCheck)
+		|| (Predicate.hasProperty(ast.annotations?.representation, "id") && ast.annotations.representation.id === "effect/schema/Redacted")
+	) {
+		return true;
+	}
+	const nested = ast._tag === "Objects" ? ast.indexSignatures.map((index) => index.type) : children(ast);
+	return (ast.encoding?.some((link) => marked(link.to, seen)) ?? false) || nested.some((type) => marked(type, seen));
+}
+function containsSensitive(ast: SchemaAST.AST, keys: ReadonlySet<string>): boolean {
+	return (
+		marked(ast)
+		|| propertiesOf(ast).some((property) => typeof property.name === "string" && (keys.has(property.name) || isSensitiveKey(property.name)))
+	);
+}
+function collectEncodedKeys(ast: SchemaAST.AST, keys: Set<string>) {
+	let source = ast;
+	let sensitive = false;
+	for (const link of ast.encoding ?? []) {
+		sensitive ||= containsSensitive(source, keys);
+		if (sensitive) {
+			for (const property of propertiesOf(link.to)) {
+				if (typeof property.name === "string") {
+					keys.add(property.name);
+				}
+			}
+		}
+		source = link.to;
+	}
+}
+function fixedOutput(ast: SchemaAST.AST): boolean {
+	if (ast._tag === "Union") {
+		return ast.types.every(fixedOutput);
+	}
+	return ast._tag === "Objects" && ast.indexSignatures.length === 0;
+}
+function maskedValue(ast: SchemaAST.AST, keys: ReadonlySet<string>, seen = new Set<SchemaAST.AST>()): boolean {
+	if (seen.has(ast)) {
+		return false;
+	}
+	seen.add(ast);
+	if (marked(ast)) {
+		return true;
+	}
+	let source = ast;
+	let sensitive = false;
+	for (const link of ast.encoding ?? []) {
+		sensitive ||= containsSensitive(source, keys);
+		if (sensitive && !fixedOutput(link.to)) {
+			return true;
+		}
+		source = link.to;
+	}
+	const nested = ast._tag === "Objects" ? ast.indexSignatures.map((index) => index.type) : children(ast);
+	return nested.some((type) => maskedValue(type, keys, seen));
+}
+function collect(ast: SchemaAST.AST, nodes: Set<SchemaAST.AST>, encodings = true) {
+	if (nodes.has(ast)) {
+		return;
+	}
+	nodes.add(ast);
+	for (const child of [...children(ast), ...(encodings ? (ast.encoding?.map((link) => link.to) ?? []) : [])]) {
+		collect(child, nodes, encodings);
+	}
+}
+function propertiesOf(ast: SchemaAST.AST) {
+	const nodes = new Set<SchemaAST.AST>();
+	collect(ast, nodes, false);
+	return [...nodes].flatMap((node) => (node._tag === "Objects" ? node.propertySignatures : []));
+}
+function collectFieldKeys(properties: SchemaAST.Objects["propertySignatures"], keys: Set<string>, sensitive: (ast: SchemaAST.AST) => boolean) {
+	for (const property of properties) {
+		if (typeof property.name === "string" && sensitive(property.type)) {
+			keys.add(property.name);
+		}
+	}
+}
+export function schemaSensitiveKeys<R extends Rpc.Any>(group: RpcGroup.RpcGroup<R>, extra: readonly string[] = []) {
+	const keys = new Set(extra);
+	const nodes = new Set<SchemaAST.AST>();
+	const requests: Rpc.AnyWithProps[] = [];
+	for (const rpc of group.requests.values()) {
+		if (Rpc.isRpc(rpc)) {
+			requests.push(rpc);
+		}
+	}
+	for (const rpc of requests) {
+		const schemas: Schema.Top[] = [
+			rpc.payloadSchema,
+			rpc.successSchema,
+			rpc.errorSchema,
+			rpc.defectSchema,
+			...[...rpc.middlewares].map((m) => m.error),
+		];
+		for (const schema of schemas) {
+			collect(schema.ast, nodes);
+		}
+	}
+	const properties = [...nodes].flatMap((ast) => (ast._tag === "Objects" ? ast.propertySignatures : []));
+	collectFieldKeys(properties, keys, marked);
+	let previousSize: number;
+	do {
+		previousSize = keys.size;
+		collectFieldKeys(properties, keys, (ast) => maskedValue(ast, keys));
+		for (const ast of nodes) {
+			collectEncodedKeys(ast, keys);
+		}
+	} while (keys.size !== previousSize);
+	const roots = requests.map((rpc) => ({
+		die: maskedValue(rpc.defectSchema.ast, keys),
+		fail: [rpc.errorSchema, ...[...rpc.middlewares].map((m) => m.error)].some((schema) => maskedValue(schema.ast, keys)),
+		tag: rpc._tag,
+	}));
+	return Object.assign((key: string) => keys.has(key) || isSensitiveKey(key), {
+		maskFailure: (tag: string | undefined, reason: "Fail" | "Die") =>
+			roots.some((root) => (tag === undefined || tag === root.tag) && (reason === "Fail" ? root.fail : root.die)),
+	});
+}