@neurocode-ai/http-recorder 1.18.14 → 1.18.15

package/src/recorder.ts

@@ -0,0 +1,62 @@
+import { Effect, Scope, SynchronizedRef } from "effect"
+import type * as CassetteService from "./cassette.js"
+import type { CassetteNotFoundError } from "./cassette.js"
+import type { Interaction } from "./schema.js"
+
+const isCI = () => {
+  const value = process.env.CI
+  return value !== undefined && value !== "" && value !== "false" && value !== "0"
+}
+
+export const resolveAutoMode = (
+  cassette: CassetteService.Interface,
+  name: string,
+): Effect.Effect<"record" | "replay" | "passthrough"> =>
+  Effect.gen(function* () {
+    if (isCI()) return "replay"
+    return (yield* cassette.exists(name)) ? "replay" : "record"
+  })
+
+export interface ReplayState<T> {
+  readonly claim: <E>(
+    validate: (interaction: T | undefined, index: number, interactions: ReadonlyArray<T>) => Effect.Effect<void, E>,
+  ) => Effect.Effect<{ readonly interaction: T; readonly index: number }, CassetteNotFoundError | E>
+}
+
+export const makeReplayState = <T>(
+  cassette: CassetteService.Interface,
+  name: string,
+  project: (interactions: ReadonlyArray<Interaction>) => ReadonlyArray<T>,
+): Effect.Effect<ReplayState<T>, never, Scope.Scope> =>
+  Effect.gen(function* () {
+    const load = yield* Effect.cached(cassette.read(name).pipe(Effect.map(project)))
+    const position = yield* SynchronizedRef.make(0)
+
+    yield* Effect.addFinalizer(() =>
+      Effect.gen(function* () {
+        const used = yield* SynchronizedRef.get(position)
+        if (used === 0) return yield* Effect.void
+        const interactions = yield* load.pipe(Effect.orDie)
+        if (used < interactions.length)
+          return yield* Effect.die(
+            new Error(`Unused recorded interactions in ${name}: used ${used} of ${interactions.length}`),
+          )
+        return yield* Effect.void
+      }),
+    )
+
+    return {
+      claim: (validate) =>
+        Effect.flatMap(load, (interactions) =>
+          SynchronizedRef.modifyEffect(position, (index) =>
+            Effect.gen(function* () {
+              const interaction = interactions[index]
+              yield* validate(interaction, index, interactions)
+              if (interaction === undefined)
+                return yield* Effect.die("Replay validation accepted a missing interaction")
+              return [{ interaction, index }, index + 1] as const
+            }),
+          ),
+        ),
+    }
+  })

package/src/redaction.ts

@@ -0,0 +1,117 @@
+import { Schema } from "effect"
+
+export const REDACTED = "[REDACTED]"
+
+const DEFAULT_REDACT_HEADERS = [
+  "authorization",
+  "cookie",
+  "proxy-authorization",
+  "set-cookie",
+  "x-api-key",
+  "x-amz-security-token",
+  "x-goog-api-key",
+]
+
+const DEFAULT_REDACT_QUERY = [
+  "access_token",
+  "api-key",
+  "api_key",
+  "apikey",
+  "code",
+  "key",
+  "signature",
+  "sig",
+  "token",
+  "x-amz-credential",
+  "x-amz-security-token",
+  "x-amz-signature",
+]
+
+const SECRET_PATTERNS: ReadonlyArray<{ readonly label: string; readonly pattern: RegExp }> = [
+  { label: "bearer token", pattern: /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}\b/i },
+  { label: "API key", pattern: /\bsk-[A-Za-z0-9][A-Za-z0-9_-]{20,}\b/ },
+  { label: "Anthropic API key", pattern: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
+  { label: "Google API key", pattern: /\bAIza[0-9A-Za-z_-]{20,}\b/ },
+  { label: "AWS access key", pattern: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/ },
+  { label: "GitHub token", pattern: /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/ },
+  { label: "private key", pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
+]
+
+const ENV_SECRET_NAMES = /(?:API|AUTH|BEARER|CREDENTIAL|KEY|PASSWORD|SECRET|TOKEN)/i
+const SAFE_ENV_VALUES = new Set(["fixture", "test", "test-key"])
+
+const envSecrets = () =>
+  Object.entries(process.env).flatMap(([name, value]) => {
+    if (!value) return []
+    if (!ENV_SECRET_NAMES.test(name)) return []
+    if (value.length < 12) return []
+    if (SAFE_ENV_VALUES.has(value.toLowerCase())) return []
+    return [{ name, value }]
+  })
+
+const pathFor = (base: string, key: string) => (base ? `${base}.${key}` : key)
+
+const stringEntries = (value: unknown, base = ""): ReadonlyArray<{ readonly path: string; readonly value: string }> => {
+  if (typeof value === "string") return [{ path: base, value }]
+  if (Array.isArray(value)) return value.flatMap((item, index) => stringEntries(item, `${base}[${index}]`))
+  if (value && typeof value === "object") {
+    return Object.entries(value).flatMap(([key, child]) => stringEntries(child, pathFor(base, key)))
+  }
+  return []
+}
+
+const redactionSet = (values: ReadonlyArray<string> | undefined, defaults: ReadonlyArray<string>) =>
+  new Set([...defaults, ...(values ?? [])].map((value) => value.toLowerCase()))
+
+export type UrlRedactor = (url: string) => string
+
+export const redactUrl = (
+  raw: string,
+  query: ReadonlyArray<string> = DEFAULT_REDACT_QUERY,
+  urlRedactor?: UrlRedactor,
+) => {
+  if (!URL.canParse(raw)) return urlRedactor?.(raw) ?? raw
+  const url = new URL(raw)
+  if (url.username) url.username = REDACTED
+  if (url.password) url.password = REDACTED
+  const redacted = redactionSet(query, DEFAULT_REDACT_QUERY)
+  for (const key of url.searchParams.keys()) {
+    if (redacted.has(key.toLowerCase())) url.searchParams.set(key, REDACTED)
+  }
+  return urlRedactor?.(url.toString()) ?? url.toString()
+}
+
+export const redactHeaders = (
+  headers: Record<string, string>,
+  allow: ReadonlyArray<string>,
+  redact: ReadonlyArray<string> = DEFAULT_REDACT_HEADERS,
+) => {
+  const allowed = new Set(allow.map((name) => name.toLowerCase()))
+  const redacted = redactionSet(redact, DEFAULT_REDACT_HEADERS)
+  return Object.fromEntries(
+    Object.entries(headers)
+      .map(([name, value]) => [name.toLowerCase(), value] as const)
+      .filter(([name]) => allowed.has(name))
+      .map(([name, value]) => [name, redacted.has(name) ? REDACTED : value] as const)
+      .toSorted(([a], [b]) => a.localeCompare(b)),
+  )
+}
+
+export const SecretFindingSchema = Schema.Struct({
+  path: Schema.String,
+  reason: Schema.String,
+})
+export type SecretFinding = Schema.Schema.Type<typeof SecretFindingSchema>
+
+export const secretFindings = (value: unknown): ReadonlyArray<SecretFinding> => {
+  const environment = envSecrets()
+  return stringEntries(value).flatMap((entry) => [
+    ...SECRET_PATTERNS.filter((item) => item.pattern.test(entry.value)).map((item) => ({
+      path: entry.path,
+      reason: item.label,
+    })),
+    ...environment
+      .filter((item) => entry.value.includes(item.value))
+      .map((item) => ({ path: entry.path, reason: `environment secret ${item.name}` })),
+  ])
+}

package/src/redactor.ts

@@ -0,0 +1,135 @@
+import { Option } from "effect"
+import { decodeJson } from "./matching.js"
+import { REDACTED, redactHeaders, redactUrl } from "./redaction.js"
+import type { RedactOptions, RequestSnapshot, ResponseSnapshot } from "./types.js"
+
+export type { RedactOptions } from "./types.js"
+
+export const DEFAULT_REQUEST_HEADERS: ReadonlyArray<string> = ["content-type", "accept", "openai-beta"]
+export const DEFAULT_RESPONSE_HEADERS: ReadonlyArray<string> = ["content-type"]
+
+const identity = <T>(value: T) => value
+
+export interface Redactor {
+  readonly request: (snapshot: RequestSnapshot) => RequestSnapshot
+  readonly response: (snapshot: ResponseSnapshot) => ResponseSnapshot
+}
+
+export const compose = (...redactors: ReadonlyArray<Partial<Redactor>>): Redactor => {
+  const requests = redactors.map((r) => r.request).filter((fn): fn is Redactor["request"] => fn !== undefined)
+  const responses = redactors.map((r) => r.response).filter((fn): fn is Redactor["response"] => fn !== undefined)
+  return {
+    request: requests.length === 0 ? identity : (snapshot) => requests.reduce((acc, fn) => fn(acc), snapshot),
+    response: responses.length === 0 ? identity : (snapshot) => responses.reduce((acc, fn) => fn(acc), snapshot),
+  }
+}
+
+export interface HeaderOptions {
+  readonly allow?: ReadonlyArray<string>
+  readonly redact?: ReadonlyArray<string>
+}
+
+export const requestHeaders = (options: HeaderOptions = {}): Partial<Redactor> => ({
+  request: (snapshot) => ({
+    ...snapshot,
+    headers: redactHeaders(snapshot.headers, options.allow ?? DEFAULT_REQUEST_HEADERS, options.redact),
+  }),
+})
+
+export const responseHeaders = (options: HeaderOptions = {}): Partial<Redactor> => ({
+  response: (snapshot) => ({
+    ...snapshot,
+    headers: redactHeaders(snapshot.headers, options.allow ?? DEFAULT_RESPONSE_HEADERS, options.redact),
+  }),
+})
+
+export interface UrlOptions {
+  readonly query?: ReadonlyArray<string>
+  readonly transform?: (url: string) => string
+}
+
+export const url = (options: UrlOptions = {}): Partial<Redactor> => ({
+  request: (snapshot) => ({ ...snapshot, url: redactUrl(snapshot.url, options.query, options.transform) }),
+})
+
+export const body = (transform: (parsed: unknown) => unknown): Partial<Redactor> => ({
+  request: (snapshot) => ({
+    ...snapshot,
+    body: Option.match(decodeJson(snapshot.body), {
+      onNone: () => snapshot.body,
+      onSome: (parsed) => JSON.stringify(transform(parsed)),
+    }),
+  }),
+})
+
+export interface DefaultRedactorOverrides {
+  readonly requestHeaders?: HeaderOptions
+  readonly responseHeaders?: HeaderOptions
+  readonly url?: UrlOptions
+  readonly body?: (parsed: unknown) => unknown
+}
+
+const DEFAULT_REDACT_JSON_FIELDS = [
+  "access_token",
+  "api_key",
+  "apikey",
+  "client_secret",
+  "password",
+  "refresh_token",
+  "secret",
+  "token",
+]
+
+const normalizeField = (field: string) => field.replace(/[^a-z0-9]/gi, "").toLowerCase()
+
+const redactJsonFields = (value: unknown, fields: ReadonlySet<string>): unknown => {
+  if (Array.isArray(value)) return value.map((item) => redactJsonFields(item, fields))
+  if (!value || typeof value !== "object") return value
+  return Object.fromEntries(
+    Object.entries(value).map(([key, child]) => [
+      key,
+      fields.has(normalizeField(key)) ? REDACTED : redactJsonFields(child, fields),
+    ]),
+  )
+}
+
+const redactBody = (value: string, fields: ReadonlySet<string>, transform: ((body: string) => string) | undefined) => {
+  const redacted = Option.match(decodeJson(value), {
+    onNone: () => value,
+    onSome: (parsed) => JSON.stringify(redactJsonFields(parsed, fields)),
+  })
+  return transform?.(redacted) ?? redacted
+}
+
+export const make = (options: RedactOptions = {}): Redactor => {
+  const fields = new Set([...DEFAULT_REDACT_JSON_FIELDS, ...(options.jsonFields ?? [])].map(normalizeField))
+  return compose(
+    requestHeaders({
+      allow: [...DEFAULT_REQUEST_HEADERS, ...(options.allowRequestHeaders ?? []), ...(options.headers ?? [])],
+      redact: options.headers,
+    }),
+    responseHeaders({
+      allow: [...DEFAULT_RESPONSE_HEADERS, ...(options.allowResponseHeaders ?? []), ...(options.headers ?? [])],
+      redact: options.headers,
+    }),
+    url({ query: options.queryParameters, transform: options.url }),
+    {
+      request: (snapshot) => ({
+        ...snapshot,
+        body: redactBody(snapshot.body, fields, options.body),
+      }),
+      response: (snapshot) => ({
+        ...snapshot,
+        body: redactBody(snapshot.body, fields, options.body),
+      }),
+    },
+  )
+}
+
+export const defaults = (overrides: DefaultRedactorOverrides = {}): Redactor =>
+  compose(
+    requestHeaders(overrides.requestHeaders),
+    responseHeaders(overrides.responseHeaders),
+    url(overrides.url),
+    ...(overrides.body ? [body(overrides.body)] : []),
+  )

package/src/schema.ts

@@ -0,0 +1,87 @@
+import { Schema } from "effect"
+import type {
+  CassetteMetadata,
+  HttpInteraction,
+  RequestSnapshot,
+  ResponseSnapshot,
+  WebSocketEvent,
+  WebSocketInteraction,
+} from "./types.js"
+
+export type {
+  CassetteMetadata,
+  HttpInteraction,
+  RequestSnapshot,
+  ResponseSnapshot,
+  WebSocketEvent,
+  WebSocketInteraction,
+} from "./types.js"
+
+export const RequestSnapshotSchema = Schema.Struct({
+  method: Schema.String,
+  url: Schema.String,
+  headers: Schema.Record(Schema.String, Schema.String),
+  body: Schema.String,
+})
+
+export const ResponseSnapshotSchema = Schema.Struct({
+  status: Schema.Number,
+  headers: Schema.Record(Schema.String, Schema.String),
+  body: Schema.String,
+  bodyEncoding: Schema.optional(Schema.Literals(["text", "base64"])),
+})
+
+export const CassetteMetadataSchema = Schema.Record(Schema.String, Schema.Unknown)
+
+export const HttpInteractionSchema = Schema.Struct({
+  transport: Schema.tag("http"),
+  request: RequestSnapshotSchema,
+  response: ResponseSnapshotSchema,
+})
+
+export const WebSocketEventSchema = Schema.Union([
+  Schema.Struct({
+    direction: Schema.Literals(["client", "server"]),
+    kind: Schema.tag("text"),
+    body: Schema.String,
+  }),
+  Schema.Struct({
+    direction: Schema.Literals(["client", "server"]),
+    kind: Schema.tag("binary"),
+    body: Schema.String,
+    bodyEncoding: Schema.Literal("base64"),
+  }),
+])
+
+export const WebSocketInteractionSchema = Schema.Struct({
+  transport: Schema.tag("websocket"),
+  open: Schema.Struct({
+    url: Schema.String,
+    headers: Schema.Record(Schema.String, Schema.String),
+  }),
+  events: Schema.Array(WebSocketEventSchema),
+})
+
+export const InteractionSchema = Schema.Union([HttpInteractionSchema, WebSocketInteractionSchema]).pipe(
+  Schema.toTaggedUnion("transport"),
+)
+export type Interaction = Schema.Schema.Type<typeof InteractionSchema>
+
+export const isHttpInteraction = InteractionSchema.guards.http
+
+export const isWebSocketInteraction = InteractionSchema.guards.websocket
+
+export const httpInteractions = (interactions: ReadonlyArray<Interaction>) => interactions.filter(isHttpInteraction)
+
+export const webSocketInteractions = (interactions: ReadonlyArray<Interaction>) =>
+  interactions.filter(isWebSocketInteraction)
+
+export const CassetteSchema = Schema.Struct({
+  version: Schema.Literal(1),
+  metadata: Schema.optional(CassetteMetadataSchema),
+  interactions: Schema.Array(InteractionSchema),
+})
+export type Cassette = Schema.Schema.Type<typeof CassetteSchema>
+
+export const decodeCassette = Schema.decodeUnknownSync(CassetteSchema)
+export const encodeCassette = Schema.encodeSync(CassetteSchema)