@slates/provider 1.0.0-rc.17 → 1.0.0-rc.19

package/src/axios/trace.ts

@@ -1,5 +1,6 @@
 import type { AxiosError, AxiosResponse, InternalAxiosRequestConfig } from 'axios';
-import type { SlateContext } from '../context';
+import { AuthConfigSecretRedactor } from '../auth/redact';
+import type { SlatePublicContext } from '../context';
 
 export interface SlateHttpTraceTextBody {
   contentType?: string;
@@ -29,7 +30,7 @@ export interface SlateHttpTrace {
 }
 
 interface SlateHttpTraceDraft {
-  context: SlateContext<any, any, any>;
+  context: SlatePublicContext<any>;
   startedAt: string;
   startedAtMs: number;
   request: SlateHttpTrace['request'];
@@ -377,10 +378,12 @@ let truncateText = (text: string) => {
   };
 };
 
-let sanitizeFreeText = (text: string) => {
+let sanitizeFreeText = (text: string, redactor?: AuthConfigSecretRedactor) => {
   if (!text) return text;
 
-  let result = text;
+  // Literal auth-config secret values are replaced first -- this catches secrets sitting
+  // under a key name the heuristics below don't recognize as secret-like.
+  let result = redactor ? redactor.redact(text) : text;
 
   result = result.replace(
     HTTP_AUTH_SCHEME_PATTERN,
@@ -406,9 +409,14 @@ let sanitizeFreeText = (text: string) => {
   return result;
 };
 
-let sanitizeScalar = (value: string) => truncateText(sanitizeFreeText(value));
+let sanitizeScalar = (value: string, redactor?: AuthConfigSecretRedactor) =>
+  truncateText(sanitizeFreeText(value, redactor));
 
-let redactStructuredValue = (value: unknown, depth = 0): unknown => {
+let redactStructuredValue = (
+  value: unknown,
+  depth = 0,
+  redactor?: AuthConfigSecretRedactor
+): unknown => {
   if (value == null) return value;
   if (depth >= STRUCTURED_DEPTH_LIMIT) return '[truncated]';
 
@@ -418,13 +426,13 @@ let redactStructuredValue = (value: unknown, depth = 0): unknown => {
     typeof value === 'boolean' ||
     typeof value === 'bigint'
   ) {
-    return typeof value === 'string' ? sanitizeFreeText(value) : value;
+    return typeof value === 'string' ? sanitizeFreeText(value, redactor) : value;
   }
 
   if (Array.isArray(value)) {
     return value
       .slice(0, STRUCTURED_ENTRY_LIMIT)
-      .map(entry => redactStructuredValue(entry, depth + 1));
+      .map(entry => redactStructuredValue(entry, depth + 1, redactor));
   }
 
   if (isRecord(value)) {
@@ -433,7 +441,9 @@ let redactStructuredValue = (value: unknown, depth = 0): unknown => {
         .slice(0, STRUCTURED_ENTRY_LIMIT)
         .map(([key, entry]) => [
           key,
-          isSecretKeyName(key) ? REDACTED_VALUE : redactStructuredValue(entry, depth + 1)
+          isSecretKeyName(key)
+            ? REDACTED_VALUE
+            : redactStructuredValue(entry, depth + 1, redactor)
         ])
     );
   }
@@ -457,7 +467,8 @@ let maybeParseJson = (text: string, contentType?: string) => {
 
 let sanitizeTextBody = (
   text: string,
-  contentType?: string
+  contentType?: string,
+  redactor?: AuthConfigSecretRedactor
 ): SlateHttpTraceTextBody | undefined => {
   if (!text) return undefined;
 
@@ -465,8 +476,8 @@ let sanitizeTextBody = (
   let parsedJson = maybeParseJson(text, normalizedContentType);
   let normalizedText =
     parsedJson !== null
-      ? JSON.stringify(redactStructuredValue(parsedJson))
-      : sanitizeFreeText(text);
+      ? JSON.stringify(redactStructuredValue(parsedJson, 0, redactor))
+      : sanitizeFreeText(text, redactor);
   let { text: truncatedText, truncated } = truncateText(normalizedText);
 
   return {
@@ -487,7 +498,8 @@ let isBinaryLike = (value: unknown) => {
 
 let serializeTextBody = (
   value: unknown,
-  contentType?: string
+  contentType?: string,
+  redactor?: AuthConfigSecretRedactor
 ): SlateHttpTraceTextBody | undefined => {
   let normalizedContentType = normalizeContentType(contentType);
 
@@ -496,19 +508,23 @@ let serializeTextBody = (
   }
 
   if (typeof value === 'string') {
-    return sanitizeTextBody(value, normalizedContentType);
+    return sanitizeTextBody(value, normalizedContentType, redactor);
   }
 
   if (value instanceof URLSearchParams) {
-    return sanitizeTextBody(value.toString(), normalizedContentType);
+    return sanitizeTextBody(value.toString(), normalizedContentType, redactor);
   }
 
   if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint') {
-    return sanitizeTextBody(String(value), normalizedContentType);
+    return sanitizeTextBody(String(value), normalizedContentType, redactor);
   }
 
   if (Array.isArray(value) || isRecord(value)) {
-    return sanitizeTextBody(JSON.stringify(redactStructuredValue(value)), 'application/json');
+    return sanitizeTextBody(
+      JSON.stringify(redactStructuredValue(value, 0, redactor)),
+      'application/json',
+      redactor
+    );
   }
 
   return undefined;
@@ -533,18 +549,21 @@ let decodeRedactedMarker = (value: string) =>
     ? value
     : value.split(REDACTED_ENCODED).join(REDACTED_VALUE);
 
-let redactHashFragment = (hash: string) => {
+let redactHashFragment = (hash: string, redactor?: AuthConfigSecretRedactor) => {
   if (!hash?.includes('=')) return hash;
   let withoutHash = hash.startsWith('#') ? hash.slice(1) : hash;
   let redacted = withoutHash.replace(
     /(?<![A-Za-z0-9])([A-Za-z0-9][A-Za-z0-9_\-.]*)=([^&]*)/g,
-    (match, key: string, _rawValue: string) =>
-      isUrlFormSecretKeyName(key) ? `${key}=${REDACTED_VALUE}` : match
+    (match, key: string, rawValue: string) => {
+      if (isUrlFormSecretKeyName(key)) return `${key}=${REDACTED_VALUE}`;
+      let literalRedacted = redactor ? redactor.redact(rawValue) : rawValue;
+      return literalRedacted !== rawValue ? `${key}=${literalRedacted}` : match;
+    }
   );
   return redacted ? `#${redacted}` : '';
 };
 
-let sanitizeUrl = (value: string) => {
+let sanitizeUrl = (value: string, redactor?: AuthConfigSecretRedactor) => {
   if (!value) return value;
 
   try {
@@ -558,14 +577,21 @@ let sanitizeUrl = (value: string) => {
     for (let key of Array.from(url.searchParams.keys())) {
       if (isUrlFormSecretKeyName(key)) {
         url.searchParams.set(key, REDACTED_VALUE);
+        continue;
       }
+
+      // Not a secret-looking key -- still catch a literal auth-config secret value.
+      let rawValue = url.searchParams.get(key) ?? '';
+      let literalRedacted = redactor ? redactor.redact(rawValue) : rawValue;
+      if (literalRedacted !== rawValue) url.searchParams.set(key, literalRedacted);
     }
 
-    url.hash = redactHashFragment(url.hash);
+    url.hash = redactHashFragment(url.hash, redactor);
 
     return decodeRedactedMarker(url.toString());
   } catch {
-    return value.replace(
+    let text = redactor ? redactor.redact(value) : value;
+    return text.replace(
       /([?&]([^=&#]+)=)([^&#]+)/g,
       (_match, prefix: string, key: string, rawValue: string) =>
         isUrlFormSecretKeyName(key) ? `${prefix}${REDACTED_VALUE}` : `${prefix}${rawValue}`
@@ -618,9 +644,12 @@ let headerEntries = (headers: unknown): [string, string][] => {
   return [];
 };
 
-let sanitizeHeaders = (headers: unknown) => {
+let sanitizeHeaders = (headers: unknown, redactor?: AuthConfigSecretRedactor) => {
   let entries = headerEntries(headers)
-    .map(([key, value]) => [key.toLowerCase(), sanitizeScalar(value).text] as [string, string])
+    .map(
+      ([key, value]) =>
+        [key.toLowerCase(), sanitizeScalar(value, redactor).text] as [string, string]
+    )
     .filter(([key]) => isSafeHeaderName(key));
 
   if (entries.length === 0) return undefined;
@@ -664,10 +693,11 @@ let clearTraceDraft = (
 
 export let attachHttpTraceDraft = (
   config: InternalAxiosRequestConfig,
-  context: SlateContext<any, any, any>
+  context: SlatePublicContext<any>
 ) => {
   let traceAwareConfig = config as TraceAwareAxiosRequestConfig;
   let contentType = getContentType(config.headers);
+  let redactor = new AuthConfigSecretRedactor(context._getAuthConfigForRedaction());
 
   traceAwareConfig.__slatesHttpTraceDraft = {
     context,
@@ -675,10 +705,12 @@ export let attachHttpTraceDraft = (
     startedAtMs: Date.now(),
     request: {
       method: (config.method ?? 'GET').toUpperCase(),
-      url: sanitizeUrl(buildUrl(config.baseURL, config.url)),
-      ...(sanitizeHeaders(config.headers) ? { headers: sanitizeHeaders(config.headers) } : {}),
-      ...(serializeTextBody(config.data, contentType)
-        ? { body: serializeTextBody(config.data, contentType) }
+      url: sanitizeUrl(buildUrl(config.baseURL, config.url), redactor),
+      ...(sanitizeHeaders(config.headers, redactor)
+        ? { headers: sanitizeHeaders(config.headers, redactor) }
+        : {}),
+      ...(serializeTextBody(config.data, contentType, redactor)
+        ? { body: serializeTextBody(config.data, contentType, redactor) }
         : {})
     }
   };
@@ -703,6 +735,7 @@ export let recordHttpTraceFromResponse = (response: AxiosResponse) => {
   if (!draft) return response;
   clearTraceDraft(config, draft);
 
+  let redactor = new AuthConfigSecretRedactor(draft.context._getAuthConfigForRedaction());
   let contentType = getContentType(response.headers);
   draft.context.recordHttpTrace({
     startedAt: draft.startedAt,
@@ -711,11 +744,11 @@ export let recordHttpTraceFromResponse = (response: AxiosResponse) => {
     response: {
       status: response.status,
       ...(response.statusText ? { statusText: response.statusText } : {}),
-      ...(sanitizeHeaders(response.headers)
-        ? { headers: sanitizeHeaders(response.headers) }
+      ...(sanitizeHeaders(response.headers, redactor)
+        ? { headers: sanitizeHeaders(response.headers, redactor) }
         : {}),
-      ...(serializeTextBody(response.data, contentType)
-        ? { body: serializeTextBody(response.data, contentType) }
+      ...(serializeTextBody(response.data, contentType, redactor)
+        ? { body: serializeTextBody(response.data, contentType, redactor) }
         : {})
     }
   });
@@ -729,6 +762,7 @@ export let recordHttpTraceFromError = (error: AxiosError) => {
   if (!draft) return;
   clearTraceDraft(config, draft);
 
+  let redactor = new AuthConfigSecretRedactor(draft.context._getAuthConfigForRedaction());
   let responseContentType = getContentType(error.response?.headers);
   draft.context.recordHttpTrace({
     startedAt: draft.startedAt,
@@ -739,11 +773,11 @@ export let recordHttpTraceFromError = (error: AxiosError) => {
           response: {
             status: error.response.status,
             ...(error.response.statusText ? { statusText: error.response.statusText } : {}),
-            ...(sanitizeHeaders(error.response.headers)
-              ? { headers: sanitizeHeaders(error.response.headers) }
+            ...(sanitizeHeaders(error.response.headers, redactor)
+              ? { headers: sanitizeHeaders(error.response.headers, redactor) }
               : {}),
-            ...(serializeTextBody(error.response.data, responseContentType)
-              ? { body: serializeTextBody(error.response.data, responseContentType) }
+            ...(serializeTextBody(error.response.data, responseContentType, redactor)
+              ? { body: serializeTextBody(error.response.data, responseContentType, redactor) }
               : {})
           }
         }

package/src/context/context.ts

@@ -1,33 +1,104 @@
+import { Readable } from 'node:stream';
+import PQueue from 'p-queue';
+import {
+  createBase64Attachment,
+  createUrlAttachment,
+  type SlateAttachment
+} from '../action/attachment';
+import { type SlateLiveInvocationInfo, uploadAttachmentDirect } from '../action/directUpload';
 import type { SlateHttpTrace } from '../axios/trace';
 import type { SlateLogger, SlateLogMessageInput } from '../logger';
 import type { SlateSpecification } from '../specification/specification';
 
-export class SlateContext<ConfigType extends {}, AuthType extends {}, InputType extends {}> {
-  #config: ConfigType;
+export type { SlateLiveInvocationInfo };
+
+export type SlateAddAttachmentContent =
+  | Buffer
+  | Uint8Array
+  | ArrayBuffer
+  | ReadableStream<Uint8Array>
+  | NodeJS.ReadableStream
+  | Response;
+
+export type SlateAddAttachmentInput =
+  | {
+      type: 'url';
+      url: string | URL;
+      mimeType?: string;
+      filename?: string;
+      headers?: Record<string, string>;
+      query?: Record<string, string>;
+      refreshReference?: unknown;
+      refreshAt?: string;
+    }
+  | {
+      type: 'content';
+      content: SlateAddAttachmentContent;
+      mimeType?: string;
+      filename?: string;
+    };
+
+let isBufferLike = (value: unknown): value is Buffer | Uint8Array | ArrayBuffer =>
+  value instanceof Uint8Array || value instanceof ArrayBuffer;
+
+let toUint8Array = (value: Buffer | Uint8Array | ArrayBuffer) =>
+  value instanceof ArrayBuffer ? new Uint8Array(value) : value;
+
+let isReadableStream = (value: unknown): value is ReadableStream<Uint8Array> =>
+  typeof ReadableStream !== 'undefined' && value instanceof ReadableStream;
+
+let isNodeReadable = (value: unknown): value is NodeJS.ReadableStream =>
+  !!value &&
+  typeof value === 'object' &&
+  typeof (value as any).pipe === 'function' &&
+  typeof (value as any).on === 'function';
+
+let isResponseLike = (value: unknown): value is Response =>
+  typeof Response !== 'undefined' && value instanceof Response;
+
+let readStreamToBuffer = async (stream: ReadableStream<Uint8Array>): Promise<Buffer> => {
+  let reader = stream.getReader();
+  let chunks: Uint8Array[] = [];
+
+  while (true) {
+    let { done, value } = await reader.read();
+    if (done) break;
+    if (value) chunks.push(value);
+  }
+
+  return Buffer.concat(chunks);
+};
+
+interface NormalizedStreamInput {
+  stream: ReadableStream<Uint8Array>;
+  mimeType?: string;
+  filename?: string;
+}
+
+export class SlatePublicContext<InputType extends {}> {
   #input: InputType;
-  #auth: AuthType;
   #httpTraces: SlateHttpTrace[] = [];
 
+  #liveInvocation: SlateLiveInvocationInfo | null;
+  #attachmentsDisabled = false;
+  #attachments: SlateAttachment[] = [];
+  #pendingUploads: Promise<void>[] = [];
+  #uploadQueue = new PQueue({ concurrency: 10 });
+
   constructor(
-    config: ConfigType,
     input: InputType,
-    auth: AuthType,
-    private readonly spec: SlateSpecification<ConfigType, AuthType>,
-    private readonly logger: SlateLogger
+    private readonly spec: SlateSpecification<any, any>,
+    private readonly logger: SlateLogger,
+    liveInvocation: SlateLiveInvocationInfo | null = null
   ) {
-    this.#config = config;
     this.#input = input;
-    this.#auth = auth;
+    this.#liveInvocation = liveInvocation;
   }
 
   get specification() {
     return this.spec;
   }
 
-  get config() {
-    return Object.freeze(this.#config);
-  }
-
   get input() {
     return Object.freeze(this.#input);
   }
@@ -56,10 +127,6 @@ export class SlateContext<ConfigType extends {}, AuthType extends {}, InputType
       : never;
   }
 
-  get auth() {
-    return Object.freeze(this.#auth);
-  }
-
   recordHttpTrace(trace: SlateHttpTrace) {
     this.#httpTraces.push(trace);
   }
@@ -100,4 +167,142 @@ export class SlateContext<ConfigType extends {}, AuthType extends {}, InputType
   progress(message: SlateLogMessageInput) {
     this.logger.progress(message);
   }
+
+  async addAttachment(input: SlateAddAttachmentInput): Promise<void> {
+    if (input.type === 'url') {
+      this.#attachments.push(
+        createUrlAttachment(input.url.toString(), {
+          mimeType: input.mimeType,
+          headers: input.headers,
+          query: input.query,
+          refreshReference: input.refreshReference,
+          refreshAt: input.refreshAt
+        })
+      );
+      return;
+    }
+
+    let content = input.content;
+
+    if (isBufferLike(content)) {
+      let bytes = toUint8Array(content);
+      this.#attachments.push(
+        createBase64Attachment(Buffer.from(bytes).toString('base64'), input.mimeType)
+      );
+      return;
+    }
+
+    let normalized = await this.#normalizeStreamInput(content, {
+      mimeType: input.mimeType,
+      filename: input.filename
+    });
+    if (!normalized) return;
+
+    if (!this.#liveInvocation || this.#attachmentsDisabled) {
+      let buffer = await readStreamToBuffer(normalized.stream);
+      this.#attachments.push(
+        createBase64Attachment(buffer.toString('base64'), normalized.mimeType)
+      );
+      return;
+    }
+
+    let live = this.#liveInvocation;
+    let task = this.#uploadQueue
+      .add(() =>
+        uploadAttachmentDirect({
+          live,
+          mimeType: normalized.mimeType,
+          filename: normalized.filename,
+          body: normalized.stream
+        })
+      )
+      .then(attachment => {
+        this.#attachments.push(attachment);
+      })
+      .catch(err => {
+        this.#attachmentsDisabled = true;
+        this.warn({
+          message: `Attachment dropped: direct upload failed (${
+            err instanceof Error ? err.message : String(err)
+          }).`
+        });
+      });
+
+    this.#pendingUploads.push(task);
+  }
+
+  async #normalizeStreamInput(
+    input: ReadableStream<Uint8Array> | NodeJS.ReadableStream | Response,
+    opts: { mimeType?: string; filename?: string }
+  ): Promise<NormalizedStreamInput | null> {
+    if (isResponseLike(input)) {
+      if (!input.body) {
+        this.warn({ message: 'Attachment dropped: response has no body.' });
+        return null;
+      }
+      return {
+        stream: input.body,
+        mimeType: opts.mimeType ?? input.headers.get('content-type') ?? undefined,
+        filename: opts.filename
+      };
+    }
+
+    if (isReadableStream(input)) {
+      return { stream: input, mimeType: opts.mimeType, filename: opts.filename };
+    }
+
+    if (isNodeReadable(input)) {
+      return {
+        stream: Readable.toWeb(input as any) as unknown as ReadableStream<Uint8Array>,
+        mimeType: opts.mimeType,
+        filename: opts.filename
+      };
+    }
+
+    this.warn({ message: 'Attachment dropped: unsupported attachment input type.' });
+    return null;
+  }
+
+  async _finalizeAttachments(): Promise<SlateAttachment[]> {
+    await Promise.allSettled(this.#pendingUploads);
+    return this.#attachments;
+  }
+
+  _getAuthConfigForRedaction(): Record<string, unknown> | undefined {
+    return undefined;
+  }
+}
+
+export class SlateContext<
+  ConfigType extends {},
+  AuthType extends {},
+  InputType extends {}
+> extends SlatePublicContext<InputType> {
+  #config: ConfigType;
+  #auth: AuthType;
+
+  constructor(
+    config: ConfigType,
+    input: InputType,
+    auth: AuthType,
+    spec: SlateSpecification<ConfigType, AuthType>,
+    logger: SlateLogger,
+    liveInvocation: SlateLiveInvocationInfo | null = null
+  ) {
+    super(input, spec, logger, liveInvocation);
+    this.#config = config;
+    this.#auth = auth;
+  }
+
+  get config() {
+    return Object.freeze(this.#config);
+  }
+
+  get auth() {
+    return Object.freeze(this.#auth);
+  }
+
+  override _getAuthConfigForRedaction(): Record<string, unknown> | undefined {
+    return this.#auth as Record<string, unknown>;
+  }
 }

package/src/context/hook.ts

@@ -1,24 +1,19 @@
 import { AsyncLocalStorage } from 'async_hooks';
-import type { SlateContext } from './context';
+import type { SlateContext, SlatePublicContext } from './context';
 
 let slateContextStorageKey = Symbol.for('slates.provider.context.asyncLocalStorage');
 let globalWithSlateContext = globalThis as typeof globalThis & {
-  [key: symbol]: AsyncLocalStorage<SlateContext<any, any, any>> | undefined;
+  [key: symbol]: AsyncLocalStorage<SlatePublicContext<any>> | undefined;
 };
 
 let asyncLocalStorage =
   globalWithSlateContext[slateContextStorageKey] ??
-  new AsyncLocalStorage<SlateContext<any, any, any>>();
+  new AsyncLocalStorage<SlatePublicContext<any>>();
 
 globalWithSlateContext[slateContextStorageKey] = asyncLocalStorage;
 
-export let runWithContext = <
-  ConfigType extends {},
-  AuthType extends {},
-  InputType extends {},
-  RV
->(
-  context: SlateContext<ConfigType, AuthType, InputType>,
+export let runWithContext = <InputType extends {}, RV>(
+  context: SlatePublicContext<InputType>,
   fn: () => Promise<RV>
 ): Promise<RV> => {
   return asyncLocalStorage.run(context, fn);

package/src/index.ts

@@ -1,5 +1,7 @@
 export * from './action';
 export * from './action/attachment';
+export * from './action/directUpload';
+export * from './adapter';
 export * from './auth';
 export * from './axios';
 export * from './config';

package/src/specification/slate.ts

@@ -1,4 +1,6 @@
 import type { SlateTool, SlateTrigger } from '../action';
+import type { SlateAdapter } from '../adapter';
+import { SlateDeclarationError } from '../error';
 import type { SlateSpecification } from './specification';
 
 export class Slate<ConfigType extends {}, AuthType extends {}> {
@@ -7,15 +9,45 @@ export class Slate<ConfigType extends {}, AuthType extends {}> {
     private readonly _actions: (
       | SlateTrigger<ConfigType, AuthType, any, any>
       | SlateTool<ConfigType, AuthType, any, any>
-    )[]
+    )[],
+    private readonly _adapters: SlateAdapter<ConfigType, AuthType>[]
   ) {}
 
   static create<ConfigType extends {}, AuthType extends {}>(params: {
     spec: SlateSpecification<ConfigType, AuthType>;
     triggers: SlateTrigger<ConfigType, AuthType, any, any>[];
     tools: SlateTool<ConfigType, AuthType, any, any>[];
+    adapters?: SlateAdapter<ConfigType, AuthType>[];
   }) {
-    return new Slate(params.spec, [...params.triggers, ...params.tools]);
+    let adapters = params.adapters ?? [];
+    let actions = [...params.triggers, ...params.tools];
+    let seenAdapterIds = new Set<string>();
+    let seenActionKeys = new Map(actions.map(action => [action.key, action]));
+
+    for (let adapter of adapters) {
+      if (seenAdapterIds.has(adapter.id)) {
+        throw new SlateDeclarationError(
+          `Adapter "${adapter.id}" is registered more than once`
+        );
+      }
+      seenAdapterIds.add(adapter.id);
+
+      for (let action of adapter.actions) {
+        let existing = seenActionKeys.get(action.key);
+        if (existing && existing !== action) {
+          throw new SlateDeclarationError(
+            `Adapter "${adapter.id}" action "${action.key}" conflicts with another action`
+          );
+        }
+
+        if (!existing) {
+          actions.push(action);
+          seenActionKeys.set(action.key, action);
+        }
+      }
+    }
+
+    return new Slate(params.spec, actions, adapters);
   }
 
   get spec() {
@@ -25,4 +57,8 @@ export class Slate<ConfigType extends {}, AuthType extends {}> {
   get actions() {
     return this._actions;
   }
+
+  get adapters() {
+    return this._adapters;
+  }
 }