@ganju/utils 0.0.1

package/src/crypto.ts

@@ -0,0 +1,108 @@
+import { Context } from 'hono';
+import { xchacha20poly1305 } from '@noble/ciphers/chacha.js';
+import { getEnv } from './getEnv';
+import { base64ToBytes, bytesToBase64 } from './base64';
+
+const PREFIX = 'enc:v1:';
+const NONCE_BYTES = 24;
+const KEY_BYTES = 32;
+const KEY_ENV = 'CRYPTO_SECRET';
+
+export const getCredentialEncryptionKey = (c: Context) => {
+  const rawKey = getEnv(c, KEY_ENV);
+  if (!rawKey) {
+    throw new Error(`Missing env: ${KEY_ENV}`);
+  }
+  return rawKey;
+};
+
+const loadKey = (rawKey: string) => {
+  const bytes = base64ToBytes(rawKey);
+  if (bytes.length !== KEY_BYTES) {
+    throw new Error(
+      `CRYPTO_SECRET must decode to ${KEY_BYTES} bytes (got ${bytes.length})`
+    );
+  }
+  return bytes;
+};
+
+export const encryptString = (plaintext: string, rawKey: string) => {
+  const key = loadKey(rawKey);
+  const nonce = crypto.getRandomValues(new Uint8Array(NONCE_BYTES));
+  const cipher = xchacha20poly1305(key, nonce);
+  const ciphertext = cipher.encrypt(new TextEncoder().encode(plaintext));
+  const bundle = new Uint8Array(nonce.length + ciphertext.length);
+  bundle.set(nonce, 0);
+  bundle.set(ciphertext, nonce.length);
+  return `${PREFIX}${bytesToBase64(bundle)}`;
+};
+
+export const decryptString = (value: string, rawKey: string) => {
+  if (!value.startsWith(PREFIX)) return value;
+  const bundle = base64ToBytes(value.slice(PREFIX.length));
+  const nonce = bundle.slice(0, NONCE_BYTES);
+  const ciphertext = bundle.slice(NONCE_BYTES);
+  const key = loadKey(rawKey);
+  const cipher = xchacha20poly1305(key, nonce);
+  const plaintext = cipher.decrypt(ciphertext);
+  return new TextDecoder().decode(plaintext);
+};
+
+export const sha256Hex = async (input: string) => {
+  const digest = await crypto.subtle.digest(
+    'SHA-256',
+    new TextEncoder().encode(input)
+  );
+  const bytes = new Uint8Array(digest);
+  let hex = '';
+  for (let i = 0; i < bytes.length; i++) {
+    hex += bytes[i].toString(16).padStart(2, '0');
+  }
+  return hex;
+};
+
+/**
+ * SHA-256 as unpadded base64url — the format `@better-auth/oauth-provider`
+ * stores hashed client secrets in. Anything comparing against a stored
+ * `oauth_client.client_secret` has to hash the candidate exactly this way.
+ */
+export const sha256Base64Url = async (input: string) => {
+  const digest = await crypto.subtle.digest(
+    'SHA-256',
+    new TextEncoder().encode(input)
+  );
+  const bytes = new Uint8Array(digest);
+  let binary = '';
+  for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+  return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+};
+
+export const hmacSha256Hex = async (key: string, message: string) => {
+  const cryptoKey = await crypto.subtle.importKey(
+    'raw',
+    new TextEncoder().encode(key),
+    { name: 'HMAC', hash: 'SHA-256' },
+    false,
+    ['sign']
+  );
+  const signature = await crypto.subtle.sign(
+    'HMAC',
+    cryptoKey,
+    new TextEncoder().encode(message)
+  );
+  const bytes = new Uint8Array(signature);
+  let hex = '';
+  for (let i = 0; i < bytes.length; i++) {
+    hex += bytes[i].toString(16).padStart(2, '0');
+  }
+  return hex;
+};
+
+export const timingSafeEqual = (a: string, b: string) => {
+  if (a.length !== b.length) return false;
+  let diff = 0;
+  for (let i = 0; i < a.length; i++) {
+    diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  }
+  return diff === 0;
+};

package/src/customCodeProject.ts

@@ -0,0 +1,182 @@
+import { constants } from './constants';
+
+/**
+ * A dashboard-authored script is a set of files, not one file.
+ *
+ * The deploy path already uploads more than one module — the SDK travels beside
+ * every script as `ganju-sdk.js` — so the runtime has never been the thing
+ * holding this to a single file. What was single was the storage: one R2 object
+ * holding one module's text.
+ *
+ * So the object becomes an envelope. `sourceKind: 'editor'` bytes are JSON of
+ * this shape; `'bundle'` bytes stay exactly what the CLI compiled, because a
+ * bundle IS one module and wrapping it would mean the stored bytes are no longer
+ * the thing that runs. `decodeProject` tells the two apart by reading the marker,
+ * which is why there is a marker rather than a bare `{ files }` — arbitrary
+ * minified JS never parses as JSON, but a bundle that happens to be a JSON
+ * module would, and guessing is not a thing to do with someone's deploy.
+ */
+export const CUSTOM_CODE_PROJECT_MARKER = 'ganjuProject';
+
+export interface CustomCodeProject {
+  [CUSTOM_CODE_PROJECT_MARKER]: 1;
+  files: Record<string, string>;
+}
+
+export const encodeProject = (files: Record<string, string>): string =>
+  JSON.stringify({ [CUSTOM_CODE_PROJECT_MARKER]: 1, files });
+
+/**
+ * Read stored bytes back as a set of files, or `null` when they are a single
+ * module — the CLI's bundle, and every editor version written before folders
+ * existed.
+ */
+export const decodeProject = (text: string): Record<string, string> | null => {
+  if (!text.trimStart().startsWith('{')) return null;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return null;
+  }
+  if (!parsed || typeof parsed !== 'object') return null;
+  const envelope = parsed as Partial<CustomCodeProject>;
+  if (envelope[CUSTOM_CODE_PROJECT_MARKER] !== 1) return null;
+  if (!envelope.files || typeof envelope.files !== 'object') return null;
+  const files: Record<string, string> = {};
+  for (const [path, content] of Object.entries(envelope.files)) {
+    if (typeof content === 'string') files[path] = content;
+  }
+  return files;
+};
+
+/**
+ * Why one path is not a legal file in a script — the rule that refused it, and
+ * what it needs to say so.
+ *
+ * A code rather than a sentence, because there are two readers. The upload path
+ * throws it as English, which is what `error_log` has always held; the
+ * dashboard's file explorer renders it in whatever language the person is
+ * reading, at the keystroke. Returning prose would have forced the explorer to
+ * either show English or keep a second copy of these rules — and a second copy
+ * is how the two surfaces come to disagree about what a legal name is.
+ */
+export type ProjectPathIssue =
+  | { code: 'required' }
+  | { code: 'tooLong'; path: string; max: number }
+  | { code: 'charset'; path: string }
+  | { code: 'dots'; path: string }
+  | { code: 'extension'; path: string }
+  | { code: 'reserved'; path: string }
+  | { code: 'taken'; path: string };
+
+/**
+ * Check ONE path against every rule a stored file has to satisfy.
+ *
+ * Split out of the set-level check below so the dashboard's file explorer can
+ * apply the identical rules as someone types a name, rather than letting them
+ * finish, click deploy, and read the same sentence back as a 400. There is one
+ * definition of what a legal path is, and both surfaces call it.
+ *
+ * `taken` is the paths already in the project, compared case-insensitively —
+ * pass the file being renamed's own path in it and the rename reports a
+ * collision with itself, so callers exclude it.
+ */
+export const projectPathIssue = (
+  path: string,
+  taken: Iterable<string> = []
+): ProjectPathIssue | null => {
+  if (!path) return { code: 'required' };
+  if (path.length > constants.CUSTOM_CODE_MAX_FILE_PATH) {
+    return {
+      code: 'tooLong',
+      path,
+      max: constants.CUSTOM_CODE_MAX_FILE_PATH
+    };
+  }
+  if (!/^[a-zA-Z0-9._-]+(\/[a-zA-Z0-9._-]+)*$/.test(path)) {
+    return { code: 'charset', path };
+  }
+  if (path.split('/').some(segment => segment === '.' || segment === '..')) {
+    return { code: 'dots', path };
+  }
+  if (!path.endsWith('.js')) {
+    return { code: 'extension', path };
+  }
+  if (path === constants.CUSTOM_CODE_SDK_MODULE) {
+    return { code: 'reserved', path };
+  }
+  // Case-insensitively, because two files differing only in case are one file
+  // on the machine of whoever downloads them next.
+  const lower = path.toLowerCase();
+  for (const other of taken) {
+    if (other.toLowerCase() === lower) {
+      return { code: 'taken', path };
+    }
+  }
+  return null;
+};
+
+/**
+ * The same check, rendered in English — what the upload path throws and what
+ * lands in `error_log`. Messages carry a word `matchStatus` recognises, so they
+ * land as 400s rather than opaque 500s.
+ */
+export const validateProjectPath = (
+  path: string,
+  taken: Iterable<string> = []
+): string | null => {
+  const issue = projectPathIssue(path, taken);
+  if (!issue) return null;
+
+  switch (issue.code) {
+    case 'required':
+      return 'A name is required';
+    case 'tooLong':
+      return `The path "${issue.path}" exceeds ${issue.max} characters`;
+    case 'charset':
+      return `Invalid file path "${issue.path}" — letters, digits, dot, dash, underscore and / only, with no leading or trailing slash`;
+    case 'dots':
+      return `Invalid file path "${issue.path}" — . and .. are not allowed`;
+    case 'extension':
+      return `Invalid file path "${issue.path}" — every file must end in .js, since it is deployed as a module exactly as written`;
+    case 'reserved':
+      return `Invalid file path "${issue.path}" — that name belongs to the SDK, which is attached to every deploy`;
+    case 'taken':
+      return `Invalid file path "${issue.path}" — it is already in use`;
+  }
+};
+
+/**
+ * Check a set of files before anything is stored or deployed.
+ *
+ * Paths are the module names the Workers upload API receives, and a script's
+ * imports resolve against them — so a path that escapes the root, collides with
+ * the SDK's own module, or arrives twice under different spellings is a deploy
+ * that fails in the runtime rather than here. Messages carry a word `matchStatus`
+ * recognises, so these land as 400s rather than opaque 500s.
+ */
+export const validateProjectFiles = (files: Record<string, string>): void => {
+  const paths = Object.keys(files);
+
+  if (paths.length === 0) {
+    throw new Error('At least one file is required');
+  }
+  if (paths.length > constants.CUSTOM_CODE_MAX_FILES) {
+    throw new Error(
+      `The project exceeds the ${constants.CUSTOM_CODE_MAX_FILES}-file limit`
+    );
+  }
+  if (!files[constants.CUSTOM_CODE_MAIN_MODULE]) {
+    throw new Error(
+      `A file named ${constants.CUSTOM_CODE_MAIN_MODULE} is required — it is the module the dispatcher calls`
+    );
+  }
+
+  const seen = new Set<string>();
+  for (const path of paths) {
+    const problem = validateProjectPath(path, seen);
+    if (problem) throw new Error(problem);
+    seen.add(path.toLowerCase());
+  }
+};

package/src/customCodeToken.ts

@@ -0,0 +1,198 @@
+import {
+  bytesToBase64,
+  toBase64Url,
+  fromBase64Url,
+  base64ToBytes
+} from './base64';
+import { constants } from './constants';
+
+// The credential a deployed user script uses to call the broker.
+//
+// A service binding carries no caller identity — the broker sees a request from
+// "some script in the namespace" and nothing more — so the script has to present
+// something that names it. That something is minted here at publish time,
+// injected as the script's `GANJU_TOOL_TOKEN` secret binding, and verified by the
+// broker on every call.
+//
+// It is a signed value rather than a random string checked against a table for
+// one reason: revocation is already free. The token names the version it was
+// minted for, and the broker refuses any token whose version is no longer the
+// artifact's active one — a check it performs against a row it has to read
+// anyway (the tool config, for the connection allow-list). Publishing a new
+// version therefore invalidates every older token without a second write, and
+// there is no token table to keep in step with the version table.
+export interface CustomCodeTokenPayload {
+  // Token format version, so a future change to the payload can be rejected
+  // rather than misread.
+  v: string;
+  artifactId: string;
+  // The artifact_tool_version this token was minted for. Checked against
+  // config.activeVersionId by the broker, which is what makes rotation real.
+  versionId: string;
+  // Issued-at, seconds. Not used as an expiry — the version check is the
+  // lifetime — but it makes two tokens for the same version distinguishable in
+  // logs, and gives an operator something to reason about.
+  iat: number;
+  // Minted for a test run rather than for a published version.
+  //
+  // The distinction matters to the broker and nowhere else: a preview token
+  // names a version that is deliberately NOT the active one, so the check that
+  // gives a live token its lifetime cannot apply to it. In exchange it carries
+  // an expiry, which a live token does not need.
+  preview?: boolean;
+  // Expiry, seconds. Present on preview tokens, absent on live ones. Checked
+  // whenever present, so adding it to live tokens later needs no broker change.
+  exp?: number;
+}
+
+const encoder = new TextEncoder();
+
+const importKey = (secret: string): Promise<CryptoKey> =>
+  crypto.subtle.importKey(
+    'raw',
+    encoder.encode(secret),
+    { name: 'HMAC', hash: 'SHA-256' },
+    false,
+    ['sign', 'verify']
+  );
+
+const signPayload = async (
+  encodedPayload: string,
+  secret: string
+): Promise<string> => {
+  const key = await importKey(secret);
+  const signature = await crypto.subtle.sign(
+    'HMAC',
+    key,
+    encoder.encode(encodedPayload)
+  );
+  return toBase64Url(bytesToBase64(new Uint8Array(signature)));
+};
+
+/**
+ * Mint a tool token for one artifact + version. Called by the publish pipeline
+ * immediately before upload; the result is written as a `secret_text` binding so
+ * it never appears in the bundle the user can read back.
+ */
+export const mintCustomCodeToken = async (
+  payload: Omit<CustomCodeTokenPayload, 'v' | 'iat' | 'exp'> & {
+    ttlMs?: number;
+  },
+  secret: string,
+  issuedAt: number = Date.now()
+): Promise<string> => {
+  const body: CustomCodeTokenPayload = {
+    v: constants.CUSTOM_CODE_TOKEN_VERSION,
+    artifactId: payload.artifactId,
+    versionId: payload.versionId,
+    iat: Math.floor(issuedAt / 1000),
+    ...(payload.preview ? { preview: true } : {}),
+    // Rounded up, not down: `exp` is in seconds, and truncating would make a
+    // stated ten-minute lifetime end up to a second early. A ceiling means the
+    // TTL is a floor, which is what the caller asked for.
+    ...(payload.ttlMs
+      ? { exp: Math.ceil((issuedAt + payload.ttlMs) / 1000) }
+      : {})
+  };
+  const encodedPayload = toBase64Url(
+    bytesToBase64(encoder.encode(JSON.stringify(body)))
+  );
+  const signature = await signPayload(encodedPayload, secret);
+  return `${encodedPayload}.${signature}`;
+};
+
+/**
+ * Verify a tool token and return its payload, or null when it is malformed, of
+ * an unknown format version, or not signed by this deployment's secret.
+ *
+ * Returning null rather than throwing keeps the broker's failure path uniform:
+ * every rejection is the same 401, so a caller learns nothing about which check
+ * failed.
+ */
+export const verifyCustomCodeToken = async (
+  token: string,
+  secret: string,
+  now?: number
+): Promise<CustomCodeTokenPayload | null> => {
+  const separator = token.lastIndexOf('.');
+  if (separator <= 0) return null;
+
+  const encodedPayload = token.slice(0, separator);
+  const signature = token.slice(separator + 1);
+
+  const key = await importKey(secret);
+  let valid: boolean;
+  try {
+    // Copied into a fresh view so its backing store is a plain ArrayBuffer —
+    // base64ToBytes is typed against ArrayBufferLike, which subtle.verify's
+    // BufferSource parameter does not accept.
+    const signatureBytes = new Uint8Array(
+      base64ToBytes(fromBase64Url(signature))
+    );
+    valid = await crypto.subtle.verify(
+      'HMAC',
+      key,
+      signatureBytes,
+      encoder.encode(encodedPayload)
+    );
+  } catch {
+    return null;
+  }
+  if (!valid) return null;
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(
+      new TextDecoder().decode(base64ToBytes(fromBase64Url(encodedPayload)))
+    );
+  } catch {
+    return null;
+  }
+
+  if (!parsed || typeof parsed !== 'object') return null;
+  const candidate = parsed as Partial<CustomCodeTokenPayload>;
+  if (candidate.v !== constants.CUSTOM_CODE_TOKEN_VERSION) return null;
+  if (typeof candidate.artifactId !== 'string' || !candidate.artifactId) {
+    return null;
+  }
+  if (typeof candidate.versionId !== 'string' || !candidate.versionId) {
+    return null;
+  }
+
+  // Checked here rather than in the broker so every caller gets it: a token past
+  // its expiry is as good as unsigned.
+  if (
+    typeof candidate.exp === 'number' &&
+    candidate.exp * 1000 <= (now ?? Date.now())
+  ) {
+    return null;
+  }
+
+  return {
+    v: candidate.v,
+    artifactId: candidate.artifactId,
+    versionId: candidate.versionId,
+    iat: typeof candidate.iat === 'number' ? candidate.iat : 0,
+    ...(candidate.preview === true ? { preview: true } : {}),
+    ...(typeof candidate.exp === 'number' ? { exp: candidate.exp } : {})
+  };
+};
+
+/**
+ * The dispatch-namespace script name for an artifact: `artifact_<id>`.
+ *
+ * The id, never the slug — slugs are user-editable and a rename would orphan the
+ * deployed script while the database still pointed at a live version.
+ */
+export const customCodeScriptName = (artifactId: string): string =>
+  `${constants.CUSTOM_CODE_SCRIPT_NAME_PREFIX}${artifactId}`;
+
+/**
+ * The script name a test run deploys into: `artifact_<id>_preview`.
+ *
+ * A second script rather than a second version of the live one, because a test
+ * must not be able to disturb what MCP clients are being served — and the only
+ * way to be certain of that is for it to run under a name nothing dispatches to.
+ */
+export const customCodePreviewScriptName = (artifactId: string): string =>
+  `${customCodeScriptName(artifactId)}${constants.CUSTOM_CODE_PREVIEW_SCRIPT_SUFFIX}`;

package/src/deadline.ts

@@ -0,0 +1,43 @@
+// Bound how long we wait for a dispatched user script, without handing the
+// binding an AbortSignal.
+//
+// The obvious version — `fetch(url, { signal })` on a Fetcher from a dispatch
+// namespace — works when the binding is in the same process and fails when it is
+// not: `AbortSignal serialization is not enabled`. A local `wrangler dev` proxies
+// the dispatch namespace to the remote account, so every test run and every tool
+// call raised that instead of running, while the deployed Worker was fine. A
+// timeout that only works in production is worse than no timeout, because it
+// breaks the environment where people are actually writing the code.
+//
+// What is lost by racing instead of aborting: the isolate keeps running after we
+// stop waiting for it. That is what the per-script `limits.cpu_ms` ceiling is
+// for, and this deadline was never a cost control — it bounds how long a person
+// watches a spinner.
+export class DeadlineError extends Error {
+  constructor(ms: number) {
+    super(`Timed out after ${ms}ms`);
+    // Named the way an aborted fetch is, so a call site that already branches on
+    // the name keeps reading the same way.
+    this.name = 'TimeoutError';
+  }
+}
+
+export const withDeadline = async <T>(
+  work: Promise<T>,
+  ms: number
+): Promise<T> => {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      work,
+      new Promise<never>((_, reject) => {
+        timer = setTimeout(() => reject(new DeadlineError(ms)), ms);
+      })
+    ]);
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+};
+
+export const isDeadlineError = (error: unknown): boolean =>
+  error instanceof Error && error.name === 'TimeoutError';

package/src/discordSend.ts

@@ -0,0 +1,41 @@
+// Wire protocol shared between the worker (which constructs the request) and the
+// resource-handler container (which parses it and calls the Discord API). The
+// worker sends a multipart/form-data POST with one `metadata` field (JSON
+// matching DiscordSendRequest) and exactly one `file` field (binary). Discord
+// uploads a file by posting a `payload_json` + `files[0]` multipart to
+// POST /channels/{channelId}/messages.
+
+export interface DiscordSendRequest {
+  botToken: string;
+  // Discord channel id (a DM channel or a guild text channel) — Discord replies
+  // are posted into the same channel the message arrived in.
+  channelId: string;
+  // Optional message id to reply to (Discord message_reference).
+  replyToMessageId?: string;
+  // Text posted alongside the file (Discord has no separate "caption" — it's the
+  // message content, capped at 2000 chars by the caller).
+  content?: string;
+}
+
+export interface DiscordSendResponse {
+  ok: boolean;
+  id?: string;
+  error?: string;
+}
+
+// Wire protocol for sending a PROXIED (remote MCP) resource as a Discord file.
+// Mirrors TelegramSendRemoteResourceRequest / SlackSendRemoteResourceRequest:
+// the worker sends only the remote connection details + resolved auth header as
+// JSON, and the resource-handler container does the remote read + decode + upload
+// itself — so a large file's bytes never transit the 128 MiB worker.
+export interface DiscordSendRemoteResourceRequest {
+  discord: DiscordSendRequest;
+  remote: {
+    url: string;
+    transport: string;
+    // Single header injected on the remote MCP connection (e.g. Authorization).
+    authHeader?: { name: string; value: string } | null;
+    uri: string;
+    timeoutMs: number;
+  };
+}

package/src/embeddable.ts

@@ -0,0 +1,4 @@
+import { constants } from './constants';
+
+export const isEmbeddableMimeType = (mimeType: string): boolean =>
+  (constants.EMBEDDABLE_MIME_TYPES as readonly string[]).includes(mimeType);

package/src/exposedResource.ts

@@ -0,0 +1,40 @@
+import { constants } from './constants';
+
+// Which of an artifact's resources are actually addressable.
+//
+// Two source types produce rows that exist to hold structure rather than
+// content, and neither should ever be handed to a caller:
+//
+//  - a WEBSITE seed (the row with no parent) is the crawl's starting point. The
+//    page at that URL is indexed separately as its child, so the seed and the
+//    page carry the SAME uri while only the child has content.
+//  - a Drive/OneDrive folder is a pure reference whose content lives in its
+//    children.
+//
+// The seed case is why this has to be shared rather than reimplemented. Applied
+// in one place and not another, a lookup by uri can return the seed in one code
+// path and the page in another — the same uri on the same artifact answering
+// "146 characters of text" and "no inline content and no file in storage"
+// depending on which surface asked.
+export interface ExposableResource {
+  sourceType?: string | null;
+  parentResourceId?: string | null;
+}
+
+export const isExposedResource = (resource: ExposableResource): boolean => {
+  if (
+    resource.sourceType === constants.RESOURCE_SOURCE_TYPE_WEBSITE &&
+    !resource.parentResourceId
+  ) {
+    return false;
+  }
+  if (
+    resource.sourceType === constants.RESOURCE_SOURCE_TYPE_GOOGLE_DRIVE_FOLDER
+  ) {
+    return false;
+  }
+  if (resource.sourceType === constants.RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER) {
+    return false;
+  }
+  return true;
+};

package/src/extractedDocument.ts

@@ -0,0 +1,62 @@
+export interface ExtractedDocumentSource {
+  fileName?: string;
+  mimeType: string;
+  sizeBytes: number;
+}
+
+export interface ExtractedDocumentMetadata {
+  loc: {
+    pageNumber: number;
+    totalPages: number;
+    sheetName?: string;
+    slideTitle?: string;
+  };
+  source: ExtractedDocumentSource;
+  pdf?: {
+    info?: Record<string, unknown>;
+    pdfMetadata?: Record<string, unknown> | null;
+  };
+  docx?: {
+    coreProps?: Record<string, string>;
+    appProps?: Record<string, string>;
+  };
+  xlsx?: {
+    sheetIndex: number;
+    range?: string;
+    rowCount?: number;
+    columnCount?: number;
+    workbookProps?: Record<string, unknown>;
+  };
+  pptx?: {
+    coreProps?: Record<string, string>;
+    appProps?: Record<string, string>;
+    notes?: string;
+  };
+  web?: {
+    url: string;
+    canonicalUrl?: string;
+    title?: string;
+    description?: string;
+    siteName?: string;
+    author?: string;
+    keywords?: string[];
+    language?: string;
+    publishedAt?: string;
+    modifiedAt?: string;
+    openGraph?: Record<string, string>;
+    twitter?: Record<string, string>;
+    jsonLd?: unknown[];
+    headings?: { tag: string; text: string }[];
+    images?: { src: string; alt?: string }[];
+    links?: { url: string; text?: string }[];
+    favicon?: string;
+    httpStatus?: number;
+    contentType?: string;
+    renderer: 'cheerio' | 'playwright';
+  };
+}
+
+export interface ExtractedDocument {
+  pageContent: string;
+  metadata: ExtractedDocumentMetadata;
+}