@aiscene/shared 8.0.2 → 8.0.4

package/src/agent-tools/error-formatter.ts

@@ -0,0 +1,177 @@
+/** A compact, transport-safe representation of an unknown thrown value. */
+export interface SerializedError {
+  name: string;
+  message: string;
+  stack?: string;
+  code?: string | number;
+  status?: string | number;
+  requestId?: string | number;
+}
+
+const maxSerializedStringLength = 4_096;
+const truncatedStringSuffix = '… [truncated]';
+
+interface SerializedErrorResult {
+  error: SerializedError;
+  hasMessage: boolean;
+}
+
+function isObject(error: unknown): error is object {
+  return (
+    (typeof error === 'object' && error !== null) || typeof error === 'function'
+  );
+}
+
+function safelyReadProperty(error: object, key: PropertyKey): unknown {
+  try {
+    return Reflect.get(error, key);
+  } catch {
+    return undefined;
+  }
+}
+
+function readBoundedDiagnostic(
+  error: object,
+  keys: readonly PropertyKey[],
+): string | number | undefined {
+  for (const key of keys) {
+    const value = safelyReadProperty(error, key);
+    if (typeof value === 'number') {
+      return value;
+    }
+    if (typeof value === 'string') {
+      return truncateSerializedErrorString(value);
+    }
+  }
+  return undefined;
+}
+
+/** Apply the same string bound used by {@link serializeError}. */
+export function truncateSerializedErrorString(value: string): string {
+  if (value.length <= maxSerializedStringLength) {
+    return value;
+  }
+
+  return `${value.slice(
+    0,
+    maxSerializedStringLength - truncatedStringSuffix.length,
+  )}${truncatedStringSuffix}`;
+}
+
+function readNonEmptyMessage(error: object): string | undefined {
+  const message = safelyReadProperty(error, 'message');
+  return typeof message === 'string' && message ? message : undefined;
+}
+
+function extractStringMessage(error: object): string | undefined {
+  const directMessage = readNonEmptyMessage(error);
+  if (directMessage) {
+    return directMessage;
+  }
+
+  for (const nestedKey of ['error', 'cause'] as const) {
+    const nestedError = safelyReadProperty(error, nestedKey);
+    if (!isObject(nestedError)) {
+      continue;
+    }
+
+    const nestedMessage = readNonEmptyMessage(nestedError);
+    if (nestedMessage) {
+      return nestedMessage;
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Extract a human-readable message from an unknown thrown value.
+ *
+ * Many SDK/transport layers reject with structured objects (e.g.
+ * `{ code, message }`, `{ error: { message } }`, `{ cause: { message } }`)
+ * rather than `Error` instances. This helper returns the bounded message from
+ * {@link serializeError}; message-less objects are summarized from the same
+ * diagnostic whitelist instead of serializing arbitrary payload fields.
+ */
+export function getErrorMessage(error: unknown): string {
+  const result = serializeErrorValue(error);
+  const serialized = result.error;
+  if (!isObject(error) || result.hasMessage) {
+    return serialized.message;
+  }
+
+  const summary: SerializedError = {
+    name: serialized.name,
+    message: serialized.message,
+  };
+  for (const key of ['code', 'status', 'requestId'] as const) {
+    const value = serialized[key];
+    if (value !== undefined) {
+      summary[key] = value;
+    }
+  }
+
+  return truncateSerializedErrorString(JSON.stringify(summary));
+}
+
+/** Safely read a stack trace from an unknown thrown value when one exists. */
+export function getErrorStack(error: unknown): string | undefined {
+  if (!isObject(error)) {
+    return undefined;
+  }
+
+  const stack = safelyReadProperty(error, 'stack');
+  return typeof stack === 'string' ? stack : undefined;
+}
+
+function serializeErrorValue(error: unknown): SerializedErrorResult {
+  if (!isObject(error)) {
+    const message = truncateSerializedErrorString(String(error));
+    return {
+      error: {
+        name: 'NonError',
+        message: message.trim() ? message : 'Empty string thrown',
+      },
+      hasMessage: Boolean(message.trim()),
+    };
+  }
+
+  const name = safelyReadProperty(error, 'name');
+  const serializedName = truncateSerializedErrorString(
+    typeof name === 'string' && name ? name : 'Error',
+  );
+  const message = extractStringMessage(error);
+  const serialized: SerializedError = {
+    name: serializedName,
+    message: truncateSerializedErrorString(
+      message ?? `${serializedName} without a message`,
+    ),
+  };
+
+  const stack = getErrorStack(error);
+  if (stack) {
+    serialized.stack = truncateSerializedErrorString(stack);
+  }
+
+  const code = readBoundedDiagnostic(error, ['code']);
+  const status = readBoundedDiagnostic(error, ['status', 'statusCode']);
+  const requestId = readBoundedDiagnostic(error, ['requestId', 'requestID']);
+  if (code !== undefined) serialized.code = code;
+  if (status !== undefined) serialized.status = status;
+  if (requestId !== undefined) serialized.requestId = requestId;
+
+  return {
+    error: serialized,
+    hasMessage: message !== undefined,
+  };
+}
+
+/**
+ * Extract a small, bounded diagnostic object suitable for JSON and test-runner
+ * transports. Only message, stack, and a few common diagnostic fields are
+ * retained. Arbitrary payloads are never visited, and nested errors are only
+ * inspected one level deep when the outer value has no message of its own.
+ */
+export function serializeError(error: unknown): SerializedError {
+  return serializeErrorValue(error).error;
+}

package/src/agent-tools/index.ts

@@ -0,0 +1,9 @@
+export * from './base-tools';
+export * from './tool-defaults';
+export * from './agent-behavior-init-args';
+export * from './init-arg-utils';
+export * from './error-formatter';
+export * from './tool-generator';
+export * from './types';
+export * from './chrome-path';
+export * from './observation-record';

package/src/agent-tools/init-arg-utils.ts

@@ -0,0 +1,105 @@
+import type { z } from 'zod';
+import { getKeyAliases, isRecord } from '../key-alias-utils';
+import type { ToolSchema } from './types';
+
+function readAliasedValue(
+  args: Record<string, unknown>,
+  key: string,
+): unknown | undefined {
+  for (const alias of getKeyAliases(key)) {
+    if (alias in args) {
+      return args[alias];
+    }
+  }
+
+  return undefined;
+}
+
+function readNamespacedArg(
+  args: Record<string, unknown>,
+  namespace: string,
+  key: string,
+): unknown | undefined {
+  // Lookup order: namespace object first, then flat dotted form, then bare key
+  // fallback. Namespace-aware inputs win so multi-platform callers cannot be
+  // cross-contaminated by a top-level bare `deviceId` leaking into the wrong
+  // platform.
+  const namespacedArgs = readAliasedValue(args, namespace);
+  if (isRecord(namespacedArgs)) {
+    const nestedValue = readAliasedValue(namespacedArgs, key);
+    if (nestedValue !== undefined) {
+      return nestedValue;
+    }
+  }
+
+  const dottedValue = readAliasedValue(args, `${namespace}.${key}`);
+  if (dottedValue !== undefined) {
+    return dottedValue;
+  }
+
+  const directValue = readAliasedValue(args, key);
+  if (directValue !== undefined) {
+    return directValue;
+  }
+
+  return undefined;
+}
+
+export function extractNamespacedArgs<
+  TFieldName extends string,
+  TArgs extends Record<string, unknown> = Record<string, unknown>,
+>(
+  args: Record<string, unknown>,
+  namespace: string,
+  keys: readonly TFieldName[],
+): TArgs | undefined {
+  const extracted: Record<string, unknown> = {};
+
+  for (const key of keys) {
+    const value = readNamespacedArg(args, namespace, key);
+    if (value !== undefined) {
+      extracted[key] = value;
+    }
+  }
+
+  return Object.keys(extracted).length > 0 ? (extracted as TArgs) : undefined;
+}
+
+export function sanitizeNamespacedArgs(
+  args: Record<string, unknown>,
+  namespace: string,
+  keys: readonly string[],
+): Record<string, unknown> {
+  const excludedKeys = new Set<string>(getKeyAliases(namespace));
+
+  for (const key of keys) {
+    for (const alias of getKeyAliases(key)) {
+      excludedKeys.add(alias);
+    }
+
+    for (const alias of getKeyAliases(`${namespace}.${key}`)) {
+      excludedKeys.add(alias);
+    }
+  }
+
+  return Object.fromEntries(
+    Object.entries(args).filter(([key]) => !excludedKeys.has(key)),
+  );
+}
+
+/**
+ * Build a flat tool schema whose keys are dotted `"<namespace>.<field>"`.
+ *
+ * We intentionally stay flat (rather than `{ namespace: z.object({...}) }`) so
+ * that CLI (`--android.device-id`) and `--help` output share the same spelling.
+ * `readNamespacedArg` understands all three input shapes:
+ * nested namespace object, dotted flat key, and bare key fallback.
+ */
+export function createNamespacedInitArgSchema(
+  namespace: string,
+  shape: Record<string, z.ZodTypeAny>,
+): ToolSchema {
+  return Object.fromEntries(
+    Object.entries(shape).map(([key, value]) => [`${namespace}.${key}`, value]),
+  );
+}

package/src/agent-tools/observation-artifact.ts

@@ -0,0 +1,29 @@
+import type {
+  BaseAgent,
+  BaseUIObservation,
+  UIObservationRecord,
+} from './types';
+
+/** CLI-only bridge between an Agent runtime and observation artifacts. */
+export interface ObservationArtifactAdapter {
+  exportRecord(observation: BaseUIObservation): Promise<UIObservationRecord>;
+  loadRecord(record: UIObservationRecord): BaseUIObservation;
+}
+
+/** @internal Property key used by Core and the CLI artifact commands. */
+export const observationArtifactAdapterSymbol = Symbol(
+  'midscene.observationArtifactAdapter',
+);
+
+type ObservationArtifactAdapterHost = {
+  [observationArtifactAdapterSymbol]?: ObservationArtifactAdapter;
+};
+
+/** Read the CLI artifact capability attached by the Core Agent. */
+export function resolveObservationArtifactAdapter(
+  agent: BaseAgent,
+): ObservationArtifactAdapter | undefined {
+  return (agent as ObservationArtifactAdapterHost)[
+    observationArtifactAdapterSymbol
+  ];
+}

package/src/agent-tools/observation-record.ts

@@ -0,0 +1,331 @@
+import { createHash } from 'node:crypto';
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import {
+  basename,
+  dirname,
+  extname,
+  isAbsolute,
+  join,
+  relative,
+  resolve,
+} from 'node:path';
+import { z } from 'zod';
+import { getMidsceneRunSubDir } from '../common';
+import type { UIObservationFrame, UIObservationRecord } from './types';
+
+const observationRecordSchema = z
+  .object({
+    type: z.literal('midscene_ui_observation'),
+    version: z.literal(1),
+    startedAt: z.number().finite().nonnegative(),
+    endedAt: z.number().finite().nonnegative(),
+    frames: z
+      .array(
+        z
+          .object({
+            path: z.string().min(1),
+            mimeType: z.enum(['image/png', 'image/jpeg']),
+            capturedAt: z.number().finite().nonnegative(),
+          })
+          .strict(),
+      )
+      .min(1),
+    shotSize: z
+      .object({
+        width: z.number().finite().positive(),
+        height: z.number().finite().positive(),
+      })
+      .strict(),
+    shrunkShotToLogicalRatio: z.number().finite().positive(),
+  })
+  .strict()
+  .refine((record) => record.endedAt >= record.startedAt, {
+    path: ['endedAt'],
+    message: 'must be greater than or equal to startedAt',
+  });
+
+export function defaultObservationRecordPath(): string {
+  const suffix = Math.random().toString(36).slice(2, 10);
+  return resolve(
+    getMidsceneRunSubDir('output'),
+    `observation-${Date.now()}-${suffix}.json`,
+  );
+}
+
+export function parseUIObservationRecord(input: unknown): UIObservationRecord {
+  const result = observationRecordSchema.safeParse(input);
+  if (!result.success) {
+    const issue = result.error.issues[0];
+    const location = issue.path.length > 0 ? issue.path.join('.') : 'record';
+    throw new Error(
+      `Invalid UI observation record at ${location}: ${issue.message}`,
+    );
+  }
+  return result.data;
+}
+
+/** Return a detached record so callers cannot mutate runtime-owned state. */
+export function cloneUIObservationRecord(
+  record: UIObservationRecord,
+): UIObservationRecord {
+  return {
+    ...record,
+    frames: record.frames.map((frame) => ({ ...frame })),
+    shotSize: { ...record.shotSize },
+  };
+}
+
+export function readUIObservationRecord(filePath: string): UIObservationRecord {
+  const resolvedPath = resolve(filePath);
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(readFileSync(resolvedPath, 'utf8'));
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(
+      `Failed to read UI observation record ${resolvedPath}: ${message}`,
+    );
+  }
+  const record = parseUIObservationRecord(parsed);
+  const manifestDirectory = dirname(resolvedPath);
+  return {
+    ...record,
+    frames: record.frames.map((frame, index) => {
+      if (isAbsolute(frame.path)) {
+        throw new Error(
+          `Invalid UI observation record at frames.${index}.path: expected a relative image path`,
+        );
+      }
+      const imagePath = resolve(manifestDirectory, frame.path);
+      const relativeToManifest = relative(manifestDirectory, imagePath);
+      if (
+        relativeToManifest === '..' ||
+        relativeToManifest.startsWith(
+          `..${process.platform === 'win32' ? '\\' : '/'}`,
+        ) ||
+        isAbsolute(relativeToManifest)
+      ) {
+        throw new Error(
+          `Invalid UI observation record at frames.${index}.path: image path escapes the manifest directory`,
+        );
+      }
+      try {
+        if (!statSync(imagePath).isFile()) {
+          throw new Error('path is not a file');
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(
+          `Invalid UI observation record at frames.${index}.path: cannot read image ${imagePath}: ${message}`,
+        );
+      }
+      return { ...frame, path: imagePath };
+    }),
+  };
+}
+
+export interface UIObservationRecordMetadata {
+  startedAt: number;
+  endedAt: number;
+  shotSize: UIObservationRecord['shotSize'];
+  shrunkShotToLogicalRatio: number;
+}
+
+/**
+ * Incrementally persists observation frames and exports a runtime record whose
+ * paths resolve to those image files.
+ */
+export class UIObservationRecordWriter {
+  readonly outputPath: string;
+  readonly framesDirectory: string;
+  private readonly temporaryFramesDirectory: string;
+  private finalized = false;
+  private disposed = false;
+  private finalizedRecord: UIObservationRecord | null = null;
+
+  constructor(filePath?: string) {
+    this.outputPath = filePath
+      ? resolve(filePath)
+      : defaultObservationRecordPath();
+    const extension = extname(this.outputPath);
+    const stem = basename(this.outputPath, extension);
+    this.framesDirectory = join(dirname(this.outputPath), `${stem}.frames`);
+    this.temporaryFramesDirectory = `${this.framesDirectory}.tmp-${process.pid}-${Math.random()
+      .toString(36)
+      .slice(2, 10)}`;
+  }
+
+  persistFrame(dataUrl: string, capturedAt: number): UIObservationFrame {
+    if (this.disposed) {
+      throw new Error('UI observation record writer has been disposed');
+    }
+    if (this.finalized) {
+      throw new Error('UI observation record has already been finalized');
+    }
+    const match = /^data:image\/(png|jpe?g);base64,([\s\S]+)$/i.exec(dataUrl);
+    if (!match) {
+      throw new Error('UI observation frame must be a PNG or JPEG data URL');
+    }
+    const format = match[1].toLowerCase() === 'png' ? 'png' : 'jpeg';
+    const mimeType = format === 'png' ? 'image/png' : 'image/jpeg';
+    const bytes = Buffer.from(match[2], 'base64');
+    if (bytes.length === 0) {
+      throw new Error('UI observation frame contains no image data');
+    }
+    const digest = createHash('sha256').update(bytes).digest('hex');
+    const fileName = `${digest}.${format}`;
+    mkdirSync(this.temporaryFramesDirectory, { recursive: true });
+    const temporaryPath = join(this.temporaryFramesDirectory, fileName);
+    if (!existsSync(temporaryPath)) {
+      writeFileSync(temporaryPath, bytes);
+    }
+    return {
+      path: `${basename(this.framesDirectory)}/${fileName}`,
+      mimeType,
+      capturedAt,
+    };
+  }
+
+  /** Resolve a persisted frame before or after the writer is finalized. */
+  resolveFramePath(frame: UIObservationFrame): string {
+    if (this.disposed) {
+      throw new Error('UI observation record writer has been disposed');
+    }
+    const directory = this.finalized
+      ? this.framesDirectory
+      : this.temporaryFramesDirectory;
+    return join(directory, basename(frame.path));
+  }
+
+  /** Remove persisted images that are no longer referenced by the frame buffer. */
+  pruneFrames(frames: UIObservationFrame[]): void {
+    if (this.disposed) {
+      throw new Error('UI observation record writer has been disposed');
+    }
+    if (this.finalized || !existsSync(this.temporaryFramesDirectory)) return;
+
+    const retainedFiles = new Set(frames.map((frame) => basename(frame.path)));
+    for (const fileName of readdirSync(this.temporaryFramesDirectory)) {
+      if (!retainedFiles.has(fileName)) {
+        rmSync(join(this.temporaryFramesDirectory, fileName), { force: true });
+      }
+    }
+  }
+
+  finalize(
+    frames: UIObservationFrame[],
+    metadata: UIObservationRecordMetadata,
+  ): UIObservationRecord {
+    if (this.disposed) {
+      throw new Error('UI observation record writer has been disposed');
+    }
+    if (this.finalizedRecord) return this.finalizedRecord;
+    const record = parseUIObservationRecord({
+      type: 'midscene_ui_observation',
+      version: 1,
+      frames,
+      ...metadata,
+    });
+    this.pruneFrames(record.frames);
+    mkdirSync(dirname(this.outputPath), { recursive: true });
+    mkdirSync(this.temporaryFramesDirectory, { recursive: true });
+    rmSync(this.framesDirectory, { recursive: true, force: true });
+    renameSync(this.temporaryFramesDirectory, this.framesDirectory);
+    this.finalized = true;
+    this.finalizedRecord = {
+      ...record,
+      frames: record.frames.map((frame) => ({
+        ...frame,
+        path: resolve(dirname(this.outputPath), frame.path),
+      })),
+    };
+    return this.finalizedRecord;
+  }
+
+  /** Delete writer-owned temporary or finalized frame files. */
+  dispose(): void {
+    if (this.disposed) return;
+    rmSync(this.temporaryFramesDirectory, { recursive: true, force: true });
+    rmSync(this.framesDirectory, { recursive: true, force: true });
+    this.disposed = true;
+    this.finalizedRecord = null;
+  }
+}
+
+/**
+ * Persist a resolved observation record as a portable JSON manifest plus an
+ * adjacent image directory. The input record remains usable after writing.
+ */
+export function writeUIObservationRecord(
+  record: UIObservationRecord,
+  filePath?: string,
+): string {
+  const validated = parseUIObservationRecord(record);
+  const outputPath = filePath
+    ? resolve(filePath)
+    : defaultObservationRecordPath();
+  const extension = extname(outputPath);
+  const stem = basename(outputPath, extension);
+  const framesDirectory = join(dirname(outputPath), `${stem}.frames`);
+  const temporaryFramesDirectory = `${framesDirectory}.tmp-${process.pid}-${Math.random()
+    .toString(36)
+    .slice(2, 10)}`;
+  const temporaryManifest = `${outputPath}.tmp-${process.pid}-${Math.random()
+    .toString(36)
+    .slice(2, 10)}`;
+  const copiedPaths = new Map<string, string>();
+
+  try {
+    mkdirSync(temporaryFramesDirectory, { recursive: true });
+    const frames = validated.frames.map((frame, index) => {
+      const sourcePath = resolve(frame.path);
+      try {
+        if (!statSync(sourcePath).isFile()) {
+          throw new Error('path is not a file');
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(
+          `Invalid UI observation record at frames.${index}.path: cannot read image ${sourcePath}: ${message}`,
+        );
+      }
+
+      let relativePath = copiedPaths.get(sourcePath);
+      if (!relativePath) {
+        const imageExtension = frame.mimeType === 'image/png' ? 'png' : 'jpeg';
+        const fileName = `${String(copiedPaths.size).padStart(4, '0')}.${imageExtension}`;
+        relativePath = `${basename(framesDirectory)}/${fileName}`;
+        copyFileSync(sourcePath, join(temporaryFramesDirectory, fileName));
+        copiedPaths.set(sourcePath, relativePath);
+      }
+      return { ...frame, path: relativePath };
+    });
+    const serializedRecord = parseUIObservationRecord({
+      ...validated,
+      frames,
+    });
+    mkdirSync(dirname(outputPath), { recursive: true });
+    writeFileSync(
+      temporaryManifest,
+      JSON.stringify(serializedRecord, null, 2),
+      'utf8',
+    );
+    rmSync(framesDirectory, { recursive: true, force: true });
+    renameSync(temporaryFramesDirectory, framesDirectory);
+    renameSync(temporaryManifest, outputPath);
+    return outputPath;
+  } finally {
+    rmSync(temporaryFramesDirectory, { recursive: true, force: true });
+    rmSync(temporaryManifest, { force: true });
+  }
+}