@aiscene/shared 8.0.3 → 8.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/es/agent-tools/agent-behavior-init-args.mjs +44 -0
  2. package/dist/es/agent-tools/base-tools.mjs +163 -0
  3. package/dist/es/agent-tools/chrome-path.mjs +50 -0
  4. package/dist/es/agent-tools/cli-report-session.mjs +78 -0
  5. package/dist/es/agent-tools/error-formatter.mjs +106 -0
  6. package/dist/es/agent-tools/index.mjs +9 -0
  7. package/dist/es/agent-tools/init-arg-utils.mjs +38 -0
  8. package/dist/es/agent-tools/observation-artifact.mjs +5 -0
  9. package/dist/es/agent-tools/observation-record.mjs +248 -0
  10. package/dist/es/agent-tools/tool-defaults.mjs +54 -0
  11. package/dist/es/agent-tools/tool-generator.mjs +529 -0
  12. package/dist/es/agent-tools/types.mjs +3 -0
  13. package/dist/es/agent-tools/user-prompt.mjs +66 -0
  14. package/dist/es/cli/interrupt.mjs +116 -0
  15. package/dist/es/cli/record-command.mjs +130 -0
  16. package/dist/es/cli/screenshot-file.mjs +24 -0
  17. package/dist/es/cli/verbose-ai-act.mjs +230 -0
  18. package/dist/es/cli/verbose-screenshot.mjs +134 -0
  19. package/dist/es/cli/verbose.mjs +404 -0
  20. package/dist/es/env/parse-model-config.mjs +1 -1
  21. package/dist/es/env/types.mjs +18 -3
  22. package/dist/es/img/index.mjs +3 -3
  23. package/dist/es/img/info.mjs +45 -1
  24. package/dist/es/img/photon-loader.mjs +4 -0
  25. package/dist/es/img/transform.mjs +113 -3
  26. package/dist/es/recorder.mjs +249 -0
  27. package/dist/lib/agent-tools/agent-behavior-init-args.js +87 -0
  28. package/dist/lib/agent-tools/base-tools.js +197 -0
  29. package/dist/lib/agent-tools/chrome-path.js +87 -0
  30. package/dist/lib/agent-tools/cli-report-session.js +121 -0
  31. package/dist/lib/agent-tools/error-formatter.js +149 -0
  32. package/dist/lib/agent-tools/index.js +114 -0
  33. package/dist/lib/agent-tools/init-arg-utils.js +78 -0
  34. package/dist/lib/agent-tools/observation-artifact.js +42 -0
  35. package/dist/lib/agent-tools/observation-record.js +297 -0
  36. package/dist/lib/agent-tools/tool-defaults.js +97 -0
  37. package/dist/lib/agent-tools/tool-generator.js +569 -0
  38. package/dist/lib/agent-tools/types.js +40 -0
  39. package/dist/lib/agent-tools/user-prompt.js +103 -0
  40. package/dist/lib/cli/interrupt.js +156 -0
  41. package/dist/lib/cli/record-command.js +164 -0
  42. package/dist/lib/cli/screenshot-file.js +58 -0
  43. package/dist/lib/cli/verbose-ai-act.js +273 -0
  44. package/dist/lib/cli/verbose-screenshot.js +177 -0
  45. package/dist/lib/cli/verbose.js +465 -0
  46. package/dist/lib/env/parse-model-config.js +1 -1
  47. package/dist/lib/env/types.js +32 -2
  48. package/dist/lib/img/index.js +29 -5
  49. package/dist/lib/img/info.js +48 -1
  50. package/dist/lib/img/photon-loader.js +38 -0
  51. package/dist/lib/img/transform.js +135 -4
  52. package/dist/lib/recorder.js +307 -0
  53. package/dist/types/agent-tools/agent-behavior-init-args.d.ts +17 -0
  54. package/dist/types/agent-tools/base-tools.d.ts +158 -0
  55. package/dist/types/agent-tools/chrome-path.d.ts +2 -0
  56. package/dist/types/agent-tools/cli-report-session.d.ts +12 -0
  57. package/dist/types/agent-tools/error-formatter.d.ts +30 -0
  58. package/dist/types/agent-tools/index.d.ts +9 -0
  59. package/dist/types/agent-tools/init-arg-utils.d.ts +13 -0
  60. package/dist/types/agent-tools/observation-artifact.d.ts +10 -0
  61. package/dist/types/agent-tools/observation-record.d.ts +38 -0
  62. package/dist/types/agent-tools/tool-defaults.d.ts +63 -0
  63. package/dist/types/agent-tools/tool-generator.d.ts +13 -0
  64. package/dist/types/agent-tools/types.d.ts +213 -0
  65. package/dist/types/agent-tools/user-prompt.d.ts +13 -0
  66. package/dist/types/cli/interrupt.d.ts +49 -0
  67. package/dist/types/cli/record-command.d.ts +3 -0
  68. package/dist/types/cli/screenshot-file.d.ts +10 -0
  69. package/dist/types/cli/verbose-ai-act.d.ts +44 -0
  70. package/dist/types/cli/verbose-screenshot.d.ts +10 -0
  71. package/dist/types/cli/verbose.d.ts +40 -0
  72. package/dist/types/env/types.d.ts +16 -7
  73. package/dist/types/img/index.d.ts +2 -2
  74. package/dist/types/img/info.d.ts +2 -0
  75. package/dist/types/img/photon-loader.d.ts +2 -0
  76. package/dist/types/img/transform.d.ts +22 -2
  77. package/dist/types/mcp/types.d.ts +1 -0
  78. package/dist/types/recorder.d.ts +113 -0
  79. package/package.json +1 -1
  80. package/src/agent-tools/agent-behavior-init-args.ts +109 -0
  81. package/src/agent-tools/base-tools.ts +399 -0
  82. package/src/agent-tools/chrome-path.ts +74 -0
  83. package/src/agent-tools/cli-report-session.ts +130 -0
  84. package/src/agent-tools/error-formatter.ts +177 -0
  85. package/src/agent-tools/index.ts +9 -0
  86. package/src/agent-tools/init-arg-utils.ts +105 -0
  87. package/src/agent-tools/observation-artifact.ts +29 -0
  88. package/src/agent-tools/observation-record.ts +331 -0
  89. package/src/agent-tools/tool-defaults.ts +119 -0
  90. package/src/agent-tools/tool-generator.ts +866 -0
  91. package/src/agent-tools/types.ts +250 -0
  92. package/src/agent-tools/user-prompt.ts +102 -0
  93. package/src/cli/interrupt.ts +207 -0
  94. package/src/cli/record-command.ts +177 -0
  95. package/src/cli/screenshot-file.ts +61 -0
  96. package/src/cli/verbose-ai-act.ts +387 -0
  97. package/src/cli/verbose-screenshot.ts +269 -0
  98. package/src/cli/verbose.ts +753 -0
  99. package/src/env/types.ts +30 -2
  100. package/src/img/index.ts +12 -0
  101. package/src/img/info.ts +61 -0
  102. package/src/img/photon-loader.ts +5 -0
  103. package/src/img/transform.ts +262 -3
  104. package/src/mcp/types.ts +2 -0
  105. package/src/recorder.ts +625 -0
@@ -0,0 +1,130 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { getMidsceneRunBaseDir, getMidsceneRunSubDir } from '../common';
4
+
5
+ export interface CliReportSession {
6
+ version: 1;
7
+ sessionName: string;
8
+ targetIdentity?: string;
9
+ reportFileName: string;
10
+ reportPath: string;
11
+ createdAt: number;
12
+ }
13
+
14
+ const sessionDirName = 'cli-report-session';
15
+
16
+ function sanitizeSessionName(sessionName: string): string {
17
+ return sessionName.replace(/[^a-zA-Z0-9._-]/g, '_') || 'default';
18
+ }
19
+
20
+ function sanitizeFileSegment(segment: string): string {
21
+ const sanitized = segment.replace(/[^a-zA-Z0-9._-]/g, '_') || 'unknown';
22
+ return sanitized.slice(0, 80);
23
+ }
24
+
25
+ function ensureHtmlFileName(reportFileName: string): string {
26
+ return reportFileName.endsWith('.html')
27
+ ? reportFileName
28
+ : `${reportFileName}.html`;
29
+ }
30
+
31
+ function formatDateForFileName(date: Date): string {
32
+ const pad = (value: number) => String(value).padStart(2, '0');
33
+ const day = [
34
+ date.getFullYear(),
35
+ pad(date.getMonth() + 1),
36
+ pad(date.getDate()),
37
+ ].join('-');
38
+ const time = [
39
+ pad(date.getHours()),
40
+ pad(date.getMinutes()),
41
+ pad(date.getSeconds()),
42
+ ].join('-');
43
+ return `${day}_${time}`;
44
+ }
45
+
46
+ function randomId(): string {
47
+ return Math.random().toString(36).slice(2, 10);
48
+ }
49
+
50
+ function getCliReportSessionDir(): string {
51
+ const dir = join(getMidsceneRunBaseDir(), sessionDirName);
52
+ if (!existsSync(dir)) {
53
+ mkdirSync(dir, { recursive: true });
54
+ }
55
+ return dir;
56
+ }
57
+
58
+ function getCliReportSessionPath(sessionName: string): string {
59
+ return join(
60
+ getCliReportSessionDir(),
61
+ `${sanitizeSessionName(sessionName)}.json`,
62
+ );
63
+ }
64
+
65
+ export function generateCliReportSession(
66
+ sessionName: string,
67
+ targetIdentity?: string,
68
+ ): CliReportSession {
69
+ const identitySegment = targetIdentity
70
+ ? `-${sanitizeFileSegment(targetIdentity)}`
71
+ : '';
72
+ const reportFileName = `${sanitizeSessionName(sessionName)}${identitySegment}-${formatDateForFileName(new Date())}-${randomId()}`;
73
+ const reportPath = join(
74
+ getMidsceneRunSubDir('report'),
75
+ ensureHtmlFileName(reportFileName),
76
+ );
77
+ const session: CliReportSession = {
78
+ version: 1,
79
+ sessionName,
80
+ ...(targetIdentity ? { targetIdentity } : {}),
81
+ reportFileName,
82
+ reportPath,
83
+ createdAt: Date.now(),
84
+ };
85
+ return session;
86
+ }
87
+
88
+ export function writeCliReportSession(session: CliReportSession): void {
89
+ writeFileSync(
90
+ getCliReportSessionPath(session.sessionName),
91
+ JSON.stringify(session, null, 2),
92
+ 'utf-8',
93
+ );
94
+ }
95
+
96
+ export function createCliReportSession(
97
+ sessionName: string,
98
+ targetIdentity?: string,
99
+ ): CliReportSession {
100
+ const session = generateCliReportSession(sessionName, targetIdentity);
101
+ writeCliReportSession(session);
102
+ return session;
103
+ }
104
+
105
+ export function readCliReportSession(
106
+ sessionName: string,
107
+ ): CliReportSession | undefined {
108
+ const sessionPath = getCliReportSessionPath(sessionName);
109
+ if (!existsSync(sessionPath)) {
110
+ return undefined;
111
+ }
112
+
113
+ try {
114
+ const raw = readFileSync(sessionPath, 'utf-8');
115
+ const parsed = JSON.parse(raw) as Partial<CliReportSession>;
116
+ if (
117
+ parsed.version !== 1 ||
118
+ parsed.sessionName !== sessionName ||
119
+ typeof parsed.reportFileName !== 'string' ||
120
+ !parsed.reportFileName.trim() ||
121
+ /[\\/]/.test(parsed.reportFileName)
122
+ ) {
123
+ return undefined;
124
+ }
125
+
126
+ return parsed as CliReportSession;
127
+ } catch {
128
+ return undefined;
129
+ }
130
+ }
@@ -0,0 +1,177 @@
1
+ /** A compact, transport-safe representation of an unknown thrown value. */
2
+ export interface SerializedError {
3
+ name: string;
4
+ message: string;
5
+ stack?: string;
6
+ code?: string | number;
7
+ status?: string | number;
8
+ requestId?: string | number;
9
+ }
10
+
11
+ const maxSerializedStringLength = 4_096;
12
+ const truncatedStringSuffix = '… [truncated]';
13
+
14
+ interface SerializedErrorResult {
15
+ error: SerializedError;
16
+ hasMessage: boolean;
17
+ }
18
+
19
+ function isObject(error: unknown): error is object {
20
+ return (
21
+ (typeof error === 'object' && error !== null) || typeof error === 'function'
22
+ );
23
+ }
24
+
25
+ function safelyReadProperty(error: object, key: PropertyKey): unknown {
26
+ try {
27
+ return Reflect.get(error, key);
28
+ } catch {
29
+ return undefined;
30
+ }
31
+ }
32
+
33
+ function readBoundedDiagnostic(
34
+ error: object,
35
+ keys: readonly PropertyKey[],
36
+ ): string | number | undefined {
37
+ for (const key of keys) {
38
+ const value = safelyReadProperty(error, key);
39
+ if (typeof value === 'number') {
40
+ return value;
41
+ }
42
+ if (typeof value === 'string') {
43
+ return truncateSerializedErrorString(value);
44
+ }
45
+ }
46
+ return undefined;
47
+ }
48
+
49
+ /** Apply the same string bound used by {@link serializeError}. */
50
+ export function truncateSerializedErrorString(value: string): string {
51
+ if (value.length <= maxSerializedStringLength) {
52
+ return value;
53
+ }
54
+
55
+ return `${value.slice(
56
+ 0,
57
+ maxSerializedStringLength - truncatedStringSuffix.length,
58
+ )}${truncatedStringSuffix}`;
59
+ }
60
+
61
+ function readNonEmptyMessage(error: object): string | undefined {
62
+ const message = safelyReadProperty(error, 'message');
63
+ return typeof message === 'string' && message ? message : undefined;
64
+ }
65
+
66
+ function extractStringMessage(error: object): string | undefined {
67
+ const directMessage = readNonEmptyMessage(error);
68
+ if (directMessage) {
69
+ return directMessage;
70
+ }
71
+
72
+ for (const nestedKey of ['error', 'cause'] as const) {
73
+ const nestedError = safelyReadProperty(error, nestedKey);
74
+ if (!isObject(nestedError)) {
75
+ continue;
76
+ }
77
+
78
+ const nestedMessage = readNonEmptyMessage(nestedError);
79
+ if (nestedMessage) {
80
+ return nestedMessage;
81
+ }
82
+ }
83
+
84
+ return undefined;
85
+ }
86
+
87
+ /**
88
+ * Extract a human-readable message from an unknown thrown value.
89
+ *
90
+ * Many SDK/transport layers reject with structured objects (e.g.
91
+ * `{ code, message }`, `{ error: { message } }`, `{ cause: { message } }`)
92
+ * rather than `Error` instances. This helper returns the bounded message from
93
+ * {@link serializeError}; message-less objects are summarized from the same
94
+ * diagnostic whitelist instead of serializing arbitrary payload fields.
95
+ */
96
+ export function getErrorMessage(error: unknown): string {
97
+ const result = serializeErrorValue(error);
98
+ const serialized = result.error;
99
+ if (!isObject(error) || result.hasMessage) {
100
+ return serialized.message;
101
+ }
102
+
103
+ const summary: SerializedError = {
104
+ name: serialized.name,
105
+ message: serialized.message,
106
+ };
107
+ for (const key of ['code', 'status', 'requestId'] as const) {
108
+ const value = serialized[key];
109
+ if (value !== undefined) {
110
+ summary[key] = value;
111
+ }
112
+ }
113
+
114
+ return truncateSerializedErrorString(JSON.stringify(summary));
115
+ }
116
+
117
+ /** Safely read a stack trace from an unknown thrown value when one exists. */
118
+ export function getErrorStack(error: unknown): string | undefined {
119
+ if (!isObject(error)) {
120
+ return undefined;
121
+ }
122
+
123
+ const stack = safelyReadProperty(error, 'stack');
124
+ return typeof stack === 'string' ? stack : undefined;
125
+ }
126
+
127
+ function serializeErrorValue(error: unknown): SerializedErrorResult {
128
+ if (!isObject(error)) {
129
+ const message = truncateSerializedErrorString(String(error));
130
+ return {
131
+ error: {
132
+ name: 'NonError',
133
+ message: message.trim() ? message : 'Empty string thrown',
134
+ },
135
+ hasMessage: Boolean(message.trim()),
136
+ };
137
+ }
138
+
139
+ const name = safelyReadProperty(error, 'name');
140
+ const serializedName = truncateSerializedErrorString(
141
+ typeof name === 'string' && name ? name : 'Error',
142
+ );
143
+ const message = extractStringMessage(error);
144
+ const serialized: SerializedError = {
145
+ name: serializedName,
146
+ message: truncateSerializedErrorString(
147
+ message ?? `${serializedName} without a message`,
148
+ ),
149
+ };
150
+
151
+ const stack = getErrorStack(error);
152
+ if (stack) {
153
+ serialized.stack = truncateSerializedErrorString(stack);
154
+ }
155
+
156
+ const code = readBoundedDiagnostic(error, ['code']);
157
+ const status = readBoundedDiagnostic(error, ['status', 'statusCode']);
158
+ const requestId = readBoundedDiagnostic(error, ['requestId', 'requestID']);
159
+ if (code !== undefined) serialized.code = code;
160
+ if (status !== undefined) serialized.status = status;
161
+ if (requestId !== undefined) serialized.requestId = requestId;
162
+
163
+ return {
164
+ error: serialized,
165
+ hasMessage: message !== undefined,
166
+ };
167
+ }
168
+
169
+ /**
170
+ * Extract a small, bounded diagnostic object suitable for JSON and test-runner
171
+ * transports. Only message, stack, and a few common diagnostic fields are
172
+ * retained. Arbitrary payloads are never visited, and nested errors are only
173
+ * inspected one level deep when the outer value has no message of its own.
174
+ */
175
+ export function serializeError(error: unknown): SerializedError {
176
+ return serializeErrorValue(error).error;
177
+ }
@@ -0,0 +1,9 @@
1
+ export * from './base-tools';
2
+ export * from './tool-defaults';
3
+ export * from './agent-behavior-init-args';
4
+ export * from './init-arg-utils';
5
+ export * from './error-formatter';
6
+ export * from './tool-generator';
7
+ export * from './types';
8
+ export * from './chrome-path';
9
+ export * from './observation-record';
@@ -0,0 +1,105 @@
1
+ import type { z } from 'zod';
2
+ import { getKeyAliases, isRecord } from '../key-alias-utils';
3
+ import type { ToolSchema } from './types';
4
+
5
+ function readAliasedValue(
6
+ args: Record<string, unknown>,
7
+ key: string,
8
+ ): unknown | undefined {
9
+ for (const alias of getKeyAliases(key)) {
10
+ if (alias in args) {
11
+ return args[alias];
12
+ }
13
+ }
14
+
15
+ return undefined;
16
+ }
17
+
18
+ function readNamespacedArg(
19
+ args: Record<string, unknown>,
20
+ namespace: string,
21
+ key: string,
22
+ ): unknown | undefined {
23
+ // Lookup order: namespace object first, then flat dotted form, then bare key
24
+ // fallback. Namespace-aware inputs win so multi-platform callers cannot be
25
+ // cross-contaminated by a top-level bare `deviceId` leaking into the wrong
26
+ // platform.
27
+ const namespacedArgs = readAliasedValue(args, namespace);
28
+ if (isRecord(namespacedArgs)) {
29
+ const nestedValue = readAliasedValue(namespacedArgs, key);
30
+ if (nestedValue !== undefined) {
31
+ return nestedValue;
32
+ }
33
+ }
34
+
35
+ const dottedValue = readAliasedValue(args, `${namespace}.${key}`);
36
+ if (dottedValue !== undefined) {
37
+ return dottedValue;
38
+ }
39
+
40
+ const directValue = readAliasedValue(args, key);
41
+ if (directValue !== undefined) {
42
+ return directValue;
43
+ }
44
+
45
+ return undefined;
46
+ }
47
+
48
+ export function extractNamespacedArgs<
49
+ TFieldName extends string,
50
+ TArgs extends Record<string, unknown> = Record<string, unknown>,
51
+ >(
52
+ args: Record<string, unknown>,
53
+ namespace: string,
54
+ keys: readonly TFieldName[],
55
+ ): TArgs | undefined {
56
+ const extracted: Record<string, unknown> = {};
57
+
58
+ for (const key of keys) {
59
+ const value = readNamespacedArg(args, namespace, key);
60
+ if (value !== undefined) {
61
+ extracted[key] = value;
62
+ }
63
+ }
64
+
65
+ return Object.keys(extracted).length > 0 ? (extracted as TArgs) : undefined;
66
+ }
67
+
68
+ export function sanitizeNamespacedArgs(
69
+ args: Record<string, unknown>,
70
+ namespace: string,
71
+ keys: readonly string[],
72
+ ): Record<string, unknown> {
73
+ const excludedKeys = new Set<string>(getKeyAliases(namespace));
74
+
75
+ for (const key of keys) {
76
+ for (const alias of getKeyAliases(key)) {
77
+ excludedKeys.add(alias);
78
+ }
79
+
80
+ for (const alias of getKeyAliases(`${namespace}.${key}`)) {
81
+ excludedKeys.add(alias);
82
+ }
83
+ }
84
+
85
+ return Object.fromEntries(
86
+ Object.entries(args).filter(([key]) => !excludedKeys.has(key)),
87
+ );
88
+ }
89
+
90
+ /**
91
+ * Build a flat tool schema whose keys are dotted `"<namespace>.<field>"`.
92
+ *
93
+ * We intentionally stay flat (rather than `{ namespace: z.object({...}) }`) so
94
+ * that CLI (`--android.device-id`) and `--help` output share the same spelling.
95
+ * `readNamespacedArg` understands all three input shapes:
96
+ * nested namespace object, dotted flat key, and bare key fallback.
97
+ */
98
+ export function createNamespacedInitArgSchema(
99
+ namespace: string,
100
+ shape: Record<string, z.ZodTypeAny>,
101
+ ): ToolSchema {
102
+ return Object.fromEntries(
103
+ Object.entries(shape).map(([key, value]) => [`${namespace}.${key}`, value]),
104
+ );
105
+ }
@@ -0,0 +1,29 @@
1
+ import type {
2
+ BaseAgent,
3
+ BaseUIObservation,
4
+ UIObservationRecord,
5
+ } from './types';
6
+
7
+ /** CLI-only bridge between an Agent runtime and observation artifacts. */
8
+ export interface ObservationArtifactAdapter {
9
+ exportRecord(observation: BaseUIObservation): Promise<UIObservationRecord>;
10
+ loadRecord(record: UIObservationRecord): BaseUIObservation;
11
+ }
12
+
13
+ /** @internal Property key used by Core and the CLI artifact commands. */
14
+ export const observationArtifactAdapterSymbol = Symbol(
15
+ 'midscene.observationArtifactAdapter',
16
+ );
17
+
18
+ type ObservationArtifactAdapterHost = {
19
+ [observationArtifactAdapterSymbol]?: ObservationArtifactAdapter;
20
+ };
21
+
22
+ /** Read the CLI artifact capability attached by the Core Agent. */
23
+ export function resolveObservationArtifactAdapter(
24
+ agent: BaseAgent,
25
+ ): ObservationArtifactAdapter | undefined {
26
+ return (agent as ObservationArtifactAdapterHost)[
27
+ observationArtifactAdapterSymbol
28
+ ];
29
+ }