@modelnex/sdk 0.5.45 → 0.5.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1119,6 +1119,159 @@ function serializeContexts(contexts) {
1119
1119
 
1120
1120
  // src/hooks/useModelNexSocket.ts
1121
1121
  init_dev_logging();
1122
+
1123
+ // src/utils/debug-tools.ts
1124
+ var lastActionEffect = null;
1125
+ function normalizeText(value) {
1126
+ return String(value || "").replace(/\s+/g, " ").trim();
1127
+ }
1128
+ function isElementVisible(el) {
1129
+ if (!(el instanceof HTMLElement)) return false;
1130
+ const style = window.getComputedStyle(el);
1131
+ if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") {
1132
+ return false;
1133
+ }
1134
+ const rect = el.getBoundingClientRect();
1135
+ return rect.width > 0 && rect.height > 0;
1136
+ }
1137
+ function safeQueryAll2(selector) {
1138
+ try {
1139
+ return Array.from(document.querySelectorAll(selector)).filter(
1140
+ (el) => el instanceof HTMLElement
1141
+ );
1142
+ } catch {
1143
+ return [];
1144
+ }
1145
+ }
1146
+ function classifyLevel(el, text) {
1147
+ const levelHint = `${el.getAttribute("data-variant") || ""} ${el.className || ""} ${text}`.toLowerCase();
1148
+ if (/\berror|danger|invalid|failed|required\b/.test(levelHint)) return "error";
1149
+ if (/\bwarn|warning|caution\b/.test(levelHint)) return "warning";
1150
+ return "info";
1151
+ }
1152
+ function classifySource(el) {
1153
+ const role = (el.getAttribute("role") || "").toLowerCase();
1154
+ if (role === "alert") return "alert";
1155
+ if (role === "status") return "status";
1156
+ if (el.hasAttribute("aria-live")) return "aria-live";
1157
+ const classHint = String(el.className || "").toLowerCase();
1158
+ if (/\bbanner\b/.test(classHint)) return "banner";
1159
+ return "inline";
1160
+ }
1161
+ function selectorHintFor(el) {
1162
+ if (el.id) return `#${el.id}`;
1163
+ if (el.getAttribute("data-testid")) return `[data-testid="${el.getAttribute("data-testid")}"]`;
1164
+ const role = el.getAttribute("role");
1165
+ return role ? `${el.tagName.toLowerCase()}[role="${role}"]` : el.tagName.toLowerCase();
1166
+ }
1167
+ function collectUiMessages(options) {
1168
+ const includeHidden = Boolean(options?.includeHidden);
1169
+ const maxItems = Math.max(1, Math.min(Number(options?.maxItems || 10), 20));
1170
+ const selectors = [
1171
+ '[role="alert"]',
1172
+ '[role="status"]',
1173
+ "[aria-live]",
1174
+ '[aria-invalid="true"]',
1175
+ "[data-error]",
1176
+ "[data-warning]",
1177
+ ".error",
1178
+ ".warning",
1179
+ ".alert",
1180
+ ".toast",
1181
+ ".banner"
1182
+ ];
1183
+ const seen = /* @__PURE__ */ new Set();
1184
+ const results = [];
1185
+ for (const selector of selectors) {
1186
+ for (const el of safeQueryAll2(selector)) {
1187
+ const visible = isElementVisible(el);
1188
+ if (!includeHidden && !visible) continue;
1189
+ const text = normalizeText(el.textContent);
1190
+ if (!text || text.length < 2) continue;
1191
+ const key = `${selector}:${text.toLowerCase()}`;
1192
+ if (seen.has(key)) continue;
1193
+ seen.add(key);
1194
+ results.push({
1195
+ id: `ui_${results.length + 1}`,
1196
+ level: classifyLevel(el, text),
1197
+ text,
1198
+ source: classifySource(el),
1199
+ visible,
1200
+ selectorHint: selectorHintFor(el)
1201
+ });
1202
+ if (results.length >= maxItems) return results;
1203
+ }
1204
+ }
1205
+ return results;
1206
+ }
1207
+ function captureDebugSnapshot() {
1208
+ return {
1209
+ route: `${window.location.pathname}${window.location.search}${window.location.hash}`,
1210
+ title: document.title || "",
1211
+ uiMessages: collectUiMessages({ maxItems: 6 }),
1212
+ timestamp: Date.now()
1213
+ };
1214
+ }
1215
+ function diffUiMessages(before, after) {
1216
+ const beforeKeys = new Set(before.map((entry) => `${entry.level}:${entry.text}`));
1217
+ return after.filter((entry) => !beforeKeys.has(`${entry.level}:${entry.text}`));
1218
+ }
1219
+ function summarizeEffect(record) {
1220
+ const routeChanged = record.before.route !== record.after.route;
1221
+ const titleChanged = record.before.title !== record.after.title;
1222
+ const newMessages = diffUiMessages(record.before.uiMessages, record.after.uiMessages);
1223
+ const changed = routeChanged || titleChanged || newMessages.length > 0 || !record.success || record.before.timestamp !== record.after.timestamp;
1224
+ const parts = [];
1225
+ if (routeChanged) parts.push(`route changed from ${record.before.route} to ${record.after.route}`);
1226
+ if (titleChanged) parts.push("page title changed");
1227
+ if (newMessages.length > 0) parts.push(`new UI messages: ${newMessages.map((entry) => entry.text).join(" | ")}`);
1228
+ if (!record.success && record.error) parts.push(`action failed: ${record.error}`);
1229
+ if (parts.length === 0) parts.push("no obvious route/title/message change detected");
1230
+ return {
1231
+ summary: parts.join("; "),
1232
+ changed
1233
+ };
1234
+ }
1235
+ function recordLastActionEffect(record) {
1236
+ const summary = summarizeEffect(record);
1237
+ lastActionEffect = {
1238
+ ...record,
1239
+ ...summary
1240
+ };
1241
+ return lastActionEffect;
1242
+ }
1243
+ function getLastActionEffectRecord() {
1244
+ return lastActionEffect;
1245
+ }
1246
+ function inspectDomElementState(el) {
1247
+ const rect = el.getBoundingClientRect();
1248
+ const describedByIds = normalizeText(el.getAttribute("aria-describedby")).split(/\s+/).filter(Boolean);
1249
+ const describedByText = describedByIds.map((id) => normalizeText(document.getElementById(id)?.textContent)).filter(Boolean).join(" | ");
1250
+ const labelText = normalizeText(
1251
+ el.labels?.[0]?.textContent || el.getAttribute("aria-label") || el.getAttribute("name") || el.getAttribute("id") || el.textContent
1252
+ );
1253
+ return {
1254
+ tag: el.tagName.toLowerCase(),
1255
+ label: labelText || null,
1256
+ text: normalizeText(el.textContent).slice(0, 300) || null,
1257
+ value: "value" in el ? normalizeText(String(el.value || "")) || null : null,
1258
+ visible: isElementVisible(el),
1259
+ enabled: !el.hasAttribute("disabled") && el.getAttribute("aria-disabled") !== "true",
1260
+ focused: document.activeElement === el,
1261
+ ariaInvalid: el.getAttribute("aria-invalid") === "true",
1262
+ describedByText: describedByText || null,
1263
+ role: el.getAttribute("role") || null,
1264
+ selectorHint: selectorHintFor(el),
1265
+ rect: {
1266
+ x: Math.round(rect.x),
1267
+ y: Math.round(rect.y),
1268
+ width: Math.round(rect.width),
1269
+ height: Math.round(rect.height)
1270
+ }
1271
+ };
1272
+ }
1273
+
1274
+ // src/hooks/useModelNexSocket.ts
1122
1275
  function useModelNexSocket({
1123
1276
  serverUrl,
1124
1277
  actions,
@@ -1216,11 +1369,22 @@ function useModelNexSocket({
1216
1369
  const NAV_ACTION_IDS = ["navigate_to_documents", "navigate_to_templates", "navigate_to_inbox", "navigate_to_settings", "navigate_to_template", "navigate_to_document", "navigate_to_folder", "navigate_editor_step"];
1217
1370
  const action = currentActions.get(actionId);
1218
1371
  if (action) {
1372
+ const beforeSnapshot = captureDebugSnapshot();
1373
+ const startedAt = Date.now();
1219
1374
  try {
1220
1375
  const validatedParams = action.schema.parse(params);
1221
1376
  log("[SDK] Executing action", { actionId });
1222
1377
  const execResult = await action.execute(validatedParams);
1223
1378
  log("[SDK] Action completed", { actionId });
1379
+ recordLastActionEffect({
1380
+ actionId,
1381
+ success: true,
1382
+ error: null,
1383
+ startedAt,
1384
+ finishedAt: Date.now(),
1385
+ before: beforeSnapshot,
1386
+ after: captureDebugSnapshot()
1387
+ });
1224
1388
  sendResult(true, execResult);
1225
1389
  if (NAV_ACTION_IDS.includes(actionId)) {
1226
1390
  requestAnimationFrame(() => {
@@ -1245,6 +1409,15 @@ function useModelNexSocket({
1245
1409
  } catch (err) {
1246
1410
  const errMsg = err instanceof Error ? err.message : String(err);
1247
1411
  console.error(`[ModelNex SDK] Execution failed for ${actionId}: ${errMsg}`);
1412
+ recordLastActionEffect({
1413
+ actionId,
1414
+ success: false,
1415
+ error: errMsg,
1416
+ startedAt,
1417
+ finishedAt: Date.now(),
1418
+ before: beforeSnapshot,
1419
+ after: captureDebugSnapshot()
1420
+ });
1248
1421
  if (isSdkDebugEnabled(devMode)) {
1249
1422
  window.dispatchEvent(
1250
1423
  new CustomEvent("modelnex-debug", {
@@ -2290,7 +2463,7 @@ function isTourEligible(tour, userProfile) {
2290
2463
  var resolutionCache = /* @__PURE__ */ new Map();
2291
2464
  var DEFAULT_WORKFLOW_SEARCH_LIMIT = 5;
2292
2465
  var WORKFLOW_TYPES = ["onboarding", "tour"];
2293
- function safeQueryAll2(selector) {
2466
+ function safeQueryAll3(selector) {
2294
2467
  try {
2295
2468
  return Array.from(document.querySelectorAll(selector)).filter(
2296
2469
  (el) => el instanceof HTMLElement
@@ -2299,6 +2472,9 @@ function safeQueryAll2(selector) {
2299
2472
  return [];
2300
2473
  }
2301
2474
  }
2475
+ function safeQueryOne(selector) {
2476
+ return safeQueryAll3(selector)[0] || null;
2477
+ }
2302
2478
  var ROW_SELECTORS = 'tr, [role="row"], [role="listitem"], li, [data-row], [class*="row"], [class*="card"], article, section';
2303
2479
  function getRowText(el) {
2304
2480
  const row = el.closest(ROW_SELECTORS);
@@ -2330,6 +2506,20 @@ function makeTarget(el, via) {
2330
2506
  resolvedVia: via
2331
2507
  };
2332
2508
  }
2509
+ function resolveInspectableElement(params, getTagStore) {
2510
+ if (params.selector) {
2511
+ const bySelector = safeQueryOne(params.selector);
2512
+ if (bySelector) return makeTarget(bySelector, "single-selector");
2513
+ }
2514
+ if (params.fingerprint) {
2515
+ return resolveTargetElement(
2516
+ params.fingerprint,
2517
+ { patternId: params.patternId, textContaining: params.textContaining },
2518
+ getTagStore()
2519
+ );
2520
+ }
2521
+ return null;
2522
+ }
2333
2523
  function resolveTargetElement(fingerprint, options, tagStore) {
2334
2524
  const elements = extractInteractiveElements();
2335
2525
  const byFp = elements.find((e) => e.fingerprint === fingerprint);
@@ -2338,7 +2528,7 @@ function resolveTargetElement(fingerprint, options, tagStore) {
2338
2528
  if (cached) {
2339
2529
  const cachedEl = elements.find((e) => e.fingerprint === cached.resolvedFingerprint);
2340
2530
  if (cachedEl?.element) return makeTarget(cachedEl.element, "cache");
2341
- const selectorMatches = safeQueryAll2(cached.selector);
2531
+ const selectorMatches = safeQueryAll3(cached.selector);
2342
2532
  if (selectorMatches.length === 1) {
2343
2533
  return makeTarget(selectorMatches[0], "cache");
2344
2534
  }
@@ -2357,7 +2547,7 @@ function resolveTargetElement(fingerprint, options, tagStore) {
2357
2547
  const fallbackTags = patternId ? selectorTags.filter((t) => t.patternId !== patternId) : [];
2358
2548
  for (const tagSet of [candidateTags, fallbackTags]) {
2359
2549
  for (const tag of tagSet) {
2360
- const matched = safeQueryAll2(tag.selector);
2550
+ const matched = safeQueryAll3(tag.selector);
2361
2551
  if (matched.length === 0) continue;
2362
2552
  for (const el of matched) {
2363
2553
  const fp = generateFingerprint(el);
@@ -2443,7 +2633,7 @@ function lastResortScan(fingerprint, options, elements) {
2443
2633
  }
2444
2634
  if (bestEl) return makeTarget(bestEl, "fuzzy");
2445
2635
  if (fpTextHint) {
2446
- const ariaMatches = safeQueryAll2(`[aria-label*="${CSS.escape(fpTextHint)}"], [data-testid*="${CSS.escape(fpTextHint)}"]`);
2636
+ const ariaMatches = safeQueryAll3(`[aria-label*="${CSS.escape(fpTextHint)}"], [data-testid*="${CSS.escape(fpTextHint)}"]`);
2447
2637
  if (textContaining && ariaMatches.length > 1) {
2448
2638
  let best = null;
2449
2639
  let bestS = 0;
@@ -2785,6 +2975,68 @@ function createWaitAction(getTagStore) {
2785
2975
  }
2786
2976
  };
2787
2977
  }
2978
+ var getUiMessagesSchema = import_zod2.z.object({
2979
+ includeHidden: import_zod2.z.boolean().optional().describe("Include currently hidden UI messages when true. Defaults to false."),
2980
+ maxItems: import_zod2.z.number().int().min(1).max(20).optional().describe("Maximum number of messages to return. Defaults to 10.")
2981
+ });
2982
+ function createGetUiMessagesAction() {
2983
+ return {
2984
+ id: "get_ui_messages",
2985
+ description: "Return current UI-facing messages such as alerts, warnings, toasts, banners, and inline validation text.",
2986
+ schema: getUiMessagesSchema,
2987
+ execute: async (params) => {
2988
+ const messages = collectUiMessages(params);
2989
+ return {
2990
+ messages,
2991
+ total: messages.length,
2992
+ capturedAt: Date.now()
2993
+ };
2994
+ }
2995
+ };
2996
+ }
2997
+ var getLastActionEffectSchema = import_zod2.z.object({});
2998
+ function createGetLastActionEffectAction() {
2999
+ return {
3000
+ id: "get_last_action_effect",
3001
+ description: "Return a compact summary of what changed after the most recent agent action.",
3002
+ schema: getLastActionEffectSchema,
3003
+ execute: async () => {
3004
+ return {
3005
+ effect: getLastActionEffectRecord()
3006
+ };
3007
+ }
3008
+ };
3009
+ }
3010
+ var inspectElementStateSchema = import_zod2.z.object({
3011
+ fingerprint: import_zod2.z.string().optional().describe("Fingerprint of the target element to inspect."),
3012
+ selector: import_zod2.z.string().optional().describe("Optional CSS selector for direct inspection when a fingerprint is unavailable."),
3013
+ patternId: import_zod2.z.string().optional().describe("Optional patternId from tagged elements for disambiguation."),
3014
+ textContaining: import_zod2.z.string().optional().describe("Optional row or label text used to disambiguate similar elements.")
3015
+ });
3016
+ function createInspectElementStateAction(getTagStore) {
3017
+ return {
3018
+ id: "inspect_element_state",
3019
+ description: "Inspect one specific element and return its current visibility, enabled state, value/text, aria state, and associated error/help text.",
3020
+ schema: inspectElementStateSchema,
3021
+ execute: async (params) => {
3022
+ const target = resolveInspectableElement(params, getTagStore);
3023
+ if (!target) {
3024
+ return {
3025
+ found: false,
3026
+ reason: "Element not found from the provided fingerprint or selector."
3027
+ };
3028
+ }
3029
+ return {
3030
+ found: true,
3031
+ target: {
3032
+ fingerprint: target.fingerprint,
3033
+ resolvedVia: target.resolvedVia
3034
+ },
3035
+ element: inspectDomElementState(target.element)
3036
+ };
3037
+ }
3038
+ };
3039
+ }
2788
3040
  var searchDocsSchema = import_zod2.z.object({
2789
3041
  query: import_zod2.z.string().describe("The search query to find relevant documentation."),
2790
3042
  limit: import_zod2.z.number().optional().describe("Maximum number of results to return (default: 5).")
@@ -2954,6 +3206,9 @@ var BUILTIN_ACTION_IDS = {
2954
3206
  click: "click_element",
2955
3207
  fill: "fill_input",
2956
3208
  wait: "request_user_action",
3209
+ getUiMessages: "get_ui_messages",
3210
+ getLastActionEffect: "get_last_action_effect",
3211
+ inspectElementState: "inspect_element_state",
2957
3212
  searchDocs: "search_docs",
2958
3213
  searchTaggedElements: "search_tagged_elements",
2959
3214
  searchWorkflows: "search_workflows",
@@ -2990,6 +3245,9 @@ function useBuiltinActions(registerAction, unregisterAction, tagStore, serverUrl
2990
3245
  registerAction(createClickAction(getTagStore));
2991
3246
  registerAction(createFillAction(getTagStore));
2992
3247
  registerAction(createWaitAction(getTagStore));
3248
+ registerAction(createGetUiMessagesAction());
3249
+ registerAction(createGetLastActionEffectAction());
3250
+ registerAction(createInspectElementStateAction(getTagStore));
2993
3251
  registerAction(createSearchDocsAction(getServerUrl, getWebsiteId));
2994
3252
  registerAction(createSearchTaggedElementsAction(getTagStore));
2995
3253
  registerAction(createSearchWorkflowsAction(workflowToolGetters));
@@ -3000,6 +3258,9 @@ function useBuiltinActions(registerAction, unregisterAction, tagStore, serverUrl
3000
3258
  unregisterAction(BUILTIN_ACTION_IDS.click);
3001
3259
  unregisterAction(BUILTIN_ACTION_IDS.fill);
3002
3260
  unregisterAction(BUILTIN_ACTION_IDS.wait);
3261
+ unregisterAction(BUILTIN_ACTION_IDS.getUiMessages);
3262
+ unregisterAction(BUILTIN_ACTION_IDS.getLastActionEffect);
3263
+ unregisterAction(BUILTIN_ACTION_IDS.inspectElementState);
3003
3264
  unregisterAction(BUILTIN_ACTION_IDS.searchDocs);
3004
3265
  unregisterAction(BUILTIN_ACTION_IDS.searchTaggedElements);
3005
3266
  unregisterAction(BUILTIN_ACTION_IDS.searchWorkflows);
package/dist/index.mjs CHANGED
@@ -305,6 +305,157 @@ function serializeContexts(contexts) {
305
305
  return Array.from(contexts.values());
306
306
  }
307
307
 
308
+ // src/utils/debug-tools.ts
309
+ var lastActionEffect = null;
310
+ function normalizeText(value) {
311
+ return String(value || "").replace(/\s+/g, " ").trim();
312
+ }
313
+ function isElementVisible(el) {
314
+ if (!(el instanceof HTMLElement)) return false;
315
+ const style = window.getComputedStyle(el);
316
+ if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") {
317
+ return false;
318
+ }
319
+ const rect = el.getBoundingClientRect();
320
+ return rect.width > 0 && rect.height > 0;
321
+ }
322
+ function safeQueryAll2(selector) {
323
+ try {
324
+ return Array.from(document.querySelectorAll(selector)).filter(
325
+ (el) => el instanceof HTMLElement
326
+ );
327
+ } catch {
328
+ return [];
329
+ }
330
+ }
331
+ function classifyLevel(el, text) {
332
+ const levelHint = `${el.getAttribute("data-variant") || ""} ${el.className || ""} ${text}`.toLowerCase();
333
+ if (/\berror|danger|invalid|failed|required\b/.test(levelHint)) return "error";
334
+ if (/\bwarn|warning|caution\b/.test(levelHint)) return "warning";
335
+ return "info";
336
+ }
337
+ function classifySource(el) {
338
+ const role = (el.getAttribute("role") || "").toLowerCase();
339
+ if (role === "alert") return "alert";
340
+ if (role === "status") return "status";
341
+ if (el.hasAttribute("aria-live")) return "aria-live";
342
+ const classHint = String(el.className || "").toLowerCase();
343
+ if (/\bbanner\b/.test(classHint)) return "banner";
344
+ return "inline";
345
+ }
346
+ function selectorHintFor(el) {
347
+ if (el.id) return `#${el.id}`;
348
+ if (el.getAttribute("data-testid")) return `[data-testid="${el.getAttribute("data-testid")}"]`;
349
+ const role = el.getAttribute("role");
350
+ return role ? `${el.tagName.toLowerCase()}[role="${role}"]` : el.tagName.toLowerCase();
351
+ }
352
+ function collectUiMessages(options) {
353
+ const includeHidden = Boolean(options?.includeHidden);
354
+ const maxItems = Math.max(1, Math.min(Number(options?.maxItems || 10), 20));
355
+ const selectors = [
356
+ '[role="alert"]',
357
+ '[role="status"]',
358
+ "[aria-live]",
359
+ '[aria-invalid="true"]',
360
+ "[data-error]",
361
+ "[data-warning]",
362
+ ".error",
363
+ ".warning",
364
+ ".alert",
365
+ ".toast",
366
+ ".banner"
367
+ ];
368
+ const seen = /* @__PURE__ */ new Set();
369
+ const results = [];
370
+ for (const selector of selectors) {
371
+ for (const el of safeQueryAll2(selector)) {
372
+ const visible = isElementVisible(el);
373
+ if (!includeHidden && !visible) continue;
374
+ const text = normalizeText(el.textContent);
375
+ if (!text || text.length < 2) continue;
376
+ const key = `${selector}:${text.toLowerCase()}`;
377
+ if (seen.has(key)) continue;
378
+ seen.add(key);
379
+ results.push({
380
+ id: `ui_${results.length + 1}`,
381
+ level: classifyLevel(el, text),
382
+ text,
383
+ source: classifySource(el),
384
+ visible,
385
+ selectorHint: selectorHintFor(el)
386
+ });
387
+ if (results.length >= maxItems) return results;
388
+ }
389
+ }
390
+ return results;
391
+ }
392
+ function captureDebugSnapshot() {
393
+ return {
394
+ route: `${window.location.pathname}${window.location.search}${window.location.hash}`,
395
+ title: document.title || "",
396
+ uiMessages: collectUiMessages({ maxItems: 6 }),
397
+ timestamp: Date.now()
398
+ };
399
+ }
400
+ function diffUiMessages(before, after) {
401
+ const beforeKeys = new Set(before.map((entry) => `${entry.level}:${entry.text}`));
402
+ return after.filter((entry) => !beforeKeys.has(`${entry.level}:${entry.text}`));
403
+ }
404
+ function summarizeEffect(record) {
405
+ const routeChanged = record.before.route !== record.after.route;
406
+ const titleChanged = record.before.title !== record.after.title;
407
+ const newMessages = diffUiMessages(record.before.uiMessages, record.after.uiMessages);
408
+ const changed = routeChanged || titleChanged || newMessages.length > 0 || !record.success || record.before.timestamp !== record.after.timestamp;
409
+ const parts = [];
410
+ if (routeChanged) parts.push(`route changed from ${record.before.route} to ${record.after.route}`);
411
+ if (titleChanged) parts.push("page title changed");
412
+ if (newMessages.length > 0) parts.push(`new UI messages: ${newMessages.map((entry) => entry.text).join(" | ")}`);
413
+ if (!record.success && record.error) parts.push(`action failed: ${record.error}`);
414
+ if (parts.length === 0) parts.push("no obvious route/title/message change detected");
415
+ return {
416
+ summary: parts.join("; "),
417
+ changed
418
+ };
419
+ }
420
+ function recordLastActionEffect(record) {
421
+ const summary = summarizeEffect(record);
422
+ lastActionEffect = {
423
+ ...record,
424
+ ...summary
425
+ };
426
+ return lastActionEffect;
427
+ }
428
+ function getLastActionEffectRecord() {
429
+ return lastActionEffect;
430
+ }
431
+ function inspectDomElementState(el) {
432
+ const rect = el.getBoundingClientRect();
433
+ const describedByIds = normalizeText(el.getAttribute("aria-describedby")).split(/\s+/).filter(Boolean);
434
+ const describedByText = describedByIds.map((id) => normalizeText(document.getElementById(id)?.textContent)).filter(Boolean).join(" | ");
435
+ const labelText = normalizeText(
436
+ el.labels?.[0]?.textContent || el.getAttribute("aria-label") || el.getAttribute("name") || el.getAttribute("id") || el.textContent
437
+ );
438
+ return {
439
+ tag: el.tagName.toLowerCase(),
440
+ label: labelText || null,
441
+ text: normalizeText(el.textContent).slice(0, 300) || null,
442
+ value: "value" in el ? normalizeText(String(el.value || "")) || null : null,
443
+ visible: isElementVisible(el),
444
+ enabled: !el.hasAttribute("disabled") && el.getAttribute("aria-disabled") !== "true",
445
+ focused: document.activeElement === el,
446
+ ariaInvalid: el.getAttribute("aria-invalid") === "true",
447
+ describedByText: describedByText || null,
448
+ role: el.getAttribute("role") || null,
449
+ selectorHint: selectorHintFor(el),
450
+ rect: {
451
+ x: Math.round(rect.x),
452
+ y: Math.round(rect.y),
453
+ width: Math.round(rect.width),
454
+ height: Math.round(rect.height)
455
+ }
456
+ };
457
+ }
458
+
308
459
  // src/hooks/useModelNexSocket.ts
309
460
  function useModelNexSocket({
310
461
  serverUrl,
@@ -403,11 +554,22 @@ function useModelNexSocket({
403
554
  const NAV_ACTION_IDS = ["navigate_to_documents", "navigate_to_templates", "navigate_to_inbox", "navigate_to_settings", "navigate_to_template", "navigate_to_document", "navigate_to_folder", "navigate_editor_step"];
404
555
  const action = currentActions.get(actionId);
405
556
  if (action) {
557
+ const beforeSnapshot = captureDebugSnapshot();
558
+ const startedAt = Date.now();
406
559
  try {
407
560
  const validatedParams = action.schema.parse(params);
408
561
  log("[SDK] Executing action", { actionId });
409
562
  const execResult = await action.execute(validatedParams);
410
563
  log("[SDK] Action completed", { actionId });
564
+ recordLastActionEffect({
565
+ actionId,
566
+ success: true,
567
+ error: null,
568
+ startedAt,
569
+ finishedAt: Date.now(),
570
+ before: beforeSnapshot,
571
+ after: captureDebugSnapshot()
572
+ });
411
573
  sendResult(true, execResult);
412
574
  if (NAV_ACTION_IDS.includes(actionId)) {
413
575
  requestAnimationFrame(() => {
@@ -432,6 +594,15 @@ function useModelNexSocket({
432
594
  } catch (err) {
433
595
  const errMsg = err instanceof Error ? err.message : String(err);
434
596
  console.error(`[ModelNex SDK] Execution failed for ${actionId}: ${errMsg}`);
597
+ recordLastActionEffect({
598
+ actionId,
599
+ success: false,
600
+ error: errMsg,
601
+ startedAt,
602
+ finishedAt: Date.now(),
603
+ before: beforeSnapshot,
604
+ after: captureDebugSnapshot()
605
+ });
435
606
  if (isSdkDebugEnabled(devMode)) {
436
607
  window.dispatchEvent(
437
608
  new CustomEvent("modelnex-debug", {
@@ -1471,7 +1642,7 @@ function isTourEligible(tour, userProfile) {
1471
1642
  var resolutionCache = /* @__PURE__ */ new Map();
1472
1643
  var DEFAULT_WORKFLOW_SEARCH_LIMIT = 5;
1473
1644
  var WORKFLOW_TYPES = ["onboarding", "tour"];
1474
- function safeQueryAll2(selector) {
1645
+ function safeQueryAll3(selector) {
1475
1646
  try {
1476
1647
  return Array.from(document.querySelectorAll(selector)).filter(
1477
1648
  (el) => el instanceof HTMLElement
@@ -1480,6 +1651,9 @@ function safeQueryAll2(selector) {
1480
1651
  return [];
1481
1652
  }
1482
1653
  }
1654
+ function safeQueryOne(selector) {
1655
+ return safeQueryAll3(selector)[0] || null;
1656
+ }
1483
1657
  var ROW_SELECTORS = 'tr, [role="row"], [role="listitem"], li, [data-row], [class*="row"], [class*="card"], article, section';
1484
1658
  function getRowText(el) {
1485
1659
  const row = el.closest(ROW_SELECTORS);
@@ -1511,6 +1685,20 @@ function makeTarget(el, via) {
1511
1685
  resolvedVia: via
1512
1686
  };
1513
1687
  }
1688
+ function resolveInspectableElement(params, getTagStore) {
1689
+ if (params.selector) {
1690
+ const bySelector = safeQueryOne(params.selector);
1691
+ if (bySelector) return makeTarget(bySelector, "single-selector");
1692
+ }
1693
+ if (params.fingerprint) {
1694
+ return resolveTargetElement(
1695
+ params.fingerprint,
1696
+ { patternId: params.patternId, textContaining: params.textContaining },
1697
+ getTagStore()
1698
+ );
1699
+ }
1700
+ return null;
1701
+ }
1514
1702
  function resolveTargetElement(fingerprint, options, tagStore) {
1515
1703
  const elements = extractInteractiveElements();
1516
1704
  const byFp = elements.find((e) => e.fingerprint === fingerprint);
@@ -1519,7 +1707,7 @@ function resolveTargetElement(fingerprint, options, tagStore) {
1519
1707
  if (cached) {
1520
1708
  const cachedEl = elements.find((e) => e.fingerprint === cached.resolvedFingerprint);
1521
1709
  if (cachedEl?.element) return makeTarget(cachedEl.element, "cache");
1522
- const selectorMatches = safeQueryAll2(cached.selector);
1710
+ const selectorMatches = safeQueryAll3(cached.selector);
1523
1711
  if (selectorMatches.length === 1) {
1524
1712
  return makeTarget(selectorMatches[0], "cache");
1525
1713
  }
@@ -1538,7 +1726,7 @@ function resolveTargetElement(fingerprint, options, tagStore) {
1538
1726
  const fallbackTags = patternId ? selectorTags.filter((t) => t.patternId !== patternId) : [];
1539
1727
  for (const tagSet of [candidateTags, fallbackTags]) {
1540
1728
  for (const tag of tagSet) {
1541
- const matched = safeQueryAll2(tag.selector);
1729
+ const matched = safeQueryAll3(tag.selector);
1542
1730
  if (matched.length === 0) continue;
1543
1731
  for (const el of matched) {
1544
1732
  const fp = generateFingerprint(el);
@@ -1624,7 +1812,7 @@ function lastResortScan(fingerprint, options, elements) {
1624
1812
  }
1625
1813
  if (bestEl) return makeTarget(bestEl, "fuzzy");
1626
1814
  if (fpTextHint) {
1627
- const ariaMatches = safeQueryAll2(`[aria-label*="${CSS.escape(fpTextHint)}"], [data-testid*="${CSS.escape(fpTextHint)}"]`);
1815
+ const ariaMatches = safeQueryAll3(`[aria-label*="${CSS.escape(fpTextHint)}"], [data-testid*="${CSS.escape(fpTextHint)}"]`);
1628
1816
  if (textContaining && ariaMatches.length > 1) {
1629
1817
  let best = null;
1630
1818
  let bestS = 0;
@@ -1966,6 +2154,68 @@ function createWaitAction(getTagStore) {
1966
2154
  }
1967
2155
  };
1968
2156
  }
2157
+ var getUiMessagesSchema = z2.object({
2158
+ includeHidden: z2.boolean().optional().describe("Include currently hidden UI messages when true. Defaults to false."),
2159
+ maxItems: z2.number().int().min(1).max(20).optional().describe("Maximum number of messages to return. Defaults to 10.")
2160
+ });
2161
+ function createGetUiMessagesAction() {
2162
+ return {
2163
+ id: "get_ui_messages",
2164
+ description: "Return current UI-facing messages such as alerts, warnings, toasts, banners, and inline validation text.",
2165
+ schema: getUiMessagesSchema,
2166
+ execute: async (params) => {
2167
+ const messages = collectUiMessages(params);
2168
+ return {
2169
+ messages,
2170
+ total: messages.length,
2171
+ capturedAt: Date.now()
2172
+ };
2173
+ }
2174
+ };
2175
+ }
2176
+ var getLastActionEffectSchema = z2.object({});
2177
+ function createGetLastActionEffectAction() {
2178
+ return {
2179
+ id: "get_last_action_effect",
2180
+ description: "Return a compact summary of what changed after the most recent agent action.",
2181
+ schema: getLastActionEffectSchema,
2182
+ execute: async () => {
2183
+ return {
2184
+ effect: getLastActionEffectRecord()
2185
+ };
2186
+ }
2187
+ };
2188
+ }
2189
+ var inspectElementStateSchema = z2.object({
2190
+ fingerprint: z2.string().optional().describe("Fingerprint of the target element to inspect."),
2191
+ selector: z2.string().optional().describe("Optional CSS selector for direct inspection when a fingerprint is unavailable."),
2192
+ patternId: z2.string().optional().describe("Optional patternId from tagged elements for disambiguation."),
2193
+ textContaining: z2.string().optional().describe("Optional row or label text used to disambiguate similar elements.")
2194
+ });
2195
+ function createInspectElementStateAction(getTagStore) {
2196
+ return {
2197
+ id: "inspect_element_state",
2198
+ description: "Inspect one specific element and return its current visibility, enabled state, value/text, aria state, and associated error/help text.",
2199
+ schema: inspectElementStateSchema,
2200
+ execute: async (params) => {
2201
+ const target = resolveInspectableElement(params, getTagStore);
2202
+ if (!target) {
2203
+ return {
2204
+ found: false,
2205
+ reason: "Element not found from the provided fingerprint or selector."
2206
+ };
2207
+ }
2208
+ return {
2209
+ found: true,
2210
+ target: {
2211
+ fingerprint: target.fingerprint,
2212
+ resolvedVia: target.resolvedVia
2213
+ },
2214
+ element: inspectDomElementState(target.element)
2215
+ };
2216
+ }
2217
+ };
2218
+ }
1969
2219
  var searchDocsSchema = z2.object({
1970
2220
  query: z2.string().describe("The search query to find relevant documentation."),
1971
2221
  limit: z2.number().optional().describe("Maximum number of results to return (default: 5).")
@@ -2135,6 +2385,9 @@ var BUILTIN_ACTION_IDS = {
2135
2385
  click: "click_element",
2136
2386
  fill: "fill_input",
2137
2387
  wait: "request_user_action",
2388
+ getUiMessages: "get_ui_messages",
2389
+ getLastActionEffect: "get_last_action_effect",
2390
+ inspectElementState: "inspect_element_state",
2138
2391
  searchDocs: "search_docs",
2139
2392
  searchTaggedElements: "search_tagged_elements",
2140
2393
  searchWorkflows: "search_workflows",
@@ -2171,6 +2424,9 @@ function useBuiltinActions(registerAction, unregisterAction, tagStore, serverUrl
2171
2424
  registerAction(createClickAction(getTagStore));
2172
2425
  registerAction(createFillAction(getTagStore));
2173
2426
  registerAction(createWaitAction(getTagStore));
2427
+ registerAction(createGetUiMessagesAction());
2428
+ registerAction(createGetLastActionEffectAction());
2429
+ registerAction(createInspectElementStateAction(getTagStore));
2174
2430
  registerAction(createSearchDocsAction(getServerUrl, getWebsiteId));
2175
2431
  registerAction(createSearchTaggedElementsAction(getTagStore));
2176
2432
  registerAction(createSearchWorkflowsAction(workflowToolGetters));
@@ -2181,6 +2437,9 @@ function useBuiltinActions(registerAction, unregisterAction, tagStore, serverUrl
2181
2437
  unregisterAction(BUILTIN_ACTION_IDS.click);
2182
2438
  unregisterAction(BUILTIN_ACTION_IDS.fill);
2183
2439
  unregisterAction(BUILTIN_ACTION_IDS.wait);
2440
+ unregisterAction(BUILTIN_ACTION_IDS.getUiMessages);
2441
+ unregisterAction(BUILTIN_ACTION_IDS.getLastActionEffect);
2442
+ unregisterAction(BUILTIN_ACTION_IDS.inspectElementState);
2184
2443
  unregisterAction(BUILTIN_ACTION_IDS.searchDocs);
2185
2444
  unregisterAction(BUILTIN_ACTION_IDS.searchTaggedElements);
2186
2445
  unregisterAction(BUILTIN_ACTION_IDS.searchWorkflows);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@modelnex/sdk",
3
- "version": "0.5.45",
3
+ "version": "0.5.46",
4
4
  "description": "React SDK for natural language control of web apps via AI agents",
5
5
  "main": "./dist/index.js",
6
6
  "module": "./dist/index.mjs",