@celljs/ta-app-use 4.17.0 → 4.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +5 -4
- package/README.md +5 -4
- package/extension/index.mjs +240 -11
- package/host/index.mjs +20 -0
- package/package.json +3 -3
- package/web/index.js +723 -90
package/README.en.md
CHANGED
|
@@ -7,7 +7,7 @@ A built-in package of Ta: agents do to Ta what you do with its pages. Say "turn
|
|
|
7
7
|
## Two kinds of doing
|
|
8
8
|
|
|
9
9
|
- **Direct calls**: what ends as data (settings, pins, stops, extensions on or off, answers to what waits for you to decide) goes through the catalog of actions, no page in between, with the phone locked too.
|
|
10
|
-
- **Operating the screen**: what needs what is on the screen now, a "show me", or a page of an extension with nothing else, is done on a page of Ta as a browser use, with a cursor and motion. By default on **
|
|
10
|
+
- **Operating the screen**: what needs what is on the screen now, a "show me", or a page of an extension with nothing else, is done on a page of Ta as a browser use, with a cursor and motion. By default on **the page you use**, as your shadow: on a phone's page on a phone, a computer's on a computer, with a pill "Ta is at work · Stop"; in a call it takes you to another page and the call goes on. Long work you need not watch (`apart: true`) is done on **Ta's own screen** (a `?ta-screen` frame in your page: in a corner of a computer, to drag, enlarge or put away; on a phone the pill has Watch too).
|
|
11
11
|
|
|
12
12
|
## Tools
|
|
13
13
|
|
|
@@ -15,9 +15,10 @@ A built-in package of Ta: agents do to Ta what you do with its pages. Say "turn
|
|
|
15
15
|
| --- | --- |
|
|
16
16
|
| `app_commands({ query? })` | The actions: id, what each does, its input and risk |
|
|
17
17
|
| `app_run({ command, input? })` | Runs one, checked for who asked the run |
|
|
18
|
-
| `app_look({ part?,
|
|
19
|
-
| `app_go({ to,
|
|
20
|
-
| `app_do({ id, action?, input?, show?,
|
|
18
|
+
| `app_look({ part?, apart? })` | What is on the screen: where it is, the places to go, the parts (the list of conversations, the conversation, the settings...) with their rows and what each does, and what else can be clicked or typed into, each with an id |
|
|
19
|
+
| `app_go({ to, apart? })` | Goes to a page, a list, a section of the settings or a conversation (`chat:<name>`), and says what is there |
|
|
20
|
+
| `app_do({ id, action?, input?, show?, apart? })` | Clicks, types, chooses, or does an action of a part, answering only what changed; on Ta's own screen one that stands for an action of the catalog runs as `app_run` unless `show: true` |
|
|
21
|
+
| `app_do({ find, to? })` | Without a look first: goes to `to`, finds what the screen calls so and does it, one step for three; the whole screen when not found |
|
|
21
22
|
|
|
22
23
|
Actions of Ta: `settings.get`, `settings.set`, `chats.list`, `chats.pin`, `chats.stop`, `asks.list`, `asks.answer`, `extensions.list`, `extensions.set`. Other packages add theirs with `host.commands.register(...)` on their server side; they go when the package is turned off.
|
|
23
24
|
|
package/README.md
CHANGED
|
@@ -7,7 +7,7 @@ Ta 的内置包:智能体替你操作 Ta。你说"打开通话字幕""把那
|
|
|
7
7
|
## 两类操作
|
|
8
8
|
|
|
9
9
|
- **直接调用**:结果是一份数据的(设置、置顶、停止、开关扩展、替你回答待你决定的),调动作目录,不经过页面,手机锁屏也能做。
|
|
10
|
-
- **界面操作**:要看屏幕上此刻有什么、要"做给你看"、或只有界面的扩展页,像 browser use 一样操作 Ta
|
|
10
|
+
- **界面操作**:要看屏幕上此刻有什么、要"做给你看"、或只有界面的扩展页,像 browser use 一样操作 Ta 的页面,带光标和动画。默认就在**你正在用的那一页**上做,像你的影子:手机上在手机界面点,电脑上在电脑界面点,顶部胶囊"Ta 正在操作 · 停止";打电话时它带你换页,通话不断。不用你看的长活(`apart: true`)在 **Ta 自己的屏幕**上做(你页面里的一个 `?ta-screen` 小窗:电脑上在角落,可拖动、放大、收起;手机上胶囊多一个"看看")。
|
|
11
11
|
|
|
12
12
|
## 工具
|
|
13
13
|
|
|
@@ -15,9 +15,10 @@ Ta 的内置包:智能体替你操作 Ta。你说"打开通话字幕""把那
|
|
|
15
15
|
| --- | --- |
|
|
16
16
|
| `app_commands({ query? })` | 查有哪些动作:id、做什么、参数、风险 |
|
|
17
17
|
| `app_run({ command, input? })` | 执行一个动作,按这一轮的发起人检查权限 |
|
|
18
|
-
| `app_look({ part?,
|
|
19
|
-
| `app_go({ to,
|
|
20
|
-
| `app_do({ id, action?, input?, show?,
|
|
18
|
+
| `app_look({ part?, apart? })` | 屏幕上现在有什么:在哪、能去哪、各部分(对话列表、对话、设置……)的行和能做的事、其余能点能输入的,每样带 id |
|
|
19
|
+
| `app_go({ to, apart? })` | 走到一页、一个列表、设置某一节或一个对话(`chat:<名字>`),返回到了以后的样子 |
|
|
20
|
+
| `app_do({ id, action?, input?, show?, apart? })` | 点、输入、选择,或做某一部分的动作,回答只说变了什么;在 Ta 自己的屏幕上时,指向动作目录的直接按 `app_run` 做,除非 `show: true` |
|
|
21
|
+
| `app_do({ find, to? })` | 不先看:`to` 先走过去,按屏幕上的字找到再做,一步顶三步;找不到就给整屏 |
|
|
21
22
|
|
|
22
23
|
核心动作:`settings.get`、`settings.set`、`chats.list`、`chats.pin`、`chats.stop`、`asks.list`、`asks.answer`、`extensions.list`、`extensions.set`。别的包在服务端用 `host.commands.register(...)` 加自己的动作,包关掉时随之去掉。
|
|
23
24
|
|
package/extension/index.mjs
CHANGED
|
@@ -298,6 +298,20 @@ var TA_WEB_SETTINGS = [
|
|
|
298
298
|
title: words("Model of tasks", "任务的模型", "任務的模型", "タスクのモデル")
|
|
299
299
|
},
|
|
300
300
|
{ key: "reduceMotion", type: "boolean", default: false, what: "Fewer animations.", title: words("Reduce motion", "减少动画", "減少動畫", "動きを減らす") },
|
|
301
|
+
{
|
|
302
|
+
key: "diagramStyle",
|
|
303
|
+
type: "enum",
|
|
304
|
+
values: ["clean", "sketch", "colour", "motion"],
|
|
305
|
+
default: "clean",
|
|
306
|
+
what: "How diagrams in replies draw: in the colours of the page, as by hand, a colour to a node, or coming in with flowing lines.",
|
|
307
|
+
title: words("Diagram style", "图表风格", "圖表風格", "図のスタイル"),
|
|
308
|
+
shown: {
|
|
309
|
+
clean: words("clean", "简洁", "簡潔", "シンプル"),
|
|
310
|
+
sketch: words("sketch", "草图", "草圖", "スケッチ"),
|
|
311
|
+
colour: words("colour", "彩色", "彩色", "カラー"),
|
|
312
|
+
motion: words("motion", "动效", "動效", "モーション")
|
|
313
|
+
}
|
|
314
|
+
},
|
|
301
315
|
{
|
|
302
316
|
key: "keepAliveMinutes",
|
|
303
317
|
type: "number",
|
|
@@ -368,6 +382,7 @@ var NAME = "@celljs/ta-app-use";
|
|
|
368
382
|
var DONE_MAX = 20;
|
|
369
383
|
|
|
370
384
|
// web-src/app-util.ts
|
|
385
|
+
var SCROLL_STEPS = ["down", "up", "top", "bottom"];
|
|
371
386
|
var DATA_CHARS = 8000;
|
|
372
387
|
var AppUseUtil;
|
|
373
388
|
((AppUseUtil) => {
|
|
@@ -408,6 +423,25 @@ var AppUseUtil;
|
|
|
408
423
|
`);
|
|
409
424
|
}
|
|
410
425
|
AppUseUtil.resultText = resultText;
|
|
426
|
+
function scroll(value) {
|
|
427
|
+
if (value === undefined || value === null || value === "") {
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
if (typeof value === "string" && SCROLL_STEPS.includes(value.trim().toLowerCase())) {
|
|
431
|
+
return value.trim().toLowerCase();
|
|
432
|
+
}
|
|
433
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
434
|
+
const { item, find: find2 } = value;
|
|
435
|
+
if (typeof item === "string" && item.trim() && find2 === undefined) {
|
|
436
|
+
return { item: item.trim() };
|
|
437
|
+
}
|
|
438
|
+
if (typeof find2 === "string" && find2.trim() && item === undefined) {
|
|
439
|
+
return { find: find2.trim() };
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
throw new Error(`scroll takes ${SCROLL_STEPS.join(", ")}, { "item": "p1.7" } or { "find": "words of a row" }, not ${JSON.stringify(value)}.`);
|
|
443
|
+
}
|
|
444
|
+
AppUseUtil.scroll = scroll;
|
|
411
445
|
function add(done, entry) {
|
|
412
446
|
return [...entries(done), entry].slice(-DONE_MAX);
|
|
413
447
|
}
|
|
@@ -422,31 +456,218 @@ var AppUseUtil;
|
|
|
422
456
|
AppUseUtil.fresh = fresh;
|
|
423
457
|
})(AppUseUtil ||= {});
|
|
424
458
|
|
|
459
|
+
// web-src/screen-protocol.ts
|
|
460
|
+
var LOOK_MAX = 3000;
|
|
461
|
+
var OLD_MAX = 200;
|
|
462
|
+
var CONTROLS_MAX = 50;
|
|
463
|
+
|
|
464
|
+
// web-src/screen-text.ts
|
|
465
|
+
var ScreenText;
|
|
466
|
+
((ScreenText) => {
|
|
467
|
+
const cut = (text, length) => text.length > length ? `${text.slice(0, length - 1)}…` : text;
|
|
468
|
+
const HEAD = /^(The user's screen|Ta's own screen) \(/m;
|
|
469
|
+
function placeNames(places) {
|
|
470
|
+
return places.map((place) => place.title && places.filter((one) => one.title === place.title).length === 1 ? place.title : `${place.id}${place.title ? ` “${place.title}”` : ""}`);
|
|
471
|
+
}
|
|
472
|
+
function scrolled(scrollable) {
|
|
473
|
+
const { at, offset, total, shown } = scrollable.position;
|
|
474
|
+
const span = total !== undefined ? `rows ${offset}–${offset + Math.max(1, shown ?? 1) - 1} of ${total}` : `${offset}% down`;
|
|
475
|
+
const where = at === "top" ? "at the top" : at === "bottom" ? "at the bottom" : "in the middle";
|
|
476
|
+
const side = scrollable.more === "top" ? "above" : "below";
|
|
477
|
+
const more = scrollable.more === "none" ? "all loaded" : scrollable.loading ? `more loading ${side}: look again` : scrollable.more ? `more ${side} as it scrolls ${scrollable.more === "top" ? "up" : "down"}` : "";
|
|
478
|
+
return [span, where, more].filter(Boolean).join(", ");
|
|
479
|
+
}
|
|
480
|
+
ScreenText.scrolled = scrolled;
|
|
481
|
+
function rowId(partIndex, part, itemIndex) {
|
|
482
|
+
const item = part.items[itemIndex];
|
|
483
|
+
if (item?.at !== undefined) {
|
|
484
|
+
return `p${partIndex + 1}.${item.at}`;
|
|
485
|
+
}
|
|
486
|
+
if (part.items.some((one) => one.at !== undefined)) {
|
|
487
|
+
return `p${partIndex + 1}.f${part.items.slice(0, itemIndex + 1).filter((one) => one.at === undefined).length}`;
|
|
488
|
+
}
|
|
489
|
+
return `p${partIndex + 1}.${itemIndex + 1}`;
|
|
490
|
+
}
|
|
491
|
+
ScreenText.rowId = rowId;
|
|
492
|
+
function whole(text) {
|
|
493
|
+
return HEAD.test(text);
|
|
494
|
+
}
|
|
495
|
+
ScreenText.whole = whole;
|
|
496
|
+
const heads = (line) => /^\[p\d+\]/.test(line) || line === "Also on the screen:" || /^ {2}Dialog “/.test(line);
|
|
497
|
+
function changes(before, after) {
|
|
498
|
+
if (!before || !whole(before) || !whole(after) || before.split(`
|
|
499
|
+
`)[0] !== after.split(`
|
|
500
|
+
`)[0]) {
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
const old = new Set(before.split(`
|
|
504
|
+
`));
|
|
505
|
+
const lines = after.split(`
|
|
506
|
+
`);
|
|
507
|
+
const now = new Set(lines);
|
|
508
|
+
const body = lines.filter((line) => line.trim());
|
|
509
|
+
const changed = body.filter((line) => !old.has(line));
|
|
510
|
+
if (changed.length > Math.max(3, body.length * 0.4)) {
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
const gone = before.split(`
|
|
514
|
+
`).filter((line) => line.trim() && !now.has(line)).length - changed.length;
|
|
515
|
+
const out = [];
|
|
516
|
+
let head;
|
|
517
|
+
let said;
|
|
518
|
+
for (const line of lines) {
|
|
519
|
+
if (heads(line)) {
|
|
520
|
+
head = line;
|
|
521
|
+
}
|
|
522
|
+
if (!line.trim() || old.has(line)) {
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
if (head && head !== line && said !== head) {
|
|
526
|
+
out.push(head);
|
|
527
|
+
}
|
|
528
|
+
out.push(line);
|
|
529
|
+
said = head;
|
|
530
|
+
}
|
|
531
|
+
if (!out.length) {
|
|
532
|
+
return gone > 0 ? `Nothing new on the screen; ${gone} line(s) went away.` : "Nothing else on the screen changed.";
|
|
533
|
+
}
|
|
534
|
+
return `What changed (the rest is as before, with the same ids):
|
|
535
|
+
${out.join(`
|
|
536
|
+
`)}${gone > 0 ? `
|
|
537
|
+
(${gone} more line(s) went away)` : ""}`;
|
|
538
|
+
}
|
|
539
|
+
ScreenText.changes = changes;
|
|
540
|
+
function forget(messages, tools) {
|
|
541
|
+
const textOf = (message) => Array.isArray(message.content) ? message.content.map((part) => part.type === "text" ? part.text ?? "" : "").join("") : typeof message.content === "string" ? message.content : "";
|
|
542
|
+
const sight = (message) => message.role === "toolResult" && tools.includes(message.toolName ?? "");
|
|
543
|
+
let last = -1;
|
|
544
|
+
messages.forEach((message, index) => {
|
|
545
|
+
if (sight(message) && whole(textOf(message))) {
|
|
546
|
+
last = index;
|
|
547
|
+
}
|
|
548
|
+
});
|
|
549
|
+
let cut2 = false;
|
|
550
|
+
const out = messages.map((message, index) => {
|
|
551
|
+
const text = index < last && sight(message) ? textOf(message) : "";
|
|
552
|
+
if (text.length <= OLD_MAX) {
|
|
553
|
+
return message;
|
|
554
|
+
}
|
|
555
|
+
cut2 = true;
|
|
556
|
+
const first = text.split(`
|
|
557
|
+
`).find((line) => line.trim()) ?? "";
|
|
558
|
+
return { ...message, content: [{ type: "text", text: `${cut1(first)} … (an older sight of the screen, left out: a later one is below)` }] };
|
|
559
|
+
});
|
|
560
|
+
return cut2 ? out : undefined;
|
|
561
|
+
}
|
|
562
|
+
ScreenText.forget = forget;
|
|
563
|
+
const cut1 = (line) => cut(line, 160);
|
|
564
|
+
function look(sight, only) {
|
|
565
|
+
const targets = new Map;
|
|
566
|
+
const lines = [];
|
|
567
|
+
if (!only) {
|
|
568
|
+
lines.push(`${sight.whose === "ta" ? "Ta's own screen" : "The user's screen"} (${sight.device}): ${sight.where || "starting"}`);
|
|
569
|
+
if (sight.places.length) {
|
|
570
|
+
lines.push(`Places (app_go to): ${placeNames(sight.places).join(" · ")} · chat:<name> a conversation`);
|
|
571
|
+
}
|
|
572
|
+
if (sight.headings.length) {
|
|
573
|
+
lines.push(`Headings: ${sight.headings.map((heading) => cut(heading, 60)).join(" · ")}`);
|
|
574
|
+
}
|
|
575
|
+
const page = sight.scroll;
|
|
576
|
+
const index = page?.part ? sight.parts.findIndex((part) => part.id === page.part) : -1;
|
|
577
|
+
if (page) {
|
|
578
|
+
lines.push(`Scrolls (app_do scroll, no id): ${index >= 0 ? `[p${index + 1}]` : `the page, ${scrolled(page.scrollable)}`}`);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
sight.parts.forEach((part, index) => {
|
|
582
|
+
const id = `p${index + 1}`;
|
|
583
|
+
targets.set(id, { part: part.id });
|
|
584
|
+
if (only && only !== part.id && only !== id) {
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
lines.push("", `[${id}] ${part.title}${part.state ? ` — ${part.state}` : ""}${part.floats ? " (over every page)" : ""}`);
|
|
588
|
+
if (part.scrollable) {
|
|
589
|
+
lines.push(` scrolls: ${scrolled(part.scrollable)}`);
|
|
590
|
+
}
|
|
591
|
+
part.items.forEach((item, itemIndex) => {
|
|
592
|
+
const itemId = rowId(index, part, itemIndex);
|
|
593
|
+
targets.set(itemId, { part: part.id, item: item.id });
|
|
594
|
+
lines.push(` [${itemId}] ${cut(item.text, 200)}${item.marks?.length ? ` · ${item.marks.map((mark) => cut(mark, 80)).join(" · ")}` : ""}`);
|
|
595
|
+
});
|
|
596
|
+
if (part.actions.length) {
|
|
597
|
+
const actions = part.actions.map((action) => `${action.name}${action.label.toLowerCase() !== action.name ? ` “${action.label}”` : ""}${action.each ? " [a row]" : ""}${action.takes ? ` (${action.takes})` : ""}${action.command ? ` = app_run ${action.command}` : ""}${action.danger ? " !danger" : ""}`);
|
|
598
|
+
lines.push(` does: ${actions.join(" · ")}`);
|
|
599
|
+
}
|
|
600
|
+
});
|
|
601
|
+
const controls = sight.controls.slice(0, CONTROLS_MAX);
|
|
602
|
+
controls.forEach((_control, index) => {
|
|
603
|
+
targets.set(`e${index + 1}`, { control: index });
|
|
604
|
+
});
|
|
605
|
+
if (!only && controls.length) {
|
|
606
|
+
lines.push("", "Also on the screen:");
|
|
607
|
+
let dialog;
|
|
608
|
+
controls.forEach((control, index) => {
|
|
609
|
+
if (control.dialog !== dialog) {
|
|
610
|
+
dialog = control.dialog;
|
|
611
|
+
if (dialog !== undefined) {
|
|
612
|
+
lines.push(` Dialog “${cut(dialog, 80)}”:`);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
const state = control.checked !== undefined ? control.checked ? " on" : " off" : control.value !== undefined ? ` = “${cut(control.value, 80)}”` : "";
|
|
616
|
+
lines.push(` ${control.dialog !== undefined ? " " : ""}[e${index + 1}] ${control.kind} “${cut(control.label, 80)}”${state}${control.danger ? " !danger" : ""}`);
|
|
617
|
+
});
|
|
618
|
+
if (sight.controls.length > controls.length) {
|
|
619
|
+
lines.push(` … ${sight.controls.length - controls.length} more: look at a part, or go elsewhere.`);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
const all = lines.join(`
|
|
623
|
+
`).trim();
|
|
624
|
+
return { text: all.length > LOOK_MAX ? `${all.slice(0, LOOK_MAX - 40)}
|
|
625
|
+
… (more: app_look a part)` : all, targets };
|
|
626
|
+
}
|
|
627
|
+
ScreenText.look = look;
|
|
628
|
+
})(ScreenText ||= {});
|
|
629
|
+
|
|
425
630
|
// web-src/extension.ts
|
|
426
631
|
var NO_SERVER = "Doing things to Ta needs the Ta service: start it with `ta serve`.";
|
|
427
632
|
var APP_RUN_DESCRIPTION = "Does something to Ta itself at once, as the user would on its pages but without them: change its settings (look, language, voice, captions of calls, the double, models), list, pin or stop conversations, read and answer what waits for the user to decide (with what they said), turn a built-in extension on or off. Prefer it to operating the screen: it is quick, needs no page open, and the user's pages show the change at once, with a notice and Undo. command: settings.get, settings.set, chats.list, chats.pin, chats.stop, asks.list, asks.answer, extensions.list, extensions.set, or another of app_commands (packages add theirs); input: as app_commands says. Do what the user asked; ask before anything they did not.";
|
|
428
|
-
var
|
|
633
|
+
var APART = {
|
|
634
|
+
type: "boolean",
|
|
635
|
+
description: "On Ta's own screen beside the user's, for long work they need not watch or that should not move their page; else on theirs."
|
|
636
|
+
};
|
|
429
637
|
var SCREEN_TOOLS = [
|
|
430
638
|
{
|
|
431
639
|
name: "app_look",
|
|
432
640
|
label: "Look at the screen",
|
|
433
|
-
description: "What is on
|
|
434
|
-
parameters: { type: "object", properties: { part: { type: "string" },
|
|
641
|
+
description: "What is on the user's screen of Ta now, the very page they see, on the device they use (a phone's on a phone, a computer's on a computer): where it is, the places to go, the parts (the list of conversations, the conversation, the settings...) with their rows and what each does, and what else can be clicked or typed into, each with an id for app_do. A part that scrolls lists only its rows in sight and says where it is (`scrolls: rows 3–12 of 40, in the middle, more below as it scrolls down`; `all loaded` at the real end), and `Scrolls` names what scrolls without an id: scroll with app_do to read more. A call going on is a part over every page, listed first (Call: what it is at, with which conversation, captions on or off, on which device when on another of the user's): hang it up, interrupt Ta, turn captions on or off with app_do from any page. You work it as the user's shadow, as they talk with you (in a call too): they see your cursor do each step. part: one part only. apart: true reads Ta's own screen instead. To change data without the screen, app_run first.",
|
|
642
|
+
parameters: { type: "object", properties: { part: { type: "string" }, apart: APART } }
|
|
435
643
|
},
|
|
436
644
|
{
|
|
437
645
|
name: "app_go",
|
|
438
646
|
label: "Go on the screen",
|
|
439
|
-
description:
|
|
440
|
-
parameters: { type: "object", properties: { to: { type: "string" },
|
|
647
|
+
description: "Goes to a place on the user's screen, as they would with the navigation, and says what is there then, as app_look: a place of app_look (list:..., page:<id>, discover, settings, settings:<section>) by its id or title, or a conversation (chat:<its name>). A call goes on while their screen shows something else. apart: true goes on Ta's own screen.",
|
|
648
|
+
parameters: { type: "object", properties: { to: { type: "string" }, apart: APART }, required: ["to"] }
|
|
441
649
|
},
|
|
442
650
|
{
|
|
443
651
|
name: "app_do",
|
|
444
652
|
label: "Do on the screen",
|
|
445
|
-
description:
|
|
653
|
+
description: `Does something on the user's screen, as they would, their button pressed under your cursor, and says what was done and what changed since you looked (the whole screen when it is another). id: of the last app_look ([p1.2] a row, [p1] a part, [e3] a control); or, without a look first, find: its words as the screen shows them ("Captions"), with to: a place to go to first, as app_go (a step less, quicker in a call; when not found, you get the screen to choose by id). action: what of a part (open, pin, send, answer...; a row opens without), none to click a control; a text box takes input.text (input.submit: true presses Enter), a choice input.text. What cannot be undone waits for the user to agree on their screen. scroll, in place of action, scrolls the part id names ([p1]), or what app_look says scrolls without an id: "down" or "up" most of a screen (one screen a call at most: call again for more), "top" or "bottom" to an end, {"item": "p1.7"} a row of the last look, {"find": "words"} a row by its words (one drawn but out of sight too; one not drawn yet comes as you scroll toward it); it says where it is then and the rows that came into sight. A game (a part \`game-<n>\`: "Breakout — running · canvas 390×678 · score 12") is played by its actions, x and y from 0 to 1 across and down its canvas: point {x, y} taps there (to start, a button, a cell of a board); move {x, y?} puts the pointer there (a paddle follows it; y kept without), with drag: true drags it there pressed; key {key} presses ArrowLeft, ArrowRight, ArrowUp, ArrowDown, Space, Enter, Escape, a letter or a digit; hold {ms, x, y} or {ms, key} presses that long (30 to 5000 ms: to charge a jump). Each call is one step and answers with the game's state (its score, when it tells it): play in steps, not frame by frame. apart: true does it on Ta's own screen, where an action marked = app_run runs that action without the screen unless show: true.`,
|
|
446
654
|
parameters: {
|
|
447
655
|
type: "object",
|
|
448
|
-
properties: {
|
|
449
|
-
|
|
656
|
+
properties: {
|
|
657
|
+
id: { type: "string" },
|
|
658
|
+
find: { type: "string" },
|
|
659
|
+
to: { type: "string" },
|
|
660
|
+
action: { type: "string" },
|
|
661
|
+
scroll: {
|
|
662
|
+
anyOf: [
|
|
663
|
+
{ type: "string", enum: SCROLL_STEPS },
|
|
664
|
+
{ type: "object", properties: { item: { type: "string" }, find: { type: "string" } } }
|
|
665
|
+
]
|
|
666
|
+
},
|
|
667
|
+
input: { type: "object" },
|
|
668
|
+
show: { type: "boolean" },
|
|
669
|
+
apart: APART
|
|
670
|
+
}
|
|
450
671
|
}
|
|
451
672
|
}
|
|
452
673
|
];
|
|
@@ -466,6 +687,11 @@ function activate(pi) {
|
|
|
466
687
|
}
|
|
467
688
|
}
|
|
468
689
|
});
|
|
690
|
+
const screenTools = SCREEN_TOOLS.map((tool) => tool.name);
|
|
691
|
+
pi.on("context", (event) => {
|
|
692
|
+
const messages = ScreenText.forget(event.messages, screenTools);
|
|
693
|
+
return messages ? { messages } : undefined;
|
|
694
|
+
});
|
|
469
695
|
pi.on("tool_call", (event, ctx) => others(ctx) && ours(event.toolName) ? { block: true, reason: "Here you speak with other people: what is the user's is not yours to change." } : undefined);
|
|
470
696
|
const tools = [
|
|
471
697
|
{
|
|
@@ -497,14 +723,17 @@ function activate(pi) {
|
|
|
497
723
|
run: async (app, params, ctx) => {
|
|
498
724
|
const kind = tool.name === "app_look" ? "look" : tool.name === "app_go" ? "go" : "do";
|
|
499
725
|
const input = object(params.input);
|
|
726
|
+
const scroll = kind === "do" ? AppUseUtil.scroll(params.scroll) : undefined;
|
|
500
727
|
return app.screen({
|
|
501
728
|
kind,
|
|
502
729
|
...typeof params.part === "string" && params.part ? { part: params.part } : {},
|
|
503
730
|
...typeof params.to === "string" ? { to: params.to } : {},
|
|
504
|
-
...typeof params.id === "string" ? { id: params.id } : {},
|
|
505
|
-
...typeof params.
|
|
731
|
+
...typeof params.id === "string" && params.id ? { id: params.id } : {},
|
|
732
|
+
...typeof params.find === "string" && params.find.trim() ? { find: params.find.trim() } : {},
|
|
733
|
+
...typeof params.action === "string" && params.action && !scroll ? { action: params.action } : {},
|
|
734
|
+
...scroll ? { scroll } : {},
|
|
506
735
|
...input ? { input } : {},
|
|
507
|
-
...params.
|
|
736
|
+
...params.apart === true ? { apart: true } : {},
|
|
508
737
|
...params.show === true ? { show: true } : {}
|
|
509
738
|
}, ctx.sessionManager?.getSessionFile());
|
|
510
739
|
}
|
package/host/index.mjs
CHANGED
|
@@ -4,6 +4,7 @@ var DONE_KEY = "done";
|
|
|
4
4
|
var DONE_MAX = 20;
|
|
5
5
|
|
|
6
6
|
// web-src/app-util.ts
|
|
7
|
+
var SCROLL_STEPS = ["down", "up", "top", "bottom"];
|
|
7
8
|
var DATA_CHARS = 8000;
|
|
8
9
|
var AppUseUtil;
|
|
9
10
|
((AppUseUtil) => {
|
|
@@ -44,6 +45,25 @@ var AppUseUtil;
|
|
|
44
45
|
`);
|
|
45
46
|
}
|
|
46
47
|
AppUseUtil.resultText = resultText;
|
|
48
|
+
function scroll(value) {
|
|
49
|
+
if (value === undefined || value === null || value === "") {
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (typeof value === "string" && SCROLL_STEPS.includes(value.trim().toLowerCase())) {
|
|
53
|
+
return value.trim().toLowerCase();
|
|
54
|
+
}
|
|
55
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
56
|
+
const { item, find: find2 } = value;
|
|
57
|
+
if (typeof item === "string" && item.trim() && find2 === undefined) {
|
|
58
|
+
return { item: item.trim() };
|
|
59
|
+
}
|
|
60
|
+
if (typeof find2 === "string" && find2.trim() && item === undefined) {
|
|
61
|
+
return { find: find2.trim() };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
throw new Error(`scroll takes ${SCROLL_STEPS.join(", ")}, { "item": "p1.7" } or { "find": "words of a row" }, not ${JSON.stringify(value)}.`);
|
|
65
|
+
}
|
|
66
|
+
AppUseUtil.scroll = scroll;
|
|
47
67
|
function add(done, entry) {
|
|
48
68
|
return [...entries(done), entry].slice(-DONE_MAX);
|
|
49
69
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celljs/ta-app-use",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.19.0",
|
|
4
4
|
"description": "A pi package for Ta: agents do to Ta what the user does with its pages, through the catalog of actions of ta serve (settings, conversations, extensions, what waits for the user) with a notice and an Undo, or on a screen of Ta's own with a cursor, as the user would",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package"
|
|
@@ -27,8 +27,8 @@
|
|
|
27
27
|
"watch": "tsc -b -w --preserveWatchOutput"
|
|
28
28
|
},
|
|
29
29
|
"devDependencies": {
|
|
30
|
-
"@celljs/ta-protocol": "4.
|
|
31
|
-
"@celljs/ta-web": "4.
|
|
30
|
+
"@celljs/ta-protocol": "4.19.0",
|
|
31
|
+
"@celljs/ta-web": "4.19.0"
|
|
32
32
|
},
|
|
33
33
|
"publishConfig": {
|
|
34
34
|
"access": "public"
|