@beeos-ai/device-agent 0.4.3 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/.pi/extensions/android-tools.ts +146 -15
  2. package/dist/adapters/agent-gateway-client.d.ts +37 -0
  3. package/dist/adapters/agent-gateway-client.js +45 -0
  4. package/dist/adapters/agent-gateway-client.js.map +1 -1
  5. package/dist/adapters/agentcore-pi.d.ts +22 -0
  6. package/dist/adapters/agentcore-pi.js +48 -0
  7. package/dist/adapters/agentcore-pi.js.map +1 -1
  8. package/dist/adapters/device-android-adb.d.ts +47 -0
  9. package/dist/adapters/device-android-adb.js +50 -0
  10. package/dist/adapters/device-android-adb.js.map +1 -1
  11. package/dist/adapters/device-desktop.d.ts +12 -0
  12. package/dist/adapters/device-desktop.js +31 -0
  13. package/dist/adapters/device-desktop.js.map +1 -1
  14. package/dist/adapters/identity-file.d.ts +33 -5
  15. package/dist/adapters/identity-file.js +52 -6
  16. package/dist/adapters/identity-file.js.map +1 -1
  17. package/dist/agent-runtime/adapters/base.d.ts +20 -0
  18. package/dist/agent-runtime/adapters/base.js.map +1 -1
  19. package/dist/agent-runtime/agent-runner.d.ts +4 -2
  20. package/dist/agent-runtime/agent-runner.js.map +1 -1
  21. package/dist/agent-runtime/core.d.ts +60 -0
  22. package/dist/agent-runtime/core.js +190 -9
  23. package/dist/agent-runtime/core.js.map +1 -1
  24. package/dist/agent-runtime/feedback.d.ts +56 -0
  25. package/dist/agent-runtime/feedback.js +54 -0
  26. package/dist/agent-runtime/feedback.js.map +1 -0
  27. package/dist/agent-runtime/index.d.ts +1 -0
  28. package/dist/agent-runtime/index.js +1 -0
  29. package/dist/agent-runtime/index.js.map +1 -1
  30. package/dist/agent-runtime/plan-executor.js +24 -0
  31. package/dist/agent-runtime/plan-executor.js.map +1 -1
  32. package/dist/agent-runtime/planner/base.d.ts +10 -0
  33. package/dist/agent-runtime/planner/base.js.map +1 -1
  34. package/dist/agent-runtime/planner/llm-planner.js +1 -0
  35. package/dist/agent-runtime/planner/llm-planner.js.map +1 -1
  36. package/dist/agent-runtime/planner/prompts.d.ts +3 -0
  37. package/dist/agent-runtime/planner/prompts.js +12 -0
  38. package/dist/agent-runtime/planner/prompts.js.map +1 -1
  39. package/dist/agent-runtime/reflection.d.ts +25 -6
  40. package/dist/agent-runtime/reflection.js +64 -22
  41. package/dist/agent-runtime/reflection.js.map +1 -1
  42. package/dist/agent-runtime/reflectors/policy.d.ts +10 -4
  43. package/dist/agent-runtime/reflectors/policy.js +19 -6
  44. package/dist/agent-runtime/reflectors/policy.js.map +1 -1
  45. package/dist/agent-runtime/shortcuts/fs-store.d.ts +41 -0
  46. package/dist/agent-runtime/shortcuts/fs-store.js +122 -0
  47. package/dist/agent-runtime/shortcuts/fs-store.js.map +1 -0
  48. package/dist/agent-runtime/shortcuts/in-memory-store.d.ts +20 -0
  49. package/dist/agent-runtime/shortcuts/in-memory-store.js +64 -0
  50. package/dist/agent-runtime/shortcuts/in-memory-store.js.map +1 -0
  51. package/dist/agent-runtime/shortcuts/index.d.ts +10 -0
  52. package/dist/agent-runtime/shortcuts/index.js +10 -0
  53. package/dist/agent-runtime/shortcuts/index.js.map +1 -0
  54. package/dist/agent-runtime/shortcuts/types.d.ts +69 -0
  55. package/dist/agent-runtime/shortcuts/types.js +17 -0
  56. package/dist/agent-runtime/shortcuts/types.js.map +1 -0
  57. package/dist/app/agent-host.d.ts +83 -0
  58. package/dist/app/agent-host.js +345 -32
  59. package/dist/app/agent-host.js.map +1 -1
  60. package/dist/app/session-manager.d.ts +33 -4
  61. package/dist/app/session-manager.js +27 -5
  62. package/dist/app/session-manager.js.map +1 -1
  63. package/dist/bootstrap.js +187 -3
  64. package/dist/bootstrap.js.map +1 -1
  65. package/dist/config/loader.js +22 -0
  66. package/dist/config/loader.js.map +1 -1
  67. package/dist/config/schema.d.ts +18 -0
  68. package/dist/config/schema.js +118 -0
  69. package/dist/config/schema.js.map +1 -1
  70. package/dist/fleet/adapters/process/node-process-manager.js +97 -17
  71. package/dist/fleet/adapters/process/node-process-manager.js.map +1 -1
  72. package/dist/fleet/adapters/state/beeos-cli-store.d.ts +75 -9
  73. package/dist/fleet/adapters/state/beeos-cli-store.js +201 -17
  74. package/dist/fleet/adapters/state/beeos-cli-store.js.map +1 -1
  75. package/dist/fleet/domain/device.d.ts +9 -0
  76. package/dist/fleet/domain/device.js.map +1 -1
  77. package/dist/fleet/domain/fsm.js +34 -0
  78. package/dist/fleet/domain/fsm.js.map +1 -1
  79. package/dist/fleet/installer/spawn-env.d.ts +3 -0
  80. package/dist/fleet/installer/spawn-env.js +186 -3
  81. package/dist/fleet/installer/spawn-env.js.map +1 -1
  82. package/dist/fleet/state/state-json.d.ts +116 -0
  83. package/dist/fleet/state/state-json.js +87 -0
  84. package/dist/fleet/state/state-json.js.map +1 -0
  85. package/dist/fleet/supervisor.js +14 -2
  86. package/dist/fleet/supervisor.js.map +1 -1
  87. package/dist/inbound/acp/handlers/session.js.map +1 -1
  88. package/dist/mcp-client/client.d.ts +6 -0
  89. package/dist/mcp-client/client.js +11 -0
  90. package/dist/mcp-client/client.js.map +1 -1
  91. package/dist/mcp-client/runtime.d.ts +13 -0
  92. package/dist/orchestrator-system-prompt.d.ts +26 -0
  93. package/dist/orchestrator-system-prompt.js +118 -2
  94. package/dist/orchestrator-system-prompt.js.map +1 -1
  95. package/dist/pi-context.d.ts +50 -0
  96. package/dist/pi-context.js.map +1 -1
  97. package/dist/ports/device.d.ts +18 -0
  98. package/dist/resources/augmenter.d.ts +48 -0
  99. package/dist/resources/augmenter.js +126 -0
  100. package/dist/resources/augmenter.js.map +1 -0
  101. package/dist/resources/cache.d.ts +57 -0
  102. package/dist/resources/cache.js +107 -0
  103. package/dist/resources/cache.js.map +1 -0
  104. package/dist/resources/index.d.ts +17 -0
  105. package/dist/resources/index.js +15 -0
  106. package/dist/resources/index.js.map +1 -0
  107. package/dist/resources/lifecycle/disk-pressure.d.ts +75 -0
  108. package/dist/resources/lifecycle/disk-pressure.js +148 -0
  109. package/dist/resources/lifecycle/disk-pressure.js.map +1 -0
  110. package/dist/resources/lifecycle/index.d.ts +7 -0
  111. package/dist/resources/lifecycle/index.js +7 -0
  112. package/dist/resources/lifecycle/index.js.map +1 -0
  113. package/dist/resources/lifecycle/manager.d.ts +38 -0
  114. package/dist/resources/lifecycle/manager.js +205 -0
  115. package/dist/resources/lifecycle/manager.js.map +1 -0
  116. package/dist/resources/lifecycle/sweeper.d.ts +36 -0
  117. package/dist/resources/lifecycle/sweeper.js +179 -0
  118. package/dist/resources/lifecycle/sweeper.js.map +1 -0
  119. package/dist/resources/lifecycle/types.d.ts +180 -0
  120. package/dist/resources/lifecycle/types.js +46 -0
  121. package/dist/resources/lifecycle/types.js.map +1 -0
  122. package/dist/resources/parsers.d.ts +28 -0
  123. package/dist/resources/parsers.js +88 -0
  124. package/dist/resources/parsers.js.map +1 -0
  125. package/dist/resources/registry.d.ts +47 -0
  126. package/dist/resources/registry.js +107 -0
  127. package/dist/resources/registry.js.map +1 -0
  128. package/dist/resources/resolvers/beeos-file-resolver.d.ts +62 -0
  129. package/dist/resources/resolvers/beeos-file-resolver.js +109 -0
  130. package/dist/resources/resolvers/beeos-file-resolver.js.map +1 -0
  131. package/dist/resources/resolvers/errors.d.ts +11 -0
  132. package/dist/resources/resolvers/errors.js +23 -0
  133. package/dist/resources/resolvers/errors.js.map +1 -0
  134. package/dist/resources/resolvers/mime-table.d.ts +41 -0
  135. package/dist/resources/resolvers/mime-table.js +92 -0
  136. package/dist/resources/resolvers/mime-table.js.map +1 -0
  137. package/dist/resources/resolvers/url-resolver.d.ts +85 -0
  138. package/dist/resources/resolvers/url-resolver.js +495 -0
  139. package/dist/resources/resolvers/url-resolver.js.map +1 -0
  140. package/dist/resources/shortcut-facade.d.ts +51 -0
  141. package/dist/resources/shortcut-facade.js +72 -0
  142. package/dist/resources/shortcut-facade.js.map +1 -0
  143. package/dist/resources/telemetry.d.ts +52 -0
  144. package/dist/resources/telemetry.js +59 -0
  145. package/dist/resources/telemetry.js.map +1 -0
  146. package/dist/resources/types.d.ts +249 -0
  147. package/dist/resources/types.js +29 -0
  148. package/dist/resources/types.js.map +1 -0
  149. package/dist/shortcuts/builtins/download-to-device.d.ts +83 -0
  150. package/dist/shortcuts/builtins/download-to-device.js +125 -0
  151. package/dist/shortcuts/builtins/download-to-device.js.map +1 -0
  152. package/dist/shortcuts/builtins/find-app-package.d.ts +79 -0
  153. package/dist/shortcuts/builtins/find-app-package.js +168 -0
  154. package/dist/shortcuts/builtins/find-app-package.js.map +1 -0
  155. package/dist/shortcuts/builtins/index.d.ts +32 -0
  156. package/dist/shortcuts/builtins/index.js +37 -0
  157. package/dist/shortcuts/builtins/index.js.map +1 -0
  158. package/dist/shortcuts/builtins/open-deeplink.d.ts +68 -0
  159. package/dist/shortcuts/builtins/open-deeplink.js +142 -0
  160. package/dist/shortcuts/builtins/open-deeplink.js.map +1 -0
  161. package/dist/shortcuts/builtins/set-clipboard.d.ts +65 -0
  162. package/dist/shortcuts/builtins/set-clipboard.js +112 -0
  163. package/dist/shortcuts/builtins/set-clipboard.js.map +1 -0
  164. package/dist/shortcuts/index.d.ts +11 -0
  165. package/dist/shortcuts/index.js +11 -0
  166. package/dist/shortcuts/index.js.map +1 -0
  167. package/dist/shortcuts/registry.d.ts +56 -0
  168. package/dist/shortcuts/registry.js +70 -0
  169. package/dist/shortcuts/registry.js.map +1 -0
  170. package/dist/shortcuts/router.d.ts +91 -0
  171. package/dist/shortcuts/router.js +158 -0
  172. package/dist/shortcuts/router.js.map +1 -0
  173. package/dist/shortcuts/types.d.ts +219 -0
  174. package/dist/shortcuts/types.js +46 -0
  175. package/dist/shortcuts/types.js.map +1 -0
  176. package/package.json +3 -3
  177. package/dist/agent-runtime/skills/sqlite-store.d.ts +0 -60
  178. package/dist/agent-runtime/skills/sqlite-store.js +0 -234
  179. package/dist/agent-runtime/skills/sqlite-store.js.map +0 -1
  180. package/dist/test-lan-autoglm.d.ts +0 -7
  181. package/dist/test-lan-autoglm.js +0 -46
  182. package/dist/test-lan-autoglm.js.map +0 -1
  183. package/dist/test-mem-breakdown.d.ts +0 -1
  184. package/dist/test-mem-breakdown.js +0 -103
  185. package/dist/test-mem-breakdown.js.map +0 -1
  186. package/dist/test-mem.d.ts +0 -1
  187. package/dist/test-mem.js +0 -60
  188. package/dist/test-mem.js.map +0 -1
  189. package/dist/test-mobile-use.d.ts +0 -1
  190. package/dist/test-mobile-use.js +0 -18
  191. package/dist/test-mobile-use.js.map +0 -1
  192. package/dist/test-screenshot-loop.d.ts +0 -1
  193. package/dist/test-screenshot-loop.js +0 -43
  194. package/dist/test-screenshot-loop.js.map +0 -1
  195. package/dist/test-screenshot.d.ts +0 -1
  196. package/dist/test-screenshot.js +0 -10
  197. package/dist/test-screenshot.js.map +0 -1
  198. package/dist/test-ui-xml-dump.d.ts +0 -15
  199. package/dist/test-ui-xml-dump.js +0 -49
  200. package/dist/test-ui-xml-dump.js.map +0 -1
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Resource Resolution Layer — telemetry shim.
3
+ *
4
+ * The device-agent already ships a structured pino logger (see
5
+ * `logger/index.ts`). We deliberately do NOT pull in `prom-client` to
6
+ * keep the bundle small; instead we emit one structured log event per
7
+ * resolution outcome that an external Promtail / Vector pipeline can
8
+ * count + histogram.
9
+ *
10
+ * Event shape (counter-friendly):
11
+ *
12
+ * {
13
+ * service: "device-agent",
14
+ * event: "resource.resolution",
15
+ * outcome: "success" | "fetch_error" | …,
16
+ * mime: "image/jpeg" | …,
17
+ * sizeBytes: number,
18
+ * durationMs: number,
19
+ * sourceUrl: "https://example.com/x.jpg"
20
+ * }
21
+ *
22
+ * The outcome label vocabulary is closed (see `ResolveOutcomeLabel` in
23
+ * `types.ts`) so a Prometheus exporter can rely on bounded cardinality.
24
+ *
25
+ * All entry points accept any pino-like logger so unit tests can pass
26
+ * a minimal stub.
27
+ */
28
+ import type { ResolveOutcomeLabel } from "./types.js";
29
+ export interface TelemetryLogger {
30
+ info(obj: Record<string, unknown>, msg?: string): void;
31
+ warn(obj: Record<string, unknown>, msg?: string): void;
32
+ }
33
+ export interface RecordResolutionOpts {
34
+ outcome: ResolveOutcomeLabel;
35
+ mime?: string;
36
+ sizeBytes?: number;
37
+ durationMs: number;
38
+ sourceUrl?: string;
39
+ /** Free-form error message for failed outcomes (NOT a counter label). */
40
+ error?: string;
41
+ }
42
+ /**
43
+ * Emit one structured log line corresponding to a single resolution
44
+ * outcome. The downstream log pipeline aggregates these into:
45
+ *
46
+ * - counter `device_agent_resource_resolution_total{outcome,mime}`
47
+ * - histogram `device_agent_resource_resolution_duration_ms{mime}`
48
+ *
49
+ * The `event` field is kept stable (`resource.resolution`) so log-based
50
+ * counters don't drift across versions.
51
+ */
52
+ export declare function recordResolution(log: TelemetryLogger, opts: RecordResolutionOpts): void;
@@ -0,0 +1,59 @@
1
+ /**
2
+ * Resource Resolution Layer — telemetry shim.
3
+ *
4
+ * The device-agent already ships a structured pino logger (see
5
+ * `logger/index.ts`). We deliberately do NOT pull in `prom-client` to
6
+ * keep the bundle small; instead we emit one structured log event per
7
+ * resolution outcome that an external Promtail / Vector pipeline can
8
+ * count + histogram.
9
+ *
10
+ * Event shape (counter-friendly):
11
+ *
12
+ * {
13
+ * service: "device-agent",
14
+ * event: "resource.resolution",
15
+ * outcome: "success" | "fetch_error" | …,
16
+ * mime: "image/jpeg" | …,
17
+ * sizeBytes: number,
18
+ * durationMs: number,
19
+ * sourceUrl: "https://example.com/x.jpg"
20
+ * }
21
+ *
22
+ * The outcome label vocabulary is closed (see `ResolveOutcomeLabel` in
23
+ * `types.ts`) so a Prometheus exporter can rely on bounded cardinality.
24
+ *
25
+ * All entry points accept any pino-like logger so unit tests can pass
26
+ * a minimal stub.
27
+ */
28
+ /**
29
+ * Emit one structured log line corresponding to a single resolution
30
+ * outcome. The downstream log pipeline aggregates these into:
31
+ *
32
+ * - counter `device_agent_resource_resolution_total{outcome,mime}`
33
+ * - histogram `device_agent_resource_resolution_duration_ms{mime}`
34
+ *
35
+ * The `event` field is kept stable (`resource.resolution`) so log-based
36
+ * counters don't drift across versions.
37
+ */
38
+ export function recordResolution(log, opts) {
39
+ const fields = {
40
+ event: "resource.resolution",
41
+ outcome: opts.outcome,
42
+ durationMs: opts.durationMs,
43
+ };
44
+ if (opts.mime)
45
+ fields.mime = opts.mime;
46
+ if (opts.sizeBytes !== undefined)
47
+ fields.sizeBytes = opts.sizeBytes;
48
+ if (opts.sourceUrl)
49
+ fields.sourceUrl = opts.sourceUrl;
50
+ if (opts.error)
51
+ fields.error = opts.error;
52
+ if (opts.outcome === "success" || opts.outcome === "cache_hit") {
53
+ log.info(fields, "resource.resolution");
54
+ }
55
+ else {
56
+ log.warn(fields, "resource.resolution");
57
+ }
58
+ }
59
+ //# sourceMappingURL=telemetry.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../../src/resources/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAmBH;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAC9B,GAAoB,EACpB,IAA0B;IAE1B,MAAM,MAAM,GAA4B;QACtC,KAAK,EAAE,qBAAqB;QAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,UAAU,EAAE,IAAI,CAAC,UAAU;KAC5B,CAAC;IACF,IAAI,IAAI,CAAC,IAAI;QAAE,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACvC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;QAAE,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IACpE,IAAI,IAAI,CAAC,SAAS;QAAE,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IACtD,IAAI,IAAI,CAAC,KAAK;QAAE,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAC1C,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,WAAW,EAAE,CAAC;QAC/D,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAC1C,CAAC;AACH,CAAC"}
@@ -0,0 +1,249 @@
1
+ /**
2
+ * Resource Resolution Layer — type contracts.
3
+ *
4
+ * Background: AutoGLM-Phone-9B (and other open-source mobile VLMs we ship)
5
+ * are pure-vision agents — they cannot fetch URLs, write files, or run
6
+ * shell commands. When a user task references a non-visual resource
7
+ * (image URL, video URL, PDF, APK), the prompt has to be **resolved**
8
+ * to a device-local path before the VLM ever sees it. This module
9
+ * defines the protocol for that resolution.
10
+ *
11
+ * Architectural seam: this layer runs inside `AgentHost.submitTask`,
12
+ * BEFORE `IAgentCore.run` is called and AFTER `task:start` has been
13
+ * emitted (so cancel-mid-resolution works). Concrete resolvers
14
+ * consume an `IDevice` (for `resolveWellKnownDir(mime)`) and an
15
+ * `IMcpRuntime` (for `file_write` / `execute_command`) without
16
+ * importing them directly — both are passed in via `ResolverContext`.
17
+ *
18
+ * Industry references that shaped this design:
19
+ * - MAS-Bench (arxiv 2509.06477v2): "GUI-Shortcut Hybrid Paradigm"
20
+ * — establishes that non-GUI resource preparation is a first-class
21
+ * orthogonal concern to GUI execution.
22
+ * - Anthropic Computer Use: tool-calling LLMs treat `bash` and
23
+ * `computer` symmetrically; we mirror that by treating
24
+ * `resolve_resource` as a real, observable tool event.
25
+ * - droidrun / mobilerun's `AndroidDriver`: GUI-only — confirms our
26
+ * gap analysis.
27
+ */
28
+ import type { IDevice } from "../ports/device.js";
29
+ import type { IMcpRuntime } from "../mcp-client/runtime.js";
30
+ /**
31
+ * A reference to a resource the agent task mentions but cannot consume
32
+ * directly. Discriminated union so we can grow the surface (file://,
33
+ * embedded base64 blob, RFC 3986 data URLs, geo coordinates) without
34
+ * breaking existing resolvers.
35
+ *
36
+ * Phase 1 implements `url` only; the others are typed placeholders so
37
+ * unit tests can pin the shape and downstream code can pattern-match
38
+ * exhaustively now (TS compiler will surface any addition).
39
+ */
40
+ export type ResourceRef = {
41
+ kind: "url";
42
+ url: string;
43
+ } | {
44
+ kind: "file";
45
+ path: string;
46
+ } | {
47
+ kind: "text";
48
+ text: string;
49
+ } | {
50
+ kind: "coordinate";
51
+ lat: number;
52
+ lon: number;
53
+ }
54
+ /**
55
+ * BeeOS-uploaded file ref. The web UI emits ACP `resource_link` blocks
56
+ * with a `beeos-file://<fileId>` URI for every attachment the user
57
+ * uploads via the chat input. The `BeeosFileResolver` exchanges the
58
+ * fileId for a short-lived presigned download URL via the Agent
59
+ * Gateway and then delegates the actual fetch/push to the same
60
+ * pipeline used by `UrlResolver`. `hintMime` / `hintName` are the
61
+ * client-supplied metadata from the `resource_link` block; the
62
+ * resolver still treats the server's `Content-Type` as authoritative.
63
+ */
64
+ | {
65
+ kind: "beeos-file";
66
+ fileId: string;
67
+ hintMime?: string;
68
+ hintName?: string;
69
+ };
70
+ /**
71
+ * What the resolver returns and what the augmenter consumes.
72
+ *
73
+ * Stable contract: every successful resolution MUST populate every
74
+ * required field. `scannedToMediaStore` is platform-specific (Android
75
+ * media-scanner) and is allowed to be `false` on platforms where the
76
+ * concept does not exist (desktop) — that does NOT count as a failure.
77
+ */
78
+ export interface ResolvedAsset {
79
+ /** Source ref (echoed back so the augmenter can render the URL). */
80
+ ref: ResourceRef;
81
+ /** Device-local absolute path (e.g. `/sdcard/Pictures/beeos_…jpg`). */
82
+ remotePath: string;
83
+ /** Validated content type (image/jpeg, video/mp4, …). */
84
+ mime: string;
85
+ /** Bytes written to the device. */
86
+ sizeBytes: number;
87
+ /** SHA-256 of the resource bytes (lowercase hex). */
88
+ sha256: string;
89
+ /** True iff the platform supports + acknowledged a media-scan trigger. */
90
+ scannedToMediaStore: boolean;
91
+ /** Echo of the original URL for `url` refs (empty for non-url refs). */
92
+ sourceUrl: string;
93
+ /** When the resolution completed (ms since epoch, host clock). */
94
+ resolvedAt: number;
95
+ }
96
+ /**
97
+ * Side-effects + capabilities a resolver may consume. The host owns
98
+ * construction so resolvers stay testable without spinning up a real
99
+ * MCP server or device.
100
+ */
101
+ export interface ResolverContext {
102
+ /** Active device — for `resolveWellKnownDir(mime)`. */
103
+ readonly device: IDevice;
104
+ /** Active MCP runtime — for `file_write` / `execute_command`. */
105
+ readonly mcp: IMcpRuntime;
106
+ /** Merge-cancel signal (host's controller + caller's signal). */
107
+ readonly abortSignal: AbortSignal;
108
+ /** Hard cap on bytes per resource. Resolvers MUST honor this. */
109
+ readonly maxBytes: number;
110
+ /** Per-resolve fetch / push timeout. Resolvers MUST honor this. */
111
+ readonly timeoutMs: number;
112
+ /** Allow-listed URL schemes (`["http", "https"]` in production). */
113
+ readonly allowedSchemes: readonly string[];
114
+ /** Whether to reject RFC1918 / loopback / link-local IPs. */
115
+ readonly denyPrivateIp: boolean;
116
+ /**
117
+ * Optional emitter for fine-grained progress events. Resolvers
118
+ * SHOULD call this during long downloads (start, periodic update,
119
+ * end) so the AgentHost can surface them as `task:tool` updates.
120
+ */
121
+ readonly emitProgress?: (ev: ResolveProgressEvent) => void;
122
+ /** Structured logger (correlation-aware). */
123
+ readonly log: ResolverLogger;
124
+ /**
125
+ * Optional disk-pressure guard (Phase 1.5 / lifecycle L3). When
126
+ * present, the resolver calls `assertCapacity(...)` immediately
127
+ * before `mcp.fileWrite`; the guard may free space via a force-
128
+ * sweep or reject the resolve with `disk_pressure`. When absent
129
+ * (tests, lab, or `lifecycle.mode=off`), the resolver behaves
130
+ * exactly as it did pre-1.5.
131
+ */
132
+ readonly diskPressure?: DiskPressureGuard;
133
+ }
134
+ /**
135
+ * Phase 1.5 disk-pressure guard contract — declared here (alongside
136
+ * `ResolverContext`) so the resolver and the lifecycle module share
137
+ * a single source of truth without `lifecycle/*` having to import
138
+ * resolver internals.
139
+ *
140
+ * Implementation lives at `resources/lifecycle/disk-pressure.ts`.
141
+ */
142
+ export interface DiskPressureGuard {
143
+ /**
144
+ * Inspect available space on the sandbox volume and either:
145
+ * 1. return silently (free space is comfortable),
146
+ * 2. trigger a force-sweep and return silently (now there's room), or
147
+ * 3. throw a `ResolverError("disk_pressure", ...)` (still tight
148
+ * after sweeping) so the resolver fails fast and the host
149
+ * records a structured outcome.
150
+ */
151
+ assertCapacity(args: DiskPressureCheckArgs): Promise<void>;
152
+ }
153
+ export interface DiskPressureCheckArgs {
154
+ /** MCP runtime (`executeCommand` is required for `df`/`find`). */
155
+ readonly mcp: IMcpRuntime;
156
+ /**
157
+ * Sandbox directory the resolver is about to write into. Used as
158
+ * both the `df` target (to read the volume's free bytes) and the
159
+ * sweep root (to evict the oldest files when we need to free up
160
+ * space).
161
+ */
162
+ readonly sandboxDir: string;
163
+ /**
164
+ * Bytes the resolver is about to write. The guard treats `free -
165
+ * incomingBytes` as the post-write headroom estimate.
166
+ */
167
+ readonly incomingBytes: number;
168
+ /** Structured logger (correlation-aware). */
169
+ readonly log: ResolverLogger;
170
+ }
171
+ /**
172
+ * Minimal log surface — the host hands a child pino instance with the
173
+ * `taskId` correlation already attached. Kept generic so unit tests can
174
+ * pass a stub.
175
+ */
176
+ export interface ResolverLogger {
177
+ debug(obj: Record<string, unknown>, msg?: string): void;
178
+ info(obj: Record<string, unknown>, msg?: string): void;
179
+ warn(obj: Record<string, unknown>, msg?: string): void;
180
+ error(obj: Record<string, unknown>, msg?: string): void;
181
+ }
182
+ /**
183
+ * Progress events for the AgentHost telemetry stream. Mapped 1:1 to
184
+ * `task:tool` events with `name = "resolve_resource"` and lifecycle
185
+ * `start | update | end`.
186
+ */
187
+ export type ResolveProgressEvent = {
188
+ phase: "start";
189
+ ref: ResourceRef;
190
+ } | {
191
+ phase: "update";
192
+ ref: ResourceRef;
193
+ downloadedBytes: number;
194
+ totalBytes?: number;
195
+ } | {
196
+ phase: "end";
197
+ ref: ResourceRef;
198
+ asset?: ResolvedAsset;
199
+ error?: Error;
200
+ };
201
+ /**
202
+ * Resolver port — every concrete resolver (URL, file://, …) implements
203
+ * this interface. `match` is intentionally separate from `resolve` so
204
+ * the registry can dispatch without invoking side effects.
205
+ */
206
+ export interface ResourceResolver {
207
+ /** Free-form name for logging / metrics ("url", "file", …). */
208
+ readonly name: string;
209
+ /** Cheap predicate: does this resolver own the supplied ref? */
210
+ match(ref: ResourceRef): boolean;
211
+ /** Heavy work: download / push / scan. Throws on failure. */
212
+ resolve(ref: ResourceRef, ctx: ResolverContext): Promise<ResolvedAsset>;
213
+ }
214
+ /**
215
+ * Parser port — extracts `ResourceRef`s from a free-form task string.
216
+ * The host iterates every registered parser and unions the results.
217
+ *
218
+ * Phase 1 ships a single `UrlParser` that finds http(s) URLs anywhere
219
+ * in the task. Future parsers may extract `file://` paths, base64
220
+ * data URLs, or geo coordinates.
221
+ */
222
+ export interface ResourceParser {
223
+ readonly name: string;
224
+ parse(task: string): ResourceRef[];
225
+ }
226
+ /**
227
+ * Augmenter port — turns the original task + the list of resolved
228
+ * assets into the final string we hand to `IAgentCore.run`.
229
+ *
230
+ * Implementations MUST:
231
+ * - preserve the original task verbatim (the VLM's reasoning depends
232
+ * on the human-supplied wording).
233
+ * - state the source-URL ↔ device-path identity so the agent never
234
+ * confuses two visually similar resources.
235
+ * - NOT prescribe any GUI path (no "tap the first photo" hints —
236
+ * that's the VLM's job).
237
+ */
238
+ export interface TaskContextAugmenter {
239
+ augment(originalTask: string, assets: readonly ResolvedAsset[]): string;
240
+ }
241
+ /**
242
+ * Telemetry label for the resource-resolution counter. Kept here so
243
+ * adapters and tests reference one symbolic source of truth.
244
+ *
245
+ * Why an enum-like type union instead of a free-form string: Prometheus
246
+ * cardinality discipline. We must not let arbitrary error strings leak
247
+ * into label values.
248
+ */
249
+ export type ResolveOutcomeLabel = "success" | "fetch_error" | "push_error" | "scanner_failed" | "disallowed_scheme" | "oversized" | "timeout" | "private_ip_blocked" | "dns_rebind_blocked" | "disallowed_mime" | "disk_pressure" | "cache_hit" | "cache_validation_miss" | "aborted";
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Resource Resolution Layer — type contracts.
3
+ *
4
+ * Background: AutoGLM-Phone-9B (and other open-source mobile VLMs we ship)
5
+ * are pure-vision agents — they cannot fetch URLs, write files, or run
6
+ * shell commands. When a user task references a non-visual resource
7
+ * (image URL, video URL, PDF, APK), the prompt has to be **resolved**
8
+ * to a device-local path before the VLM ever sees it. This module
9
+ * defines the protocol for that resolution.
10
+ *
11
+ * Architectural seam: this layer runs inside `AgentHost.submitTask`,
12
+ * BEFORE `IAgentCore.run` is called and AFTER `task:start` has been
13
+ * emitted (so cancel-mid-resolution works). Concrete resolvers
14
+ * consume an `IDevice` (for `resolveWellKnownDir(mime)`) and an
15
+ * `IMcpRuntime` (for `file_write` / `execute_command`) without
16
+ * importing them directly — both are passed in via `ResolverContext`.
17
+ *
18
+ * Industry references that shaped this design:
19
+ * - MAS-Bench (arxiv 2509.06477v2): "GUI-Shortcut Hybrid Paradigm"
20
+ * — establishes that non-GUI resource preparation is a first-class
21
+ * orthogonal concern to GUI execution.
22
+ * - Anthropic Computer Use: tool-calling LLMs treat `bash` and
23
+ * `computer` symmetrically; we mirror that by treating
24
+ * `resolve_resource` as a real, observable tool event.
25
+ * - droidrun / mobilerun's `AndroidDriver`: GUI-only — confirms our
26
+ * gap analysis.
27
+ */
28
+ export {};
29
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/resources/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG"}
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Shortcut: `download_to_device`
3
+ *
4
+ * Download an HTTP(S) URL onto the connected device using the same
5
+ * `ResourceResolver` backend that the AgentHost auto-trigger
6
+ * (Phase 1) uses.
7
+ *
8
+ * Why a *second* entry point for the resolver:
9
+ *
10
+ * D8 — phase 1 wires resolution into `AgentHost.submitTask` so the
11
+ * *initial* prompt's URLs are pre-fetched before the VLM ever sees
12
+ * them. But the orchestrator may also encounter URLs **mid-task**:
13
+ *
14
+ * - the user attaches a follow-up message with another image
15
+ * - the GUI flow asks the agent to copy a link the agent just
16
+ * extracted from a screen
17
+ * - a shortcut chain (`open_deeplink → screenshot → ...`) needs
18
+ * to refer to the resolved file path
19
+ *
20
+ * Without `download_to_device`, the LLM would be stuck — it has no
21
+ * `bash` and no `file_write`. The shortcut closes that gap by
22
+ * exposing the same resolver under an LLM-callable name. Both
23
+ * call sites share the **same** `ResourceResolutionCache` so a
24
+ * download primed by either path is reused by the other. (The
25
+ * bootstrap wires the `ShortcutResourceFacade` against the same
26
+ * registry instance the `AgentHostImpl` holds — see
27
+ * `bootstrap.ts`.)
28
+ *
29
+ * Design constraints:
30
+ *
31
+ * - No LLM-visible flag for SSRF / size cap / timeout — they all
32
+ * come from the same env-var feature flags as the auto path
33
+ * (`RESOURCE_RESOLUTION_*`), so the security posture is identical
34
+ * across both surfaces. The LLM cannot weaken it.
35
+ * - No `mime` / `path` knob: the resolver picks the well-known dir
36
+ * by Content-Type. Letting the LLM pick a destination would be a
37
+ * path-traversal hazard.
38
+ *
39
+ * Failure mapping:
40
+ *
41
+ * - The underlying resolver throws `ResolverError` with codes from
42
+ * `ResolverErrorCode`. We translate them into shortcut-error
43
+ * codes the orchestrator already understands. See
44
+ * `mapResolverError`.
45
+ *
46
+ * Platform: any (image / video / audio / document downloads work on
47
+ * desktop too; APK is rejected by `IDevice.resolveWellKnownDir` on
48
+ * desktop with `unsupported_platform`).
49
+ */
50
+ import type { Shortcut, ShortcutContext, ShortcutResult } from "../types.js";
51
+ interface DownloadArgs {
52
+ url: string;
53
+ }
54
+ interface DownloadOutput {
55
+ remotePath: string;
56
+ mime: string;
57
+ sizeBytes: number;
58
+ sha256: string;
59
+ sourceUrl: string;
60
+ }
61
+ export declare class DownloadToDeviceShortcut implements Shortcut<DownloadArgs, DownloadOutput> {
62
+ readonly descriptor: {
63
+ id: string;
64
+ displayName: string;
65
+ description: string;
66
+ jsonSchema: {
67
+ type: string;
68
+ properties: {
69
+ url: {
70
+ type: string;
71
+ description: string;
72
+ };
73
+ };
74
+ required: string[];
75
+ additionalProperties: boolean;
76
+ };
77
+ kind: "api";
78
+ platforms: readonly ["any"];
79
+ timeoutMs: number;
80
+ };
81
+ execute(args: DownloadArgs, ctx: ShortcutContext): Promise<ShortcutResult<DownloadOutput>>;
82
+ }
83
+ export {};
@@ -0,0 +1,125 @@
1
+ /**
2
+ * Shortcut: `download_to_device`
3
+ *
4
+ * Download an HTTP(S) URL onto the connected device using the same
5
+ * `ResourceResolver` backend that the AgentHost auto-trigger
6
+ * (Phase 1) uses.
7
+ *
8
+ * Why a *second* entry point for the resolver:
9
+ *
10
+ * D8 — phase 1 wires resolution into `AgentHost.submitTask` so the
11
+ * *initial* prompt's URLs are pre-fetched before the VLM ever sees
12
+ * them. But the orchestrator may also encounter URLs **mid-task**:
13
+ *
14
+ * - the user attaches a follow-up message with another image
15
+ * - the GUI flow asks the agent to copy a link the agent just
16
+ * extracted from a screen
17
+ * - a shortcut chain (`open_deeplink → screenshot → ...`) needs
18
+ * to refer to the resolved file path
19
+ *
20
+ * Without `download_to_device`, the LLM would be stuck — it has no
21
+ * `bash` and no `file_write`. The shortcut closes that gap by
22
+ * exposing the same resolver under an LLM-callable name. Both
23
+ * call sites share the **same** `ResourceResolutionCache` so a
24
+ * download primed by either path is reused by the other. (The
25
+ * bootstrap wires the `ShortcutResourceFacade` against the same
26
+ * registry instance the `AgentHostImpl` holds — see
27
+ * `bootstrap.ts`.)
28
+ *
29
+ * Design constraints:
30
+ *
31
+ * - No LLM-visible flag for SSRF / size cap / timeout — they all
32
+ * come from the same env-var feature flags as the auto path
33
+ * (`RESOURCE_RESOLUTION_*`), so the security posture is identical
34
+ * across both surfaces. The LLM cannot weaken it.
35
+ * - No `mime` / `path` knob: the resolver picks the well-known dir
36
+ * by Content-Type. Letting the LLM pick a destination would be a
37
+ * path-traversal hazard.
38
+ *
39
+ * Failure mapping:
40
+ *
41
+ * - The underlying resolver throws `ResolverError` with codes from
42
+ * `ResolverErrorCode`. We translate them into shortcut-error
43
+ * codes the orchestrator already understands. See
44
+ * `mapResolverError`.
45
+ *
46
+ * Platform: any (image / video / audio / document downloads work on
47
+ * desktop too; APK is rejected by `IDevice.resolveWellKnownDir` on
48
+ * desktop with `unsupported_platform`).
49
+ */
50
+ export class DownloadToDeviceShortcut {
51
+ descriptor = {
52
+ id: "download_to_device",
53
+ displayName: "Download to Device",
54
+ description: "Fetch an HTTP(S) URL and place the resulting file on the device " +
55
+ "in the well-known directory matching its Content-Type (image/*→Pictures, " +
56
+ "video/*→Movies, audio/*→Music, application/*→Download). Returns " +
57
+ "the absolute device path the file was written to plus its MIME / " +
58
+ "size / SHA-256. Re-uses the same cache as the host-side auto " +
59
+ "trigger so duplicate URLs are deduplicated.",
60
+ jsonSchema: {
61
+ type: "object",
62
+ properties: {
63
+ url: {
64
+ type: "string",
65
+ description: "HTTPS URL to fetch (HTTP allowed for development).",
66
+ },
67
+ },
68
+ required: ["url"],
69
+ additionalProperties: false,
70
+ },
71
+ kind: "api",
72
+ platforms: ["any"],
73
+ timeoutMs: 60_000,
74
+ };
75
+ async execute(args, ctx) {
76
+ if (typeof args?.url !== "string" || args.url.length === 0) {
77
+ return { ok: false, code: "invalid_args", message: "url must be a non-empty string" };
78
+ }
79
+ if (!ctx.resources || typeof ctx.resources.resolveUrl !== "function") {
80
+ return {
81
+ ok: false,
82
+ code: "permission_denied",
83
+ message: "shortcut runtime has no resource resolver attached",
84
+ };
85
+ }
86
+ try {
87
+ const out = await ctx.resources.resolveUrl(args.url);
88
+ return { ok: true, output: out };
89
+ }
90
+ catch (e) {
91
+ return mapResolverError(e);
92
+ }
93
+ }
94
+ }
95
+ /**
96
+ * Translate `ResolverError`-shaped failures into the shortcut error
97
+ * vocabulary. We deliberately don't import `ResolverError` (avoids a
98
+ * dependency from `shortcuts/builtins` into `resources/resolvers`) —
99
+ * we duck-type on `code`.
100
+ */
101
+ function mapResolverError(e) {
102
+ const err = e;
103
+ const code = err?.code ?? "";
104
+ const message = err?.message ?? String(e);
105
+ switch (code) {
106
+ case "disallowed_scheme":
107
+ case "disallowed_mime":
108
+ case "oversized":
109
+ return { ok: false, code: "invalid_args", message };
110
+ case "private_ip_blocked":
111
+ case "dns_rebind_blocked":
112
+ return { ok: false, code: "permission_denied", message };
113
+ case "timeout":
114
+ case "aborted":
115
+ return { ok: false, code: "timeout", message };
116
+ case "fetch_error":
117
+ return { ok: false, code: "intent_rejected", message };
118
+ case "push_error":
119
+ case "scanner_failed":
120
+ return { ok: false, code: "internal_error", message };
121
+ default:
122
+ return { ok: false, code: "internal_error", message };
123
+ }
124
+ }
125
+ //# sourceMappingURL=download-to-device.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"download-to-device.js","sourceRoot":"","sources":["../../../src/shortcuts/builtins/download-to-device.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAgBH,MAAM,OAAO,wBAAwB;IAG1B,UAAU,GAAG;QACpB,EAAE,EAAE,oBAAoB;QACxB,WAAW,EAAE,oBAAoB;QACjC,WAAW,EACT,kEAAkE;YAClE,2EAA2E;YAC3E,kEAAkE;YAClE,mEAAmE;YACnE,+DAA+D;YAC/D,6CAA6C;QAC/C,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,oDAAoD;iBAClE;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;YACjB,oBAAoB,EAAE,KAAK;SAC5B;QACD,IAAI,EAAE,KAAc;QACpB,SAAS,EAAE,CAAC,KAAK,CAAU;QAC3B,SAAS,EAAE,MAAM;KAClB,CAAC;IAEF,KAAK,CAAC,OAAO,CACX,IAAkB,EAClB,GAAoB;QAEpB,IAAI,OAAO,IAAI,EAAE,GAAG,KAAK,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3D,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;QACxF,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YACrE,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,IAAI,EAAE,mBAAmB;gBACzB,OAAO,EAAE,oDAAoD;aAC9D,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;CACF;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CACvB,CAAU;IAEV,MAAM,GAAG,GAAG,CAAuD,CAAC;IACpE,MAAM,IAAI,GAAG,GAAG,EAAE,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,mBAAmB,CAAC;QACzB,KAAK,iBAAiB,CAAC;QACvB,KAAK,WAAW;YACd,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;QACtD,KAAK,oBAAoB,CAAC;QAC1B,KAAK,oBAAoB;YACvB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,mBAAmB,EAAE,OAAO,EAAE,CAAC;QAC3D,KAAK,SAAS,CAAC;QACf,KAAK,SAAS;YACZ,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;QACjD,KAAK,aAAa;YAChB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC;QACzD,KAAK,YAAY,CAAC;QAClB,KAAK,gBAAgB;YACnB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC;QACxD;YACE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC;IAC1D,CAAC;AACH,CAAC"}