@hasna/computer 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/loop.d.ts.map +1 -1
- package/dist/cli/index.js +10411 -252
- package/dist/db/agents.d.ts +26 -0
- package/dist/db/agents.d.ts.map +1 -0
- package/dist/db/index.d.ts +4 -3
- package/dist/db/index.d.ts.map +1 -1
- package/dist/drivers/mac/accessibility.d.ts +33 -0
- package/dist/drivers/mac/accessibility.d.ts.map +1 -0
- package/dist/drivers/mac/index.d.ts +7 -1
- package/dist/drivers/mac/index.d.ts.map +1 -1
- package/dist/drivers/mac/screenshot.d.ts +1 -1
- package/dist/drivers/mac/screenshot.d.ts.map +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9586 -223
- package/dist/lib/integrations.d.ts +30 -0
- package/dist/lib/integrations.d.ts.map +1 -0
- package/dist/lib/terminal-image.d.ts +29 -0
- package/dist/lib/terminal-image.d.ts.map +1 -0
- package/dist/mcp/index.js +10294 -230
- package/dist/server/index.js +9329 -181
- package/dist/types/index.d.ts +5 -0
- package/dist/types/index.d.ts.map +1 -1
- package/helpers/accessibility +0 -0
- package/helpers/accessibility.swift +161 -0
- package/package.json +4 -1
- package/src/db/migrations/001_initial.sql +57 -0
package/dist/types/index.d.ts
CHANGED
|
@@ -128,6 +128,7 @@ export interface Session {
|
|
|
128
128
|
total_tokens_in: number;
|
|
129
129
|
total_tokens_out: number;
|
|
130
130
|
total_duration_ms: number;
|
|
131
|
+
tags?: string[];
|
|
131
132
|
error?: string;
|
|
132
133
|
created_at: string;
|
|
133
134
|
completed_at?: string;
|
|
@@ -152,6 +153,10 @@ export interface RunOptions {
|
|
|
152
153
|
screenshotMaxWidth?: number;
|
|
153
154
|
/** Dry-run mode — model plans actions but they are not executed */
|
|
154
155
|
dryRun?: boolean;
|
|
156
|
+
/** Tags for this session */
|
|
157
|
+
tags?: string[];
|
|
158
|
+
/** Display number to capture (1=main, 2=secondary, etc.) */
|
|
159
|
+
displayNumber?: number;
|
|
155
160
|
/** Callback for each step */
|
|
156
161
|
onStep?: (step: number, response: ModelResponse, result: ActionResult) => void;
|
|
157
162
|
/** Callback when done */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,8CAA8C;AAC9C,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE9C,yBAAyB;AACzB,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEtD,0BAA0B;AAC1B,MAAM,WAAW,KAAK;IACpB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX;AAED,wBAAwB;AACxB,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,4BAA4B;AAC5B,MAAM,WAAW,UAAU;IACzB,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,IAAI,EAAE,UAAU,CAAC;IACjB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,+CAA+C;AAC/C,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,GACtB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,KAAK,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC7B;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAChE;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,KAAK,CAAC;IAAC,EAAE,EAAE,KAAK,CAAA;CAAE,GACxC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAC5B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvC,0CAA0C;AAC1C,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,sEAAsE;AACtE,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;IAClC,wBAAwB;IACxB,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACrD,yBAAyB;IACzB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC5B,yDAAyD;IACzD,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,IAAI,EAAE,OAAO,CAAC;IACd,gCAAgC;IAChC,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AAED,mEAAmE;AACnE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,wDAAwD;IACxD,OAAO,CAAC,MAAM,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,UAAU,CAAC;QACvB,OAAO,EAAE,aAAa,EAAE,CAAC;QACzB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;CAC5B;AAED,oBAAoB;AACpB,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAExF,uCAAuC;AACvC,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,6BAA6B;AAC7B,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,iCAAiC;AACjC,MAAM,WAAW,UAAU;IACzB,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,yBAAyB;IACzB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,4BAA4B;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,2EAA2E;IAC3E,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,mEAAmE;IACnE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,6BAA6B;IAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,KAAK,IAAI,CAAC;IAC/E,yBAAyB;IACzB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACpC,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,2BAA2B;AAC3B,MAAM,WAAW,YAAY;IAC3B,gDAAgD;IAChD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,gDAAgD;IAChD,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,oDAAoD;IACpD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iCAAiC;IACjC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,wCAAwC;IACxC,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,8CAA8C;AAC9C,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE9C,yBAAyB;AACzB,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEtD,0BAA0B;AAC1B,MAAM,WAAW,KAAK;IACpB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX;AAED,wBAAwB;AACxB,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,4BAA4B;AAC5B,MAAM,WAAW,UAAU;IACzB,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,IAAI,EAAE,UAAU,CAAC;IACjB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,+CAA+C;AAC/C,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,GACtB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,KAAK,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC7B;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAChE;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,KAAK,CAAC;IAAC,EAAE,EAAE,KAAK,CAAA;CAAE,GACxC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAC5B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvC,0CAA0C;AAC1C,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,sEAAsE;AACtE,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;IAClC,wBAAwB;IACxB,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACrD,yBAAyB;IACzB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC5B,yDAAyD;IACzD,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,IAAI,EAAE,OAAO,CAAC;IACd,gCAAgC;IAChC,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AAED,mEAAmE;AACnE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,wDAAwD;IACxD,OAAO,CAAC,MAAM,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,UAAU,CAAC;QACvB,OAAO,EAAE,aAAa,EAAE,CAAC;QACzB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;CAC5B;AAED,oBAAoB;AACpB,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAExF,uCAAuC;AACvC,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,6BAA6B;AAC7B,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,iCAAiC;AACjC,MAAM,WAAW,UAAU;IACzB,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,yBAAyB;IACzB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,4BAA4B;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,2EAA2E;IAC3E,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,mEAAmE;IACnE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,4BAA4B;IAC5B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,6BAA6B;IAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,KAAK,IAAI,CAAC;IAC/E,yBAAyB;IACzB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACpC,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,2BAA2B;AAC3B,MAAM,WAAW,YAAY;IAC3B,gDAAgD;IAChD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,gDAAgD;IAChD,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,oDAAoD;IACpD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iCAAiC;IACjC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,wCAAwC;IACxC,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B"}
|
|
Binary file
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
#!/usr/bin/env swift
|
|
2
|
+
// accessibility.swift — Query macOS Accessibility tree (AXUIElement)
|
|
3
|
+
// Usage: accessibility [--app <name>] [--focused] [--depth <n>]
|
|
4
|
+
// Output: JSON array of UI elements with role, title, position, size
|
|
5
|
+
//
|
|
6
|
+
// Requires: Accessibility permissions in System Settings
|
|
7
|
+
|
|
8
|
+
import AppKit
|
|
9
|
+
import Foundation
|
|
10
|
+
|
|
11
|
+
struct UIElement: Codable {
|
|
12
|
+
let role: String?
|
|
13
|
+
let title: String?
|
|
14
|
+
let value: String?
|
|
15
|
+
let label: String?
|
|
16
|
+
let x: Int
|
|
17
|
+
let y: Int
|
|
18
|
+
let width: Int
|
|
19
|
+
let height: Int
|
|
20
|
+
let enabled: Bool
|
|
21
|
+
let focused: Bool
|
|
22
|
+
let children: Int
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
func getAXValue<T>(_ element: AXUIElement, _ attribute: String) -> T? {
|
|
26
|
+
var value: AnyObject?
|
|
27
|
+
let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value)
|
|
28
|
+
guard result == .success else { return nil }
|
|
29
|
+
return value as? T
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
func getPosition(_ element: AXUIElement) -> CGPoint? {
|
|
33
|
+
var value: AnyObject?
|
|
34
|
+
let result = AXUIElementCopyAttributeValue(element, kAXPositionAttribute as String as CFString, &value)
|
|
35
|
+
guard result == .success, let val = value else { return nil }
|
|
36
|
+
var point = CGPoint.zero
|
|
37
|
+
AXValueGetValue(val as! AXValue, .cgPoint, &point)
|
|
38
|
+
return point
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
func getSize(_ element: AXUIElement) -> CGSize? {
|
|
42
|
+
var value: AnyObject?
|
|
43
|
+
let result = AXUIElementCopyAttributeValue(element, kAXSizeAttribute as String as CFString, &value)
|
|
44
|
+
guard result == .success, let val = value else { return nil }
|
|
45
|
+
var size = CGSize.zero
|
|
46
|
+
AXValueGetValue(val as! AXValue, .cgSize, &size)
|
|
47
|
+
return size
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
func getChildren(_ element: AXUIElement) -> [AXUIElement] {
|
|
51
|
+
var value: AnyObject?
|
|
52
|
+
let result = AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as String as CFString, &value)
|
|
53
|
+
guard result == .success, let children = value as? [AXUIElement] else { return [] }
|
|
54
|
+
return children
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
func elementToStruct(_ element: AXUIElement) -> UIElement {
|
|
58
|
+
let role: String? = getAXValue(element, kAXRoleAttribute as String)
|
|
59
|
+
let title: String? = getAXValue(element, kAXTitleAttribute as String)
|
|
60
|
+
let value: String? = getAXValue(element, kAXValueAttribute as String)
|
|
61
|
+
let label: String? = getAXValue(element, kAXDescriptionAttribute as String)
|
|
62
|
+
let enabled: Bool = getAXValue(element, kAXEnabledAttribute as String) ?? true
|
|
63
|
+
let focused: Bool = getAXValue(element, kAXFocusedAttribute as String) ?? false
|
|
64
|
+
|
|
65
|
+
let pos = getPosition(element) ?? CGPoint.zero
|
|
66
|
+
let sz = getSize(element) ?? CGSize.zero
|
|
67
|
+
let children = getChildren(element)
|
|
68
|
+
|
|
69
|
+
return UIElement(
|
|
70
|
+
role: role,
|
|
71
|
+
title: title,
|
|
72
|
+
value: (value?.count ?? 0) > 200 ? String(value!.prefix(200)) + "..." : value,
|
|
73
|
+
label: label,
|
|
74
|
+
x: Int(pos.x),
|
|
75
|
+
y: Int(pos.y),
|
|
76
|
+
width: Int(sz.width),
|
|
77
|
+
height: Int(sz.height),
|
|
78
|
+
enabled: enabled,
|
|
79
|
+
focused: focused,
|
|
80
|
+
children: children.count
|
|
81
|
+
)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
func walkTree(_ element: AXUIElement, depth: Int, maxDepth: Int) -> [UIElement] {
|
|
85
|
+
var results: [UIElement] = []
|
|
86
|
+
let el = elementToStruct(element)
|
|
87
|
+
|
|
88
|
+
// Skip tiny/invisible elements
|
|
89
|
+
if el.width > 5 && el.height > 5 {
|
|
90
|
+
results.append(el)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if depth < maxDepth {
|
|
94
|
+
for child in getChildren(element) {
|
|
95
|
+
results.append(contentsOf: walkTree(child, depth: depth + 1, maxDepth: maxDepth))
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return results
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Parse arguments
|
|
103
|
+
var appName: String? = nil
|
|
104
|
+
var focusedOnly = false
|
|
105
|
+
var maxDepth = 3
|
|
106
|
+
|
|
107
|
+
var args = Array(CommandLine.arguments.dropFirst())
|
|
108
|
+
while !args.isEmpty {
|
|
109
|
+
let arg = args.removeFirst()
|
|
110
|
+
switch arg {
|
|
111
|
+
case "--app":
|
|
112
|
+
if !args.isEmpty { appName = args.removeFirst() }
|
|
113
|
+
case "--focused":
|
|
114
|
+
focusedOnly = true
|
|
115
|
+
case "--depth":
|
|
116
|
+
if !args.isEmpty { maxDepth = Int(args.removeFirst()) ?? 3 }
|
|
117
|
+
default:
|
|
118
|
+
break
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Get target application
|
|
123
|
+
var targetApp: NSRunningApplication?
|
|
124
|
+
if let name = appName {
|
|
125
|
+
targetApp = NSWorkspace.shared.runningApplications.first {
|
|
126
|
+
$0.localizedName?.lowercased() == name.lowercased()
|
|
127
|
+
}
|
|
128
|
+
if targetApp == nil {
|
|
129
|
+
fputs("Error: Application '\(name)' not found\n", stderr)
|
|
130
|
+
exit(1)
|
|
131
|
+
}
|
|
132
|
+
} else {
|
|
133
|
+
targetApp = NSWorkspace.shared.frontmostApplication
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
guard let app = targetApp, let pid = Optional(app.processIdentifier) else {
|
|
137
|
+
fputs("Error: No application found\n", stderr)
|
|
138
|
+
exit(1)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
let appElement = AXUIElementCreateApplication(pid)
|
|
142
|
+
|
|
143
|
+
var elements: [UIElement]
|
|
144
|
+
if focusedOnly {
|
|
145
|
+
var focusedElement: AnyObject?
|
|
146
|
+
AXUIElementCopyAttributeValue(appElement, kAXFocusedUIElementAttribute as CFString, &focusedElement)
|
|
147
|
+
if let focused = focusedElement {
|
|
148
|
+
elements = walkTree(focused as! AXUIElement, depth: 0, maxDepth: maxDepth)
|
|
149
|
+
} else {
|
|
150
|
+
elements = []
|
|
151
|
+
}
|
|
152
|
+
} else {
|
|
153
|
+
elements = walkTree(appElement, depth: 0, maxDepth: maxDepth)
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Output as JSON
|
|
157
|
+
let encoder = JSONEncoder()
|
|
158
|
+
encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
|
|
159
|
+
if let data = try? encoder.encode(elements) {
|
|
160
|
+
print(String(data: data, encoding: .utf8)!)
|
|
161
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hasna/computer",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Open-source computer use for AI agents — control your Mac with Anthropic or OpenAI. CLI + MCP server + REST API + Dashboard.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -20,6 +20,9 @@
|
|
|
20
20
|
"dist",
|
|
21
21
|
"helpers/scroll",
|
|
22
22
|
"helpers/scroll.swift",
|
|
23
|
+
"helpers/accessibility",
|
|
24
|
+
"helpers/accessibility.swift",
|
|
25
|
+
"src/db/migrations",
|
|
23
26
|
"dashboard/dist",
|
|
24
27
|
"LICENSE",
|
|
25
28
|
"README.md"
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
-- PostgreSQL migration: 001_initial
|
|
2
|
+
-- Creates sessions and action_logs tables for @hasna/computer
|
|
3
|
+
|
|
4
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
5
|
+
id TEXT PRIMARY KEY,
|
|
6
|
+
task TEXT NOT NULL,
|
|
7
|
+
provider TEXT NOT NULL,
|
|
8
|
+
model TEXT NOT NULL,
|
|
9
|
+
status TEXT NOT NULL DEFAULT 'running',
|
|
10
|
+
steps INTEGER NOT NULL DEFAULT 0,
|
|
11
|
+
total_tokens_in INTEGER NOT NULL DEFAULT 0,
|
|
12
|
+
total_tokens_out INTEGER NOT NULL DEFAULT 0,
|
|
13
|
+
total_duration_ms INTEGER NOT NULL DEFAULT 0,
|
|
14
|
+
error TEXT,
|
|
15
|
+
created_at TEXT NOT NULL,
|
|
16
|
+
completed_at TEXT
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
CREATE TABLE IF NOT EXISTS action_logs (
|
|
20
|
+
id SERIAL PRIMARY KEY,
|
|
21
|
+
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
|
|
22
|
+
step INTEGER NOT NULL,
|
|
23
|
+
action_type TEXT NOT NULL,
|
|
24
|
+
action_data JSONB NOT NULL,
|
|
25
|
+
reasoning TEXT,
|
|
26
|
+
screenshot_path TEXT,
|
|
27
|
+
success BOOLEAN NOT NULL DEFAULT TRUE,
|
|
28
|
+
error TEXT,
|
|
29
|
+
duration_ms INTEGER NOT NULL DEFAULT 0,
|
|
30
|
+
tokens_in INTEGER,
|
|
31
|
+
tokens_out INTEGER,
|
|
32
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
CREATE INDEX IF NOT EXISTS idx_action_logs_session ON action_logs(session_id);
|
|
36
|
+
CREATE INDEX IF NOT EXISTS idx_sessions_status ON sessions(status);
|
|
37
|
+
CREATE INDEX IF NOT EXISTS idx_sessions_created ON sessions(created_at);
|
|
38
|
+
|
|
39
|
+
-- PostgreSQL full-text search (using tsvector instead of FTS5)
|
|
40
|
+
ALTER TABLE sessions ADD COLUMN IF NOT EXISTS task_tsv TSVECTOR
|
|
41
|
+
GENERATED ALWAYS AS (to_tsvector('english', task)) STORED;
|
|
42
|
+
CREATE INDEX IF NOT EXISTS idx_sessions_fts ON sessions USING GIN(task_tsv);
|
|
43
|
+
|
|
44
|
+
ALTER TABLE action_logs ADD COLUMN IF NOT EXISTS reasoning_tsv TSVECTOR
|
|
45
|
+
GENERATED ALWAYS AS (to_tsvector('english', COALESCE(reasoning, ''))) STORED;
|
|
46
|
+
CREATE INDEX IF NOT EXISTS idx_action_logs_fts ON action_logs USING GIN(reasoning_tsv);
|
|
47
|
+
|
|
48
|
+
-- Feedback table (for @hasna/cloud feedback system)
|
|
49
|
+
CREATE TABLE IF NOT EXISTS feedback (
|
|
50
|
+
id TEXT PRIMARY KEY,
|
|
51
|
+
service TEXT NOT NULL DEFAULT 'computer',
|
|
52
|
+
version TEXT,
|
|
53
|
+
message TEXT NOT NULL,
|
|
54
|
+
email TEXT,
|
|
55
|
+
machine_id TEXT,
|
|
56
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
57
|
+
);
|