@chalky/chalky 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -6
- package/{app → agent}/SYSTEM.md +16 -11
- package/agent/extensions/app/README.md +68 -0
- package/{app/extensions/ui → agent/extensions/app}/agents.js +22 -16
- package/agent/extensions/app/guide.md +5 -0
- package/agent/extensions/app/index.ts +66 -0
- package/{app/extensions/ui/skills/agent-apps → agent/extensions/app/skills/app-agents}/SKILL.md +17 -18
- package/agent/extensions/board/browser.js +359 -0
- package/agent/extensions/board/guide.md +4 -0
- package/agent/extensions/board/index.ts +100 -0
- package/agent/extensions/card/README.md +88 -0
- package/agent/extensions/card/blocks/Button.js +34 -0
- package/{app/extensions/ui/components → agent/extensions/card/blocks}/Column.js +2 -2
- package/agent/extensions/card/blocks/Markdown.js +37 -0
- package/{app/extensions/ui/components/Card.js → agent/extensions/card/blocks/Panel.js} +3 -3
- package/{app/extensions/ui/components → agent/extensions/card/blocks}/Row.js +2 -2
- package/{app/extensions/ui/components → agent/extensions/card/blocks}/TextField.js +3 -3
- package/{app/extensions/ui → agent/extensions/card}/browser.js +155 -128
- package/agent/extensions/card/guide.md +13 -0
- package/agent/extensions/card/index.ts +370 -0
- package/agent/extensions/command/browser.js +13 -0
- package/agent/extensions/command/guide.md +2 -0
- package/agent/extensions/command/index.ts +94 -0
- package/agent/extensions/doodle/README.md +57 -0
- package/agent/extensions/doodle/browser.js +1013 -0
- package/agent/extensions/doodle/guide.md +22 -0
- package/agent/extensions/doodle/index.ts +274 -0
- package/agent/extensions/file/browser.js +143 -0
- package/agent/extensions/file/guide.md +5 -0
- package/{app/extensions/files → agent/extensions/file}/index.ts +34 -16
- package/{app/extensions/frames → agent/extensions/frame}/README.md +13 -13
- package/{app/extensions/frames → agent/extensions/frame}/browser.js +24 -19
- package/agent/extensions/frame/guide.md +3 -0
- package/agent/extensions/frame/index.ts +81 -0
- package/{app/extensions/ui_history → agent/extensions/history}/browser.js +18 -18
- package/{app/extensions/ui_history → agent/extensions/history}/index.ts +32 -33
- package/{app → agent}/extensions/mcp/browser.js +1 -1
- package/agent/extensions/mcp/guide.md +3 -0
- package/{app → agent}/extensions/mcp/index.ts +2 -4
- package/agent/extensions/model/browser.js +363 -0
- package/{app → agent}/extensions/model/index.ts +8 -4
- package/agent/extensions/model/login.ts +215 -0
- package/agent/extensions/note/README.md +35 -0
- package/agent/extensions/note/browser.js +356 -0
- package/agent/extensions/note/guide.md +8 -0
- package/agent/extensions/note/index.ts +192 -0
- package/{app → agent}/extensions/page/index.ts +2 -2
- package/{app → agent}/extensions/page/page.html +55 -49
- package/agent/extensions/prompt/index.ts +38 -0
- package/{app → agent}/extensions/reload/index.ts +3 -3
- package/{app → agent}/extensions/screen/browser.js +4 -4
- package/{app → agent}/extensions/screen/index.ts +11 -11
- package/{app/extensions/steps → agent/extensions/step}/index.ts +1 -1
- package/{app/extensions/swarm → agent/extensions/team}/README.md +19 -19
- package/{app/extensions/swarm → agent/extensions/team}/guide.md +4 -4
- package/{app/extensions/swarm → agent/extensions/team}/index.ts +41 -66
- package/{app/extensions/versions → agent/extensions/version}/browser.js +10 -10
- package/{app/extensions/versions → agent/extensions/version}/index.ts +39 -54
- package/{app → agent}/extensions/viewer/README.md +4 -4
- package/{app → agent}/extensions/viewer/browser.js +8 -8
- package/agent/extensions/viewer/guide.md +3 -0
- package/{app → agent}/extensions/viewer/index.ts +22 -17
- package/agent/extensions/voice/browser.js +72 -0
- package/agent/extensions/voice/index.ts +39 -0
- package/{app/extensions/ui → agent/extensions/voice}/voice.js +5 -5
- package/agent/settings.json +8 -0
- package/{app/extensions/ui/skills/ui-check → agent/skills/browser-check}/SKILL.md +4 -4
- package/{app/extensions/ui/skills/ui-check → agent/skills/browser-check}/scripts/check.mjs +1 -1
- package/{app/extensions/ui/skills/ui-design → agent/skills/design}/SKILL.md +14 -14
- package/agent/skills/extension-map/SKILL.md +56 -0
- package/{app → agent}/skills/write-skill/SKILL.md +1 -1
- package/bin/chalky.mjs +3 -3
- package/package.json +8 -8
- package/server.ts +162 -63
- package/app/extensions/appdata/README.md +0 -10
- package/app/extensions/appdata/guide.md +0 -4
- package/app/extensions/appdata/index.ts +0 -47
- package/app/extensions/doodle/README.md +0 -15
- package/app/extensions/doodle/browser.js +0 -388
- package/app/extensions/doodle/guide.md +0 -6
- package/app/extensions/doodle/index.ts +0 -54
- package/app/extensions/files/browser.js +0 -96
- package/app/extensions/files/guide.md +0 -7
- package/app/extensions/frames/guide.md +0 -7
- package/app/extensions/frames/index.ts +0 -76
- package/app/extensions/mcp/guide.md +0 -5
- package/app/extensions/point/README.md +0 -22
- package/app/extensions/point/browser.js +0 -245
- package/app/extensions/point/guide.md +0 -6
- package/app/extensions/point/index.ts +0 -97
- package/app/extensions/prompt/index.ts +0 -19
- package/app/extensions/ui/README.md +0 -69
- package/app/extensions/ui/components/App.js +0 -28
- package/app/extensions/ui/components/Button.js +0 -28
- package/app/extensions/ui/components/Markdown.js +0 -37
- package/app/extensions/ui/guide.md +0 -7
- package/app/extensions/ui/index.ts +0 -271
- package/app/extensions/viewer/guide.md +0 -5
- package/app/mcp.json +0 -3
- package/app/skills/plugin-map/SKILL.md +0 -49
- package/app/ui/apps/.keep +0 -0
- /package/{app/extensions/ui/components → agent/extensions/card/blocks}/Table.js +0 -0
- /package/{app/extensions/ui/components → agent/extensions/card/blocks}/Text.js +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// Viewer: previews of the user's files (the ones in the
|
|
2
|
-
//
|
|
1
|
+
// Viewer: previews of the user's files (the ones in the file extension's folder), in a window on the page (the
|
|
2
|
+
// browser half: browser.js). browser.js opens a window when files' browser.js fires
|
|
3
3
|
// "file-open"; the window lives only in that page (a note pointed at it says which file and which spot).
|
|
4
4
|
// Parsing happens here, one library per format (plain previews, no LibreOffice):
|
|
5
5
|
// images, pdf shown by the browser itself, from /viewer/raw
|
|
@@ -12,11 +12,12 @@
|
|
|
12
12
|
// GET /viewer/view?path= {"kind": "image"|"pdf"|"text"|"sheets"|"doc"|"slides"|"none", ...} (see preview())
|
|
13
13
|
// GET /viewer/raw?path= the file's bytes, images and pdf only
|
|
14
14
|
// POST /viewer/shot a png of an area the user drew around on a pdf; saved in SHOTS, returns {"path"}
|
|
15
|
-
// Tool
|
|
15
|
+
// Tool viewer_read(path, offset?): the same content as text, for the agent (pdf text with unpdf). The Python tool was
|
|
16
16
|
// read_file; renamed so it isn't mistaken for pi's read (text and pictures, which the guide now names for pictures).
|
|
17
17
|
import fs from "node:fs";
|
|
18
|
+
import { createRequire } from "node:module";
|
|
18
19
|
import path from "node:path";
|
|
19
|
-
import
|
|
20
|
+
import type { WorkSheet } from "xlsx"; // types only: erased when this runs
|
|
20
21
|
import mammoth from "mammoth";
|
|
21
22
|
import PptxParser from "node-pptx-parser";
|
|
22
23
|
import WordExtractor from "word-extractor";
|
|
@@ -25,13 +26,16 @@ import { Type } from "typebox";
|
|
|
25
26
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
26
27
|
|
|
27
28
|
const host = (globalThis as any).chalky;
|
|
28
|
-
|
|
29
|
+
// xlsx is a bundled dependency: npm keeps it inside our package's node_modules and never moves it up next to the
|
|
30
|
+
// others, so a plain import from this home copy doesn't find it. Look for it from the package's own folder instead.
|
|
31
|
+
const XLSX: typeof import("xlsx") = createRequire(path.join(host.root, "package.json"))("xlsx");
|
|
32
|
+
const DIR = path.join(host.workspace, "drops"); // the file extension's folder
|
|
29
33
|
const IMAGES: Record<string, string> = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif",
|
|
30
34
|
".webp": "image/webp", ".bmp": "image/bmp", ".svg": "image/svg+xml", ".avif": "image/avif", ".ico": "image/x-icon" };
|
|
31
35
|
const MAX_ROWS = 1000, MAX_COLS = 60; // per sheet shown
|
|
32
36
|
const MAX_TEXT = 300_000; // characters shown
|
|
33
|
-
const MAX_TOOL_TEXT = 40_000; // characters
|
|
34
|
-
const SHOTS = path.join(host.
|
|
37
|
+
const MAX_TOOL_TEXT = 40_000; // characters viewer_read returns
|
|
38
|
+
const SHOTS = path.join(host.data, "viewer"); // pictures of drawn-around pdf areas (outside the user's folder)
|
|
35
39
|
const MAX_SHOT = 10 * 1024 * 1024;
|
|
36
40
|
|
|
37
41
|
function inside(p: string | null) { // the real path, if it is a file in DIR; else null (no reading elsewhere)
|
|
@@ -41,7 +45,7 @@ function inside(p: string | null) { // the real path, if it is a file in DIR; e
|
|
|
41
45
|
} catch { return null; }
|
|
42
46
|
}
|
|
43
47
|
|
|
44
|
-
function sheet(name: string, ws:
|
|
48
|
+
function sheet(name: string, ws: WorkSheet) { // {name, rows, cut}: from A1, at most MAX_ROWS x MAX_COLS
|
|
45
49
|
if (!ws["!ref"]) return { name, rows: [], cut: false };
|
|
46
50
|
const r = XLSX.utils.decode_range(ws["!ref"]);
|
|
47
51
|
const cut = r.e.r >= MAX_ROWS || r.e.c >= MAX_COLS;
|
|
@@ -132,7 +136,7 @@ async function preview(p: string) {
|
|
|
132
136
|
return { kind: "text", text: head.subarray(0, n).toString("utf8"), cut: fs.statSync(p).size > MAX_TEXT };
|
|
133
137
|
}
|
|
134
138
|
|
|
135
|
-
async function asText(p: string): Promise<string> { // what
|
|
139
|
+
async function asText(p: string): Promise<string> { // what viewer_read returns: the preview, written out as text
|
|
136
140
|
const v: any = await preview(p);
|
|
137
141
|
if (v.kind === "image") return `${p} is an image: look at it with read.`;
|
|
138
142
|
if (v.kind === "pdf") {
|
|
@@ -188,22 +192,23 @@ export default function (pi: ExtensionAPI) {
|
|
|
188
192
|
|
|
189
193
|
|
|
190
194
|
pi.registerTool({
|
|
191
|
-
name: "
|
|
195
|
+
name: "viewer_read", label: "Viewer read", exposure: "deferred",
|
|
192
196
|
description: "Read a document as text: docx, pptx, xlsx, xls, csv, pdf, doc, or any text file "
|
|
193
197
|
+ "(sheets come back as CSV, slides and pages numbered).",
|
|
194
|
-
parameters: Type.Object({ path: Type.String()
|
|
195
|
-
|
|
198
|
+
parameters: Type.Object({ path: Type.String(),
|
|
199
|
+
offset: Type.Optional(Type.Number({ description: "the character to start from, to read on in a long document" })) }),
|
|
200
|
+
async execute(_id, { path: p, offset = 0 }) {
|
|
196
201
|
let text: string;
|
|
197
202
|
if (!fs.existsSync(p) || !fs.statSync(p).isFile()) text = `error: no file ${p}`;
|
|
198
203
|
else {
|
|
199
|
-
try {
|
|
200
|
-
|
|
204
|
+
try {
|
|
205
|
+
const whole = await asText(p), end = offset + MAX_TOOL_TEXT;
|
|
206
|
+
text = whole.slice(offset, end);
|
|
207
|
+
if (end < whole.length) text += `\n[... cut: ${whole.length} characters in all; call again with offset=${end} for the rest]`;
|
|
208
|
+
} catch (e: any) { text = `error: could not read ${p}: ${e.name}: ${e.message}`; }
|
|
201
209
|
}
|
|
202
210
|
return { content: [{ type: "text", text }], details: undefined };
|
|
203
211
|
},
|
|
204
212
|
});
|
|
205
213
|
|
|
206
|
-
const guide = fs.readFileSync(path.join(import.meta.dirname, "guide.md"), "utf8") // short; README.md: the whole of it, read when it says
|
|
207
|
-
.replace("README.md", path.join(import.meta.dirname, "README.md"));
|
|
208
|
-
pi.on("before_agent_start", (event) => { event.systemPromptOptions.sections.viewer = guide; });
|
|
209
214
|
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// The page side of the voice extension: a microphone button left of the send button. Click to dictate, click again
|
|
2
|
+
// to stop; each finished piece of speech is added to the input (to read, fix and send). /voice/voice.js does the
|
|
3
|
+
// listening: the server's speech to text when it has STT_BASE_URL, else the browser's own recognition.
|
|
4
|
+
// While dictating, the icon is a sound wave whose bars follow the microphone's loudness (its own AnalyserNode, so
|
|
5
|
+
// it works in both modes).
|
|
6
|
+
import voice from "/voice/voice.js";
|
|
7
|
+
|
|
8
|
+
const SVG = '<svg viewBox="0 0 24 24" aria-hidden="true" fill="none" stroke="currentColor" stroke-linecap="round" ' +
|
|
9
|
+
'stroke-linejoin="round">';
|
|
10
|
+
const MIC = SVG + '<rect x="9" y="3" width="6" height="11" rx="3"/><path d="M5 11a7 7 0 0 0 14 0M12 18v3"/></svg>';
|
|
11
|
+
const WAVE = SVG + [4, 8, 12, 16, 20].map((x) => `<line x1="${x}" x2="${x}" y1="10" y2="14"/>`).join("") + "</svg>";
|
|
12
|
+
const SHAPE = [0.45, 0.8, 1, 0.75, 0.5]; // the middle bars reach higher
|
|
13
|
+
|
|
14
|
+
export default function (page) {
|
|
15
|
+
const input = document.getElementById("text");
|
|
16
|
+
const button = document.createElement("button");
|
|
17
|
+
button.type = "button"; // in the form: not a send
|
|
18
|
+
button.className = "mic";
|
|
19
|
+
button.dataset.name = "dictate";
|
|
20
|
+
button.title = "Dictate";
|
|
21
|
+
button.innerHTML = MIC;
|
|
22
|
+
input.after(button);
|
|
23
|
+
const style = document.createElement("style");
|
|
24
|
+
style.textContent = "#f button.mic.on { color: #fff; background: #e5484d; border-color: #e5484d; }";
|
|
25
|
+
document.head.append(style);
|
|
26
|
+
|
|
27
|
+
let on = false, stopWave = null;
|
|
28
|
+
|
|
29
|
+
// the wave: loudness (RMS of the mic's samples) sets the bars' height, each wobbling a little on its own
|
|
30
|
+
async function startWave() {
|
|
31
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
32
|
+
const ctx = new AudioContext();
|
|
33
|
+
const analyser = ctx.createAnalyser();
|
|
34
|
+
ctx.createMediaStreamSource(stream).connect(analyser);
|
|
35
|
+
const samples = new Float32Array(analyser.fftSize);
|
|
36
|
+
button.innerHTML = WAVE;
|
|
37
|
+
const bars = button.querySelectorAll("line");
|
|
38
|
+
let level = 0, frame;
|
|
39
|
+
const draw = (t) => {
|
|
40
|
+
analyser.getFloatTimeDomainData(samples);
|
|
41
|
+
const rms = Math.sqrt(samples.reduce((s, x) => s + x * x, 0) / samples.length);
|
|
42
|
+
level += (Math.min(1, rms * 8) - level) * 0.3; // smoothed: rises and falls over a few frames
|
|
43
|
+
bars.forEach((bar, i) => {
|
|
44
|
+
const half = 1 + 8 * level * SHAPE[i] * (0.7 + 0.3 * Math.sin(t / 90 + i * 1.7));
|
|
45
|
+
bar.setAttribute("y1", 12 - half);
|
|
46
|
+
bar.setAttribute("y2", 12 + half);
|
|
47
|
+
});
|
|
48
|
+
frame = requestAnimationFrame(draw);
|
|
49
|
+
};
|
|
50
|
+
frame = requestAnimationFrame(draw);
|
|
51
|
+
return () => { cancelAnimationFrame(frame); stream.getTracks().forEach((tr) => tr.stop()); ctx.close(); button.innerHTML = MIC; };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const set = (state) => {
|
|
55
|
+
on = state;
|
|
56
|
+
button.classList.toggle("on", on);
|
|
57
|
+
button.title = on ? "Stop dictating" : "Dictate";
|
|
58
|
+
if (!on && stopWave) { stopWave(); stopWave = null; }
|
|
59
|
+
};
|
|
60
|
+
button.addEventListener("click", async () => {
|
|
61
|
+
if (on) return voice.stop();
|
|
62
|
+
set(true);
|
|
63
|
+
startWave().then((stop) => { if (on) stopWave = stop; else stop(); }).catch(() => {}); // no wave: still dictates
|
|
64
|
+
await voice.start((text) => {
|
|
65
|
+
input.value = input.value.trim() ? `${input.value.trimEnd()} ${text}` : text;
|
|
66
|
+
input.focus();
|
|
67
|
+
}, { onState: (state, detail) => {
|
|
68
|
+
if (state === "error") { page.note(`voice: ${detail}`); voice.stop(); set(false); }
|
|
69
|
+
if (state === "off") set(false);
|
|
70
|
+
} });
|
|
71
|
+
});
|
|
72
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// Voice input: speech to text. The browser half: a microphone button by the send
|
|
2
|
+
// button (browser.js) and /voice/voice.js, which apps can import too.
|
|
3
|
+
// Settings (environment, like MODEL_*): STT_BASE_URL, STT_MODEL, STT_API_KEY. With STT_BASE_URL set, audio goes to
|
|
4
|
+
// POST {STT_BASE_URL}/audio/transcriptions, OpenAI's endpoint, which most providers speak (OpenAI, Groq, Mistral,
|
|
5
|
+
// vLLM; others through LiteLLM). Without it, voice.js uses the browser's own speech recognition.
|
|
6
|
+
// The key stays here: apps are agent-written code, so they post audio to this server, never to the provider.
|
|
7
|
+
// GET /voice/config {"api": true | false}: whether audio can go to the API
|
|
8
|
+
// POST /voice/transcribe multipart: file (a complete audio file, e.g. chunk.webm), language?, prompt?
|
|
9
|
+
// GET /voice/voice.js the browser side, for the page (browser.js) and for apps
|
|
10
|
+
// -> {"text"} or {"error"} (with the provider's status)
|
|
11
|
+
import { Readable } from "node:stream";
|
|
12
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
13
|
+
|
|
14
|
+
const host = (globalThis as any).chalky;
|
|
15
|
+
|
|
16
|
+
export default function (_pi: ExtensionAPI) {
|
|
17
|
+
const base = (process.env.STT_BASE_URL || "").replace(/\/$/, "");
|
|
18
|
+
const json = (res: any, status: number, data: object) =>
|
|
19
|
+
res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(data));
|
|
20
|
+
|
|
21
|
+
host.routes.set("/voice/voice.js", (_req: any, res: any) => host.file(res, `${import.meta.dirname}/voice.js`));
|
|
22
|
+
host.routes.set("/voice/config", (_req: any, res: any) => json(res, 200, { api: !!base }));
|
|
23
|
+
|
|
24
|
+
host.routes.set("/voice/transcribe", async (req: any, res: any) => {
|
|
25
|
+
if (!base) return json(res, 400, { error: "no STT_BASE_URL: use the browser's speech recognition" });
|
|
26
|
+
try {
|
|
27
|
+
// the page's form as is, plus the model (Node reads multipart itself: Request.formData)
|
|
28
|
+
const form = await new Request("http://local", { method: "POST", headers: req.headers,
|
|
29
|
+
body: Readable.toWeb(req) as any, duplex: "half" } as any).formData();
|
|
30
|
+
if (!(form.get("file") instanceof Blob)) return json(res, 400, { error: "no audio file" });
|
|
31
|
+
form.set("model", process.env.STT_MODEL || "whisper-1");
|
|
32
|
+
const r = await fetch(`${base}/audio/transcriptions`, { method: "POST", body: form,
|
|
33
|
+
headers: process.env.STT_API_KEY ? { authorization: `Bearer ${process.env.STT_API_KEY}` } : {} });
|
|
34
|
+
const data: any = await r.json().catch(() => ({}));
|
|
35
|
+
if (!r.ok) return json(res, 502, { error: `speech to text: ${r.status} ${data.error?.message ?? ""}`.trim() });
|
|
36
|
+
json(res, 200, { text: data.text ?? "" });
|
|
37
|
+
} catch (e: any) { json(res, 502, { error: `speech to text: ${e.message}` }); }
|
|
38
|
+
});
|
|
39
|
+
}
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
// Voice input for apps: speech from the microphone, as text. An app imports it:
|
|
2
|
-
// import voice from "/
|
|
2
|
+
// import voice from "/voice/voice.js";
|
|
3
3
|
// await voice.start(onText, opts) start listening; onText(text) gets each finished piece of speech, in order
|
|
4
4
|
// opts: language ("en", optional), onState(state, detail) with state one of
|
|
5
5
|
// "listening", "speech", "sending", "off", "error" (detail: the message)
|
|
6
6
|
// voice.stop() stop listening; a piece still being spoken is sent first
|
|
7
7
|
// voice.mode() "api" or "browser", known after start
|
|
8
|
-
// "api" (the
|
|
9
|
-
// (2 to 10 s), each sent as a complete audio file to /voice
|
|
8
|
+
// "api" (the server has STT_BASE_URL: the voice extension's index.ts): the mic is cut into pieces at pauses
|
|
9
|
+
// (2 to 10 s), each sent as a complete audio file to /voice/transcribe; silent pieces aren't sent.
|
|
10
10
|
// "browser": the browser's own speech recognition (Chrome, Edge, Safari; audio goes to their vendor).
|
|
11
11
|
const shot = new URLSearchParams(window.top.location.search).has("shot"); // a screenshot: don't listen
|
|
12
12
|
let mode = null, stopNow = null;
|
|
13
13
|
|
|
14
14
|
async function start(onText, { language = "", onState = () => {} } = {}) {
|
|
15
15
|
if (shot || stopNow) return;
|
|
16
|
-
const config = await fetch("/voice
|
|
16
|
+
const config = await fetch("/voice/config").then((r) => r.json()).catch(() => ({ api: false }));
|
|
17
17
|
mode = config.api ? "api" : "browser";
|
|
18
18
|
stopNow = mode === "api" ? await startApi(onText, language, onState) : startBrowser(onText, language, onState);
|
|
19
19
|
}
|
|
@@ -57,7 +57,7 @@ async function startApi(onText, language, onState) {
|
|
|
57
57
|
if (lastText) form.append("prompt", lastText.slice(-200));
|
|
58
58
|
onState("sending");
|
|
59
59
|
try {
|
|
60
|
-
const r = await fetch("/voice
|
|
60
|
+
const r = await fetch("/voice/transcribe", { method: "POST", body: form });
|
|
61
61
|
const data = await r.json();
|
|
62
62
|
if (data.error) return onState("error", data.error);
|
|
63
63
|
if (data.text.trim()) { lastText = data.text; onText(data.text.trim()); }
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
|
-
name:
|
|
3
|
-
description: Check a web UI in a real headless browser - JavaScript errors, failed requests, clipped or overflowing elements, how it looks, and whether clicks and keys work. Use
|
|
2
|
+
name: browser-check
|
|
3
|
+
description: Check a web UI in a real headless browser - JavaScript errors, failed requests, clipped or overflowing elements, how it looks, and whether clicks and keys work. Use to check the page itself or a block you wrote, or for steps beyond the app README's example (drag, keys, dialogs, sizes).
|
|
4
4
|
---
|
|
5
5
|
# Checking a UI in a headless browser
|
|
6
6
|
|
|
@@ -13,7 +13,7 @@ From this skill's folder (paths here are relative to it):
|
|
|
13
13
|
node scripts/check.mjs URL [STEPS ...] [--size 1280x800] [--shot /tmp/ui.png] [--full]
|
|
14
14
|
```
|
|
15
15
|
- URL: the user's page is `{page}/`. An app you built is served by it too
|
|
16
|
-
(`{page}/
|
|
16
|
+
(`{page}/built/apps/<name>/index.html`); any other file: `file:///<full path>`.
|
|
17
17
|
- Steps run in order after the page loads (it gets 1.5 s first):
|
|
18
18
|
`wait:500`, `click:#pause`, `drag:#card=200,0`, `altdrag:#top=-150,0` (Alt held), `hold:.label=800` (press and hold),
|
|
19
19
|
`dblclick:.label`, `type:#message=hello`, `key:Enter`, `eval:document.querySelectorAll('.agent').length`,
|
|
@@ -65,7 +65,7 @@ The command you ran, its counts ("0 errors, 0 layout problems at 1280x800 and 39
|
|
|
65
65
|
eval steps returned, and what you saw in the screenshot.
|
|
66
66
|
|
|
67
67
|
## Pitfalls
|
|
68
|
-
- `type:` splits at the first `=`: use a selector without one (`
|
|
68
|
+
- `type:` splits at the first `=`: use a selector without one (`block-textfield input`, not `[data-card=x] input`).
|
|
69
69
|
- `querySelector` doesn't reach inside a shadow root; use `eval:` with `el.shadowRoot.querySelector(...)`.
|
|
70
70
|
- Content that arrives over the WebSocket may need `wait:1000` before you check it.
|
|
71
71
|
- Chrome starts fresh for each run (no cookies, no storage); each run takes a few seconds.
|
|
@@ -247,7 +247,7 @@ async function main(url, steps, size, shotPath, full, fit) {
|
|
|
247
247
|
return 2;
|
|
248
248
|
}
|
|
249
249
|
const [w, h] = size.split("x").map(Number);
|
|
250
|
-
const profile = fs.mkdtempSync(path.join(os.tmpdir(), "
|
|
250
|
+
const profile = fs.mkdtempSync(path.join(os.tmpdir(), "browser-check-"));
|
|
251
251
|
const headless = path.basename(chromePath).startsWith("chrome-headless-shell") ? [] : ["--headless"]; // a full browser: no window
|
|
252
252
|
const chrome = spawn(chromePath, [...headless, "--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu", "--no-first-run",
|
|
253
253
|
`--user-data-dir=${profile}`, "--remote-debugging-port=0", "about:blank"], { stdio: "ignore" });
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
|
-
name:
|
|
3
|
-
description: How to make what you build for the user look good and read clearly -
|
|
2
|
+
name: design
|
|
3
|
+
description: How to make what you build for the user look good and read clearly - charts, diagrams, game art, dashboards, one look across cards. Use for those, for a restyle, or for a look the user asks for; the app README has the basics a usual app needs.
|
|
4
4
|
---
|
|
5
5
|
# Designing what the user sees
|
|
6
6
|
|
|
@@ -11,21 +11,21 @@ Precedence: the user's own words, then the page's existing look, then your choic
|
|
|
11
11
|
## 1. Fit the page
|
|
12
12
|
The page is light, glassy and quiet (Apple-like): soft gradient background, translucent white glass,
|
|
13
13
|
thin edges, system font. Match it unless the user asks for another look.
|
|
14
|
-
- Use the page's tokens instead of literal colours (they are defined on the page and reach
|
|
14
|
+
- Use the page's tokens instead of literal colours (they are defined on the page and reach blocks;
|
|
15
15
|
apps in a frame copy the values they need):
|
|
16
16
|
`--text #1d1d1f`, `--soft rgba(29,29,31,.72)`, `--muted rgba(60,60,67,.55)`, `--accent #0071e3`,
|
|
17
17
|
`--danger #e5484d`, `--line rgba(0,0,0,.06)`, `--glass rgba(255,255,255,.45)`,
|
|
18
18
|
`--glass-strong rgba(255,255,255,.7)`, `--edge rgba(255,255,255,.75)`, `--radius 16px`.
|
|
19
|
-
- The built-in
|
|
20
|
-
prefer them, and style your own
|
|
19
|
+
- The built-in blocks (Panel, Button, Table, Text, TextField, Markdown) already use these tokens;
|
|
20
|
+
prefer them, and style your own blocks the same way.
|
|
21
21
|
- Font: inherit the page's system font. Use a Google Font only when the subject calls for it
|
|
22
22
|
(a pixel-art game, a poster), and always give a fallback stack.
|
|
23
23
|
- Show just the thing: no frame, card or background around it unless it needs one. Transparent app backgrounds.
|
|
24
|
-
- A user-requested look (pixel art, neon, paper) wins. Commit to it fully inside that
|
|
24
|
+
- A user-requested look (pixel art, neon, paper) wins. Commit to it fully inside that card,
|
|
25
25
|
and keep the rest of the page as it is.
|
|
26
26
|
|
|
27
27
|
## 2. Ground it in the subject
|
|
28
|
-
- Pick one concrete job for each
|
|
28
|
+
- Pick one concrete job for each card. Use real content, never lorem ipsum.
|
|
29
29
|
- Take at least one detail from the subject's own world: its real units, terms, conventions
|
|
30
30
|
(a chess clock shows mm:ss; a kanban card has an owner and a state).
|
|
31
31
|
- Structure must mean something: number things only when order matters; a divider only between real groups.
|
|
@@ -87,19 +87,19 @@ talk, what changes between two options.
|
|
|
87
87
|
|
|
88
88
|
## 9. Make it easy to reshape
|
|
89
89
|
The user changes what you build by hand (pins, levers on the back side, combining, undo), so build for that.
|
|
90
|
-
- One clear thing per
|
|
90
|
+
- One clear thing per card, with a name the user would say (`lunch-orders`, not `card3`).
|
|
91
91
|
- Write `about`: what it is for and any decision a later change should respect ("drinks are not counted").
|
|
92
|
-
- Put what changes in data (rows, scores, settings) and the look in
|
|
92
|
+
- Put what changes in data (rows, scores, settings) and the look in blocks: data changes are instant
|
|
93
93
|
for the user and never need you.
|
|
94
|
-
- Read other
|
|
95
|
-
- A
|
|
94
|
+
- Read other cards' data with `{"ref": "<card>", "path": "/..."}` instead of copying it, so it stays live.
|
|
95
|
+
- A block you write: its about block lists every prop with a type, and `levers` for the few a user
|
|
96
96
|
would want to change (a size, a mode, a label), written as plain sentences.
|
|
97
97
|
|
|
98
98
|
## 10. One look
|
|
99
99
|
What the user builds over time should look like one hand made it: a garden and the wrestler standing
|
|
100
100
|
in it share a palette, a way of shading and a level of detail.
|
|
101
101
|
- Before drawing something new, look at what is already there (a screenshot, and the code of the
|
|
102
|
-
|
|
102
|
+
cards it sits beside). Match its palette, shading, line weight and detail. Read their code and
|
|
103
103
|
reuse their colours and helpers rather than redrawing them from memory.
|
|
104
104
|
- Keep a small palette: a ground, a main colour for shapes, an ink for words, and one accent for the
|
|
105
105
|
one thing to look at. Mix shades from those (`color-mix(in srgb, <main> 70%, <ground>)`, lighter and
|
|
@@ -114,10 +114,10 @@ in it share a palette, a way of shading and a level of detail.
|
|
|
114
114
|
three. Neon, glows, gloss and full saturation everywhere look machine-made unless the subject calls for them.
|
|
115
115
|
- Games and 3D: the same idea as art direction, not exact colours. One sky or fog colour and one light
|
|
116
116
|
colour across the scene hold it together, however many materials it has; the accent marks what the
|
|
117
|
-
player should notice; the HUD and menus use the palette like any
|
|
117
|
+
player should notice; the HUD and menus use the palette like any card.
|
|
118
118
|
- A default, not a rule: when the user asks for another look, or a piece clearly needs its own world
|
|
119
119
|
(a dark space game beside a light garden), give it one, commit to it, and say so in its `about`.
|
|
120
120
|
|
|
121
121
|
## 11. Check it
|
|
122
|
-
Load
|
|
122
|
+
Load browser-check and look at a screenshot: overlaps, clipped text, labels colliding, things outside their box.
|
|
123
123
|
Fix what you see once, then stop polishing; the user will point at what they want changed.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: extension-map
|
|
3
|
+
description: Find your way around your own code (the pi extensions in your agent folder, the page among them) without reading whole files. Use before changing yourself, or before planning work for others that will.
|
|
4
|
+
---
|
|
5
|
+
# Getting a map of your own code
|
|
6
|
+
|
|
7
|
+
Your agent folder ({agent} in your prompt) holds `extensions/`. Every feature is one extension, and every extension
|
|
8
|
+
is one folder, `extensions/<name>/`, named after the one thing it is (card, board, note, team...), holding all of it:
|
|
9
|
+
|
|
10
|
+
| File | What it is |
|
|
11
|
+
|---------------|--------------------------------------------------------------------|
|
|
12
|
+
| `index.ts` | the extension itself (pi loads it); starts with a header comment |
|
|
13
|
+
| `guide.md` | its short prompt section (found by itself: the prompt extension adds every one) |
|
|
14
|
+
| `README.md` | the whole of what the guide says, read when the guide says |
|
|
15
|
+
| `browser.js` | its part of the page (found by itself: the page extension loads every one) |
|
|
16
|
+
| `skills/` | skills only this feature needs (its index.ts gives them to pi) |
|
|
17
|
+
|
|
18
|
+
An extension can have other files too (`card/` has its `blocks/`, `page/` has `page.html`, `model/` has
|
|
19
|
+
`login.ts`). The headers are the documentation, kept with the code, so read them before the code itself.
|
|
20
|
+
|
|
21
|
+
## The naming rule
|
|
22
|
+
Everything an extension adds carries its name: tools `<name>_<verb>` (card_show, note_done, team_ask), page
|
|
23
|
+
messages `<name>_<verb>` (card_move, history_undo), page events `<name>` (card, history), routes `/<name>/...`
|
|
24
|
+
(/card/all, /note/all), its data in {data} (`<name>.json` or `<name>/`), and what it offers others as
|
|
25
|
+
`host.api.<name>` (host.api.card.watch, host.api.team.send). So from any one name you can find the rest.
|
|
26
|
+
|
|
27
|
+
## 1. The map: one line per extension
|
|
28
|
+
From your agent folder:
|
|
29
|
+
```
|
|
30
|
+
for d in extensions/*/; do printf '%-24s %s\n' "$d" "$(grep -m1 '^//' "$d/index.ts" | sed 's|^// \?||' | cut -c1-140)"; done
|
|
31
|
+
```
|
|
32
|
+
Also there:
|
|
33
|
+
- `extensions/page/`: the page itself (`page.html`), its list of browser parts, and what serves them.
|
|
34
|
+
- `extensions/card/blocks/*.js`: the blocks cards are built from, each starting with a comment line on what it
|
|
35
|
+
is, then its `about` block (props, levers, example; read its first 20 lines).
|
|
36
|
+
- `skills/<name>/SKILL.md`: skills about you as a whole (this one); a feature's own skills are in its folder.
|
|
37
|
+
- {built}: what you made for the user, not code: `card.json` (the cards), `board.json`, `apps/<name>/`.
|
|
38
|
+
- The host, `server.ts` (its path is in your prompt): the four folders, sessions, the web server, the version
|
|
39
|
+
history's rule, `globalThis.chalky`. Read its header; don't edit it.
|
|
40
|
+
|
|
41
|
+
## 2. The headers of the extensions involved
|
|
42
|
+
```
|
|
43
|
+
awk '/^import /{exit} 1' extensions/team/index.ts # everything above the imports
|
|
44
|
+
```
|
|
45
|
+
Read only the headers of the extensions your work touches. All of them at once is over 13,000 characters.
|
|
46
|
+
|
|
47
|
+
## 3. Then only the code you will change
|
|
48
|
+
Use `grep -n` to find the function, then `read` with `offset` (its first line) and `limit` for that part.
|
|
49
|
+
A whole file can be long (team's index.ts is over 15,000 characters). If you are planning work for other agents,
|
|
50
|
+
the headers are usually enough: let each agent read the code it owns.
|
|
51
|
+
|
|
52
|
+
## Keep the headers true
|
|
53
|
+
When you change what an extension provides (a tool, a route, a page message, an event, a host service), update
|
|
54
|
+
its header in the same change, and follow the naming rule. Other agents, and you after a reload, rely on it. A new
|
|
55
|
+
extension is a new folder with an `index.ts` whose header starts with a one-line summary: that line is its entry
|
|
56
|
+
in the map. After editing, call the reload tool.
|
|
@@ -19,7 +19,7 @@ skills/<name>/ in your code folder (the one your prompt names, with extension
|
|
|
19
19
|
assets/ optional: templates, data files
|
|
20
20
|
```
|
|
21
21
|
A skill that only one feature needs goes in that extension's folder instead, `extensions/<name>/skills/<skill>/`
|
|
22
|
-
(as `extensions/
|
|
22
|
+
(as `extensions/app/skills/`), and its `index.ts` gives the folder to pi:
|
|
23
23
|
`pi.on("resources_discover", () => ({ skillPaths: [path.join(import.meta.dirname, "skills")] }))`.
|
|
24
24
|
|
|
25
25
|
## SKILL.md
|
package/bin/chalky.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// chalky: starts Chalky from the installed package. Its home (
|
|
3
|
-
// ~/.chalky; the first run copies the package's
|
|
2
|
+
// chalky: starts Chalky from the installed package. Its home (agent/, built/, data/, workspace/) is CHALKY_HOME,
|
|
3
|
+
// else ~/.chalky; the first run copies the package's agent/ there. Node won't run TypeScript under node_modules
|
|
4
4
|
// (ERR_UNSUPPORTED_NODE_MODULES_TYPE_STRIPPING), so server.ts loads through jiti, as pi loads extensions.
|
|
5
|
-
// MODEL_BASE_URL=... MODEL_API_KEY=... [MODEL_NAME=...]
|
|
5
|
+
// MODEL_BASE_URL=... MODEL_API_KEY=... [MODEL_NAME=...] chalky
|
|
6
6
|
import os from "node:os";
|
|
7
7
|
import path from "node:path";
|
|
8
8
|
import { createJiti } from "jiti";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chalky/chalky",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"description": "Chalky on pi: agents build what you see in the browser, and you point at it and mark it up",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -10,13 +10,13 @@
|
|
|
10
10
|
"files": [
|
|
11
11
|
"bin/",
|
|
12
12
|
"server.ts",
|
|
13
|
-
"
|
|
14
|
-
"!
|
|
15
|
-
"!
|
|
16
|
-
"!
|
|
17
|
-
"!
|
|
18
|
-
"!
|
|
19
|
-
"
|
|
13
|
+
"agent/",
|
|
14
|
+
"!agent/auth.json",
|
|
15
|
+
"!agent/models-store.json",
|
|
16
|
+
"!agent/models.json",
|
|
17
|
+
"!agent/mcp-auth.json",
|
|
18
|
+
"!agent/mcp.json",
|
|
19
|
+
"!agent/ui/"
|
|
20
20
|
],
|
|
21
21
|
"publishConfig": {
|
|
22
22
|
"access": "public"
|