apple-notes-mcp 2.9.9 → 2.9.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/build/index.js +540 -45
- package/native/public-helper/apple-notes-public-helper.swift +331 -5
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -261,7 +261,7 @@ Creates a new note in Apple Notes.
|
|
|
261
261
|
| `account` | string | No | Account name (defaults to Notes.app's default account; matched exactly or by a *unique* prefix — an ambiguous prefix is refused). Must be an account Notes.app already has configured — see [`list-accounts`](#list-accounts) |
|
|
262
262
|
| `format` | string | No | Content format: `"plaintext"` (default), `"html"`, or `"markdown"`. In all formats, the title is automatically prepended as the note's title line. In plaintext mode, newlines become `<br>`, tabs become `<br>`, and backslashes are preserved as HTML entities. `"markdown"` produces real Title/Heading/Subheading styles through a Shortcut; see [Markdown notes](#markdown-notes) |
|
|
263
263
|
| `markdownRoute` | string | No | With `format: "markdown"` only: `"shortcut"` (default) or `"html"`. See [Markdown through HTML](#markdown-through-html) |
|
|
264
|
-
| `timeoutSeconds` | number | No | Whole seconds, 1–120, for each Notes.app automation step this call runs; overrides `APPLE_NOTES_MCP_TIMEOUT_MS` for this call only. A timed-out write is uncertain, not failed: read the note by id before any retry. Also accepted by `update-note`, `append-to-note`, `delete-note` and `move-note` |
|
|
264
|
+
| `timeoutSeconds` | number | No | Whole seconds, 1–120, for each Notes.app automation step this call runs; overrides `APPLE_NOTES_MCP_TIMEOUT_MS` for this call only. A timed-out write is uncertain, not failed: read the note by id before any retry. Also accepted by `get-note-content`, `update-note`, `append-to-note`, `delete-note` and `move-note` |
|
|
265
265
|
|
|
266
266
|
**Example (tagged with inline hashtags):**
|
|
267
267
|
```json
|
|
@@ -531,9 +531,12 @@ Retrieves the full content of a specific note.
|
|
|
531
531
|
| `id` | string | No | Note ID (preferred - more reliable than title) |
|
|
532
532
|
| `title` | string | No | Note title (use `id` instead when available) |
|
|
533
533
|
| `account` | string | No | Account containing the note (defaults to Notes.app's default account; exact or unique-prefix match, ignored if `id` is provided) |
|
|
534
|
+
| `timeoutSeconds` | number | No | Whole seconds, 1–120, for the body read; overrides `APPLE_NOTES_MCP_TIMEOUT_MS` for this call only |
|
|
534
535
|
|
|
535
536
|
**Note:** Either `id` or `title` must be provided. Using `id` is recommended as it's unique and avoids issues with duplicate titles.
|
|
536
537
|
|
|
538
|
+
**Large images:** Notes.app returns images inside the body as base64, so a note holding a very large image (tens of MB) can take longer to read than the timeout allows. When the read times out or overflows the output buffer, the error says so and, with Full Disk Access, names the attachments of 5 MB or more. Retry with a larger `timeoutSeconds`; `delete-note` accepts the same argument, and it needs a successful read to verify the note before deleting it.
|
|
539
|
+
|
|
537
540
|
**Example - Using ID (recommended):**
|
|
538
541
|
```json
|
|
539
542
|
{
|
|
@@ -1854,6 +1857,37 @@ Decodes a note's classic PencilKit drawings (`com.apple.drawing.2` and the older
|
|
|
1854
1857
|
|
|
1855
1858
|
---
|
|
1856
1859
|
|
|
1860
|
+
#### `transcribe-note-audio`
|
|
1861
|
+
|
|
1862
|
+
Transcribes a note's voice recordings and audio attachments now, on this Mac, with Apple's Speech framework through the [public native helper](#public-native-helper). Recognition is on-device only: `SpeechAnalyzer` on macOS 26 and later, or `SFSpeechRecognizer` with on-device recognition required on older systems (a locale without on-device support is refused, never sent to a server). The audio files are opened read-only where Notes keeps them.
|
|
1863
|
+
|
|
1864
|
+
**Requires:** Full Disk Access for the MCP host process, and the public native helper built once with `apple-notes-mcp setup --public-helper`.
|
|
1865
|
+
|
|
1866
|
+
| Parameter | Type | Required | Description |
|
|
1867
|
+
|-----------|------|----------|-------------|
|
|
1868
|
+
| `id` | string | Yes | Note ID (use `search-notes` to find it first) |
|
|
1869
|
+
| `locale` | string | No | BCP-47 language of the speech, such as `"en-US"` (default), `"it-IT"`, or `"fr-FR"` |
|
|
1870
|
+
| `attachmentId` | string | No | Only transcribe this audio attachment (an `x-coredata://…/ICAttachment/pN` id) |
|
|
1871
|
+
| `includeText` | boolean | No | Include transcript text (default `true`); `false` returns statuses and word counts only |
|
|
1872
|
+
| `downloadAssets` | boolean | No | Let macOS download the language's on-device speech model when it is missing (default `false`: the call returns `asset_unavailable` at once) |
|
|
1873
|
+
| `maxSeconds` | integer | No | Total time budget for the call, 30 to 3600 seconds (default 900) |
|
|
1874
|
+
|
|
1875
|
+
**Returns:** Overall `status` and, per audio attachment, `status`, `durationSeconds`, `wordCount`, `transcript`, and `takes` (a Notes recording can hold several takes; each is transcribed and the texts are joined in stored order). Statuses:
|
|
1876
|
+
|
|
1877
|
+
- `ok`: the whole recording was transcribed.
|
|
1878
|
+
- `partial`: some text came back, but a take failed or the helper stopped at its deadline (`code: "incomplete"`).
|
|
1879
|
+
- `error`: nothing was transcribed; `code` says why (`asset_unavailable` when the audio file is not on this Mac or the language's speech model is not installed, `permission_required`, `unsupported_locale`, `unsupported_audio`, `time_limit`, ...).
|
|
1880
|
+
- `indeterminate`: the helper did not answer in time, so the outcome is unknown and a retry may succeed.
|
|
1881
|
+
- `none` (overall only): the note has no audio.
|
|
1882
|
+
|
|
1883
|
+
Each take gets a deadline of 1.5 times its length plus a minute (at most 30 minutes), shortened to what is left of `maxSeconds`. A take that cannot start before the budget runs out reports `code: "time_limit"`. The helper runs as a separate process without blocking the server, and cancelling the request stops it. On macOS 27 a 16-minute recording took about 22 seconds. Some MCP clients stop waiting for a tool after a fixed time, so transcribe long recordings one at a time with `attachmentId`. Transcripts longer than `APPLE_NOTES_MCP_EXPORT_MAX_BYTES` are shortened and marked `transcriptTruncated`.
|
|
1884
|
+
|
|
1885
|
+
**Speech models:** the call never starts a download on its own. When the language's on-device model is not installed, it returns `asset_unavailable` right away. Pass `downloadAssets: true` to let macOS download it (a one-time download); if it is still downloading when the call ends, try again shortly.
|
|
1886
|
+
|
|
1887
|
+
**Speech Recognition permission:** the server never shows the Speech Recognition prompt, because nobody may be watching an MCP server to answer it. The helper reads the current authorization first. On older macOS, the `SFSpeechRecognizer` path needs that access, and macOS attributes the grant to the app that launches the MCP server (Claude Desktop, Codex, Terminal, and so on), not to the helper. Without it the call returns `code: "permission_required"`; allow the app under System Settings > Privacy & Security > Speech Recognition. On macOS 26 and later, `SpeechAnalyzer` transcribed files without any grant in testing (authorization stayed "not determined"), so only an explicit refusal (denied or restricted) stops it.
|
|
1888
|
+
|
|
1889
|
+
---
|
|
1890
|
+
|
|
1857
1891
|
#### `add-attachment`
|
|
1858
1892
|
|
|
1859
1893
|
Adds one nonempty local file of at most 64 MiB to an exact note using `id`, the
|
|
@@ -2493,13 +2527,13 @@ Every tool that does not read the Notes database works normally without Full Dis
|
|
|
2493
2527
|
- `get-note-link` returns an error on macOS 26+; on macOS 12–15 it still works via the AppleScript `note link` fallback
|
|
2494
2528
|
- `get-note-markdown` returns plain list items without `[x]`/`[ ]` annotations (graceful fallback)
|
|
2495
2529
|
- `get-sync-status` still answers, but reports no pending uploads and no active sync — treat that as "unknown", not "idle"
|
|
2496
|
-
- `get-note-drawings`
|
|
2530
|
+
- `get-note-drawings` and `transcribe-note-audio` return the same Full Disk Access error
|
|
2497
2531
|
|
|
2498
2532
|
---
|
|
2499
2533
|
|
|
2500
2534
|
## Public native helper
|
|
2501
2535
|
|
|
2502
|
-
`get-note-drawings` needs Apple's PencilKit framework
|
|
2536
|
+
`get-note-drawings` needs Apple's PencilKit framework and `transcribe-note-audio` needs the Speech framework; neither has an AppleScript or command-line interface. For them, the server uses a small Swift helper that links public Apple frameworks only (AppKit, PencilKit, AVFoundation, and Speech). No prebuilt binary ships with the package. Build it once on your Mac:
|
|
2503
2537
|
|
|
2504
2538
|
```bash
|
|
2505
2539
|
apple-notes-mcp setup --public-helper # compile, sign, verify, install
|
|
@@ -2508,7 +2542,7 @@ apple-notes-mcp setup --public-helper --check # report the installed state only
|
|
|
2508
2542
|
|
|
2509
2543
|
Setup compiles `native/public-helper/apple-notes-public-helper.swift` with `xcrun swiftc` (install the Command Line Tools with `xcode-select --install` if it is missing), signs it ad hoc, runs its `hello` handshake, and installs it in `~/Library/Application Support/apple-notes-mcp/public-helper/` next to a manifest recording the SHA-256 of the source and the binary. Before every use the server re-checks both digests: after an upgrade that changes the helper source, or if the binary is replaced, the helper is refused until you run setup again. `APPLE_NOTES_MCP_PUBLIC_HELPER_DIR` overrides the install folder and `APPLE_NOTES_MCP_PUBLIC_HELPER_TIMEOUT_MS` the per-call timeout.
|
|
2510
2544
|
|
|
2511
|
-
The helper never opens the Notes database and never writes under the Notes group container. The server reads the bytes it needs (read-only) and passes them to the helper on stdin; the helper answers with one JSON object on stdout.
|
|
2545
|
+
The helper never opens the Notes database and never writes under the Notes group container. The server reads the bytes it needs (read-only) and passes them to the helper on stdin, or, for transcription, names one audio file that the helper opens for reading; the helper answers with one JSON object on stdout.
|
|
2512
2546
|
|
|
2513
2547
|
---
|
|
2514
2548
|
|
package/build/index.js
CHANGED
|
@@ -24465,14 +24465,14 @@ var require_turndown_cjs = __commonJS({
|
|
|
24465
24465
|
} else if (node.nodeType === 1) {
|
|
24466
24466
|
replacement = replacementForNode.call(self, node);
|
|
24467
24467
|
}
|
|
24468
|
-
return
|
|
24468
|
+
return join28(output, replacement);
|
|
24469
24469
|
}, "");
|
|
24470
24470
|
}
|
|
24471
24471
|
function postProcess(output) {
|
|
24472
24472
|
var self = this;
|
|
24473
24473
|
this.rules.forEach(function(rule) {
|
|
24474
24474
|
if (typeof rule.append === "function") {
|
|
24475
|
-
output =
|
|
24475
|
+
output = join28(output, rule.append(self.options));
|
|
24476
24476
|
}
|
|
24477
24477
|
});
|
|
24478
24478
|
return output.replace(/^[\t\r\n]+/, "").replace(/[\t\r\n\s]+$/, "");
|
|
@@ -24484,7 +24484,7 @@ var require_turndown_cjs = __commonJS({
|
|
|
24484
24484
|
if (whitespace.leading || whitespace.trailing) content = content.trim();
|
|
24485
24485
|
return whitespace.leading + rule.replacement(content, node, this.options) + whitespace.trailing;
|
|
24486
24486
|
}
|
|
24487
|
-
function
|
|
24487
|
+
function join28(output, replacement) {
|
|
24488
24488
|
var s1 = trimTrailingNewlines(output);
|
|
24489
24489
|
var s2 = trimLeadingNewlines(replacement);
|
|
24490
24490
|
var nls = Math.max(output.length - s1.length, replacement.length - s2.length);
|
|
@@ -42825,15 +42825,27 @@ var AppleNotesManager = class {
|
|
|
42825
42825
|
* @returns HTML content of the note, or empty string if not found
|
|
42826
42826
|
*/
|
|
42827
42827
|
getNoteContentById(id2) {
|
|
42828
|
+
return this.readNoteBodyById(id2).body;
|
|
42829
|
+
}
|
|
42830
|
+
/**
|
|
42831
|
+
* Like {@link getNoteContentById}, but keeps the automation error so a
|
|
42832
|
+
* caller can explain a failed read (for example a timeout on a note whose
|
|
42833
|
+
* body carries a very large inline image, #237) instead of reporting a bare
|
|
42834
|
+
* failure.
|
|
42835
|
+
*
|
|
42836
|
+
* @param id - CoreData URL identifier for the note
|
|
42837
|
+
* @returns The HTML body, or an empty body plus the error that stopped the read
|
|
42838
|
+
*/
|
|
42839
|
+
readNoteBodyById(id2) {
|
|
42828
42840
|
const safeId = sanitizeId(id2);
|
|
42829
42841
|
const getCommand = `get body of note id "${safeId}"`;
|
|
42830
42842
|
const script = buildAppLevelScript(getCommand);
|
|
42831
42843
|
const result = executeAppleScript(script);
|
|
42832
42844
|
if (!result.success) {
|
|
42833
42845
|
console.error(`Failed to get content of note with ID "${id2}":`, result.error);
|
|
42834
|
-
return "";
|
|
42846
|
+
return { body: "", error: result.error };
|
|
42835
42847
|
}
|
|
42836
|
-
return result.output;
|
|
42848
|
+
return { body: result.output };
|
|
42837
42849
|
}
|
|
42838
42850
|
/**
|
|
42839
42851
|
* Retrieves the plain-text content of a note by its exact title.
|
|
@@ -49593,6 +49605,37 @@ function readNoteStructure(id2, { dbPath: dbPath2 = NOTES_DB_PATH7, includeText
|
|
|
49593
49605
|
};
|
|
49594
49606
|
}
|
|
49595
49607
|
|
|
49608
|
+
// src/utils/bodyReadFailure.ts
|
|
49609
|
+
var LARGE_ATTACHMENT_BYTES = 5 * 1024 * 1024;
|
|
49610
|
+
function classifyBodyReadError(error2) {
|
|
49611
|
+
if (!error2) return "other";
|
|
49612
|
+
if (/ENOBUFS|maxBuffer/i.test(error2)) return "buffer";
|
|
49613
|
+
if (/timed out|timeout|-1712/i.test(error2)) return "timeout";
|
|
49614
|
+
return "other";
|
|
49615
|
+
}
|
|
49616
|
+
function formatBytes2(bytes) {
|
|
49617
|
+
if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
49618
|
+
if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
49619
|
+
return `${bytes} bytes`;
|
|
49620
|
+
}
|
|
49621
|
+
function largeAttachments(attachments, threshold = LARGE_ATTACHMENT_BYTES) {
|
|
49622
|
+
const flat = attachments.flatMap((a) => [a, ...a.children ?? []]);
|
|
49623
|
+
return flat.filter((a) => typeof a.fileSize === "number" && a.fileSize >= threshold).map((a) => ({ name: a.filename || a.title || "attachment", bytes: a.fileSize })).sort((a, b) => b.bytes - a.bytes);
|
|
49624
|
+
}
|
|
49625
|
+
function describeBodyReadFailure(title, error2, attachments) {
|
|
49626
|
+
const base = `Failed to read content of note "${title}"${error2 ? `: ${error2}` : ""}`;
|
|
49627
|
+
const kind = classifyBodyReadError(error2);
|
|
49628
|
+
if (kind === "other") return base;
|
|
49629
|
+
const large = attachments ? largeAttachments(attachments) : [];
|
|
49630
|
+
const cause = large.length > 0 ? `The note holds ${large.length === 1 ? "a large attachment" : "large attachments"} (${large.map((a) => `${a.name}, ${formatBytes2(a.bytes)}`).join(
|
|
49631
|
+
"; "
|
|
49632
|
+
)}). Notes.app returns images inside the note body as base64, so a large image makes the body too big to read in time.` : "Notes.app returns images inside the note body as base64, so a note with a very large image can take longer to read than the timeout allows.";
|
|
49633
|
+
const remedy = kind === "buffer" ? "Raise APPLE_NOTES_MCP_MAX_BUFFER (bytes) and retry, or remove the attachment in Notes.app." : "Retry with a longer timeoutSeconds (up to 120) or raise APPLE_NOTES_MCP_TIMEOUT_MS. If it still fails, delete or shrink the attachment in Notes.app; delete-note needs a successful read to verify the note first.";
|
|
49634
|
+
return `${base}
|
|
49635
|
+
|
|
49636
|
+
${cause} ${remedy}`;
|
|
49637
|
+
}
|
|
49638
|
+
|
|
49596
49639
|
// src/utils/noteLinkInventory.ts
|
|
49597
49640
|
import { dirname as dirname5 } from "node:path";
|
|
49598
49641
|
var BODY_BATCH = 100;
|
|
@@ -54637,7 +54680,7 @@ function formatShortcutSetup(report) {
|
|
|
54637
54680
|
}
|
|
54638
54681
|
|
|
54639
54682
|
// src/services/publicHelper.ts
|
|
54640
|
-
import { spawnSync as spawnSync3 } from "node:child_process";
|
|
54683
|
+
import { spawn, spawnSync as spawnSync3 } from "node:child_process";
|
|
54641
54684
|
import { createHash as createHash5 } from "node:crypto";
|
|
54642
54685
|
import {
|
|
54643
54686
|
chmodSync,
|
|
@@ -54659,7 +54702,11 @@ var PUBLIC_HELPER_BINARY = "apple-notes-public-helper";
|
|
|
54659
54702
|
var PUBLIC_HELPER_SOURCE = "native/public-helper/apple-notes-public-helper.swift";
|
|
54660
54703
|
var PUBLIC_HELPER_MANIFEST = "manifest.json";
|
|
54661
54704
|
var PUBLIC_HELPER_SETUP_COMMAND = "apple-notes-mcp setup --public-helper";
|
|
54662
|
-
var PUBLIC_HELPER_ACTIONS = /* @__PURE__ */ new Set([
|
|
54705
|
+
var PUBLIC_HELPER_ACTIONS = /* @__PURE__ */ new Set([
|
|
54706
|
+
"hello",
|
|
54707
|
+
"decode_drawing",
|
|
54708
|
+
"transcribe"
|
|
54709
|
+
]);
|
|
54663
54710
|
var DEFAULT_TIMEOUT_MS2 = 3e4;
|
|
54664
54711
|
var MAX_OUTPUT_BYTES = 256 * 1024 * 1024;
|
|
54665
54712
|
var publicManifestSchema = external_exports.object({
|
|
@@ -54695,6 +54742,7 @@ function defaultPublicHelperDeps(overrides = {}) {
|
|
|
54695
54742
|
exists: existsSync14,
|
|
54696
54743
|
readFile: (path10) => readFileSync4(path10),
|
|
54697
54744
|
spawn: spawnSync3,
|
|
54745
|
+
spawnAsync: spawn,
|
|
54698
54746
|
...overrides
|
|
54699
54747
|
};
|
|
54700
54748
|
}
|
|
@@ -54759,10 +54807,14 @@ var UNAVAILABLE_CODES = /* @__PURE__ */ new Set([
|
|
|
54759
54807
|
"helper_modified",
|
|
54760
54808
|
"helper_manifest_invalid"
|
|
54761
54809
|
]);
|
|
54810
|
+
var ENVELOPE_CODES = {
|
|
54811
|
+
invalid_request: "validation_error",
|
|
54812
|
+
attachment_not_found: "not_found"
|
|
54813
|
+
};
|
|
54762
54814
|
var PublicHelperError = class extends CodedError {
|
|
54763
54815
|
constructor(code, message) {
|
|
54764
54816
|
super(message, {
|
|
54765
|
-
code: UNAVAILABLE_CODES.has(code) ? "unsupported" : "operation_failed",
|
|
54817
|
+
code: UNAVAILABLE_CODES.has(code) ? "unsupported" : ENVELOPE_CODES[code] ?? "operation_failed",
|
|
54766
54818
|
helperCode: code
|
|
54767
54819
|
});
|
|
54768
54820
|
this.code = code;
|
|
@@ -54778,21 +54830,9 @@ var publicHelloSchema = external_exports.object({
|
|
|
54778
54830
|
actions: external_exports.array(external_exports.string())
|
|
54779
54831
|
}).passthrough();
|
|
54780
54832
|
function callPublicHelper(action, fields = {}, deps = defaultPublicHelperDeps(), options = {}) {
|
|
54781
|
-
|
|
54782
|
-
throw new PublicHelperError(
|
|
54783
|
-
"unknown_action",
|
|
54784
|
-
`"${action}" is not an action the server sends to the public helper.`
|
|
54785
|
-
);
|
|
54786
|
-
let binaryPath = options.binaryPath;
|
|
54787
|
-
if (!binaryPath) {
|
|
54788
|
-
const install = inspectPublicHelper(deps);
|
|
54789
|
-
if (!install.ready)
|
|
54790
|
-
throw new PublicHelperError(install.reason ?? "helper_not_installed", install.detail ?? "");
|
|
54791
|
-
binaryPath = install.binaryPath;
|
|
54792
|
-
}
|
|
54793
|
-
const timeout = Number.parseInt(deps.env[PUBLIC_HELPER_TIMEOUT_ENV] || "", 10) || options.timeoutMs || DEFAULT_TIMEOUT_MS2;
|
|
54833
|
+
const { binaryPath, timeout, input } = prepareCall(action, fields, deps, options);
|
|
54794
54834
|
const result = deps.spawn(binaryPath, [], {
|
|
54795
|
-
input
|
|
54835
|
+
input,
|
|
54796
54836
|
encoding: "utf8",
|
|
54797
54837
|
timeout,
|
|
54798
54838
|
killSignal: "SIGKILL",
|
|
@@ -54806,24 +54846,100 @@ function callPublicHelper(action, fields = {}, deps = defaultPublicHelperDeps(),
|
|
|
54806
54846
|
"helper_unreachable",
|
|
54807
54847
|
`Could not run the helper: ${result.error.message}`
|
|
54808
54848
|
);
|
|
54849
|
+
return parseHelperOutput(result.status, String(result.stdout ?? ""));
|
|
54850
|
+
}
|
|
54851
|
+
async function callPublicHelperAsync(action, fields = {}, deps = defaultPublicHelperDeps(), options = {}) {
|
|
54852
|
+
const { binaryPath, timeout, input } = prepareCall(action, fields, deps, options);
|
|
54853
|
+
const { signal } = options;
|
|
54854
|
+
const aborted2 = () => new PublicHelperError("aborted", "The request was cancelled; the helper was stopped.");
|
|
54855
|
+
if (signal?.aborted) throw aborted2();
|
|
54856
|
+
return new Promise((resolvePromise, reject) => {
|
|
54857
|
+
const child = (deps.spawnAsync ?? spawn)(binaryPath, [], {
|
|
54858
|
+
stdio: ["pipe", "pipe", "ignore"]
|
|
54859
|
+
});
|
|
54860
|
+
const chunks = [];
|
|
54861
|
+
let size = 0;
|
|
54862
|
+
let failure = null;
|
|
54863
|
+
const stop = (error2) => {
|
|
54864
|
+
failure ??= error2;
|
|
54865
|
+
child.kill("SIGKILL");
|
|
54866
|
+
};
|
|
54867
|
+
const timer = setTimeout(
|
|
54868
|
+
() => stop(new PublicHelperError("timeout", `The helper did not answer within ${timeout} ms.`)),
|
|
54869
|
+
timeout
|
|
54870
|
+
);
|
|
54871
|
+
const onAbort = () => stop(aborted2());
|
|
54872
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
54873
|
+
const settle = () => {
|
|
54874
|
+
clearTimeout(timer);
|
|
54875
|
+
signal?.removeEventListener("abort", onAbort);
|
|
54876
|
+
};
|
|
54877
|
+
child.stdout?.on("data", (chunk) => {
|
|
54878
|
+
size += chunk.length;
|
|
54879
|
+
if (size > MAX_OUTPUT_BYTES)
|
|
54880
|
+
stop(new PublicHelperError("invalid_response", "The helper response is too large."));
|
|
54881
|
+
else chunks.push(chunk);
|
|
54882
|
+
});
|
|
54883
|
+
child.on("error", (error2) => {
|
|
54884
|
+
settle();
|
|
54885
|
+
reject(
|
|
54886
|
+
failure ?? new PublicHelperError("helper_unreachable", `Could not run the helper: ${error2.message}`)
|
|
54887
|
+
);
|
|
54888
|
+
});
|
|
54889
|
+
child.on("close", (status, exitSignal) => {
|
|
54890
|
+
settle();
|
|
54891
|
+
if (failure) return reject(failure);
|
|
54892
|
+
if (status === null && exitSignal)
|
|
54893
|
+
return reject(
|
|
54894
|
+
new PublicHelperError("helper_crashed", `The helper stopped on signal ${exitSignal}.`)
|
|
54895
|
+
);
|
|
54896
|
+
try {
|
|
54897
|
+
resolvePromise(parseHelperOutput(status, Buffer.concat(chunks).toString("utf8")));
|
|
54898
|
+
} catch (error2) {
|
|
54899
|
+
reject(error2);
|
|
54900
|
+
}
|
|
54901
|
+
});
|
|
54902
|
+
child.stdin?.on("error", () => {
|
|
54903
|
+
});
|
|
54904
|
+
child.stdin?.end(input);
|
|
54905
|
+
});
|
|
54906
|
+
}
|
|
54907
|
+
function prepareCall(action, fields, deps, options) {
|
|
54908
|
+
if (!PUBLIC_HELPER_ACTIONS.has(action))
|
|
54909
|
+
throw new PublicHelperError(
|
|
54910
|
+
"unknown_action",
|
|
54911
|
+
`"${action}" is not an action the server sends to the public helper.`
|
|
54912
|
+
);
|
|
54913
|
+
let binaryPath = options.binaryPath;
|
|
54914
|
+
if (!binaryPath) {
|
|
54915
|
+
const install = inspectPublicHelper(deps);
|
|
54916
|
+
if (!install.ready)
|
|
54917
|
+
throw new PublicHelperError(install.reason ?? "helper_not_installed", install.detail ?? "");
|
|
54918
|
+
binaryPath = install.binaryPath;
|
|
54919
|
+
}
|
|
54920
|
+
const timeout = options.timeoutMs || Number.parseInt(deps.env[PUBLIC_HELPER_TIMEOUT_ENV] || "", 10) || DEFAULT_TIMEOUT_MS2;
|
|
54921
|
+
const input = JSON.stringify({ protocol: PUBLIC_HELPER_PROTOCOL, action, ...fields });
|
|
54922
|
+
return { binaryPath, timeout, input };
|
|
54923
|
+
}
|
|
54924
|
+
function parseHelperOutput(status, stdout) {
|
|
54809
54925
|
let parsed;
|
|
54810
54926
|
try {
|
|
54811
|
-
parsed = JSON.parse(
|
|
54927
|
+
parsed = JSON.parse(stdout.trim());
|
|
54812
54928
|
} catch {
|
|
54813
54929
|
throw new PublicHelperError(
|
|
54814
54930
|
"invalid_response",
|
|
54815
|
-
`The helper exited with status ${
|
|
54931
|
+
`The helper exited with status ${status} and no JSON response.`
|
|
54816
54932
|
);
|
|
54817
54933
|
}
|
|
54818
54934
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
54819
54935
|
throw new PublicHelperError("invalid_response", "The helper response is not a JSON object.");
|
|
54820
54936
|
const object3 = parsed;
|
|
54821
|
-
if (
|
|
54937
|
+
if (status !== 0 || object3.status !== "ok") {
|
|
54822
54938
|
const error2 = errorSchema.safeParse(object3);
|
|
54823
54939
|
if (!error2.success)
|
|
54824
54940
|
throw new PublicHelperError(
|
|
54825
54941
|
"invalid_response",
|
|
54826
|
-
`The helper failed with an unrecognized response (exit ${
|
|
54942
|
+
`The helper failed with an unrecognized response (exit ${status}).`
|
|
54827
54943
|
);
|
|
54828
54944
|
throw new PublicHelperError(error2.data.code, error2.data.message);
|
|
54829
54945
|
}
|
|
@@ -54870,6 +54986,10 @@ function publicHelperCompileArguments(sourcePath, digestPath, plistPath, outputP
|
|
|
54870
54986
|
"AppKit",
|
|
54871
54987
|
"-framework",
|
|
54872
54988
|
"PencilKit",
|
|
54989
|
+
"-framework",
|
|
54990
|
+
"AVFoundation",
|
|
54991
|
+
"-framework",
|
|
54992
|
+
"Speech",
|
|
54873
54993
|
"-Xlinker",
|
|
54874
54994
|
"-sectcreate",
|
|
54875
54995
|
"-Xlinker",
|
|
@@ -55236,6 +55356,313 @@ function formatNoteDrawings(result) {
|
|
|
55236
55356
|
return lines.join("\n");
|
|
55237
55357
|
}
|
|
55238
55358
|
|
|
55359
|
+
// src/utils/noteAudio.ts
|
|
55360
|
+
import { readdirSync as readdirSync4, statSync as statSync4 } from "node:fs";
|
|
55361
|
+
import { join as join25 } from "node:path";
|
|
55362
|
+
var EXTRA_AUDIO_UTIS = [
|
|
55363
|
+
"public.mp3",
|
|
55364
|
+
"public.aiff-audio",
|
|
55365
|
+
"public.aifc-audio",
|
|
55366
|
+
"com.microsoft.waveform-audio"
|
|
55367
|
+
];
|
|
55368
|
+
function audioRowsSql(columns, generationColumn) {
|
|
55369
|
+
const generation = generationColumn ? `m.${generationColumn}` : "NULL";
|
|
55370
|
+
const media = (alias) => `json((SELECT json_object('identifier', m.ZIDENTIFIER, 'generation', ${generation}, 'filename', m.ZFILENAME) FROM ZICCLOUDSYNCINGOBJECT m WHERE m.Z_PK = ${alias}.ZMEDIA))`;
|
|
55371
|
+
const live = (alias) => notTombstonedSql(columns, alias);
|
|
55372
|
+
const utis = EXTRA_AUDIO_UTIS.map((u) => `'${u}'`).join(", ");
|
|
55373
|
+
const accountCols = [...columns].filter((c) => /^ZACCOUNT\d*$/.test(c)).sort();
|
|
55374
|
+
const account = accountCols.length ? `SELECT (SELECT acc.ZIDENTIFIER FROM ZICCLOUDSYNCINGOBJECT acc WHERE acc.Z_ENT = ${entity("ICAccount")} AND acc.Z_PK IN (${accountCols.map((c) => `n.${c}`).join(", ")}) LIMIT 1) FROM ZICCLOUDSYNCINGOBJECT n WHERE n.Z_PK = @pk;` : "SELECT NULL;";
|
|
55375
|
+
return [
|
|
55376
|
+
NOTE_STATE_SQL,
|
|
55377
|
+
`SELECT json_group_array(json_object('pk', a.Z_PK, 'identifier', a.ZIDENTIFIER, 'uti', a.ZTYPEUTI, 'duration', a.ZDURATION, 'media', ${media("a")}, 'children', json((SELECT json_group_array(json_object('pk', c.Z_PK, 'identifier', c.ZIDENTIFIER, 'duration', c.ZDURATION, 'media', ${media("c")})) FROM (SELECT * FROM ZICCLOUDSYNCINGOBJECT k WHERE k.ZPARENTATTACHMENT = a.Z_PK AND ${live("k")} ORDER BY k.Z_PK) c)))) FROM (SELECT * FROM ZICCLOUDSYNCINGOBJECT t WHERE t.ZNOTE = @pk AND t.ZPARENTATTACHMENT IS NULL AND ${live("t")} AND (t.ZTYPEUTI LIKE '%audio%' OR t.ZTYPEUTI IN (${utis})) ORDER BY t.Z_PK) a;`,
|
|
55378
|
+
account
|
|
55379
|
+
].join("\n");
|
|
55380
|
+
}
|
|
55381
|
+
function accountDirsFor(containerDir, accountIdentifier) {
|
|
55382
|
+
const own = resolveAccountDir(containerDir, accountIdentifier);
|
|
55383
|
+
if (own) return [own];
|
|
55384
|
+
let names;
|
|
55385
|
+
try {
|
|
55386
|
+
names = readdirSync4(join25(containerDir, "Accounts")).sort();
|
|
55387
|
+
} catch {
|
|
55388
|
+
return [];
|
|
55389
|
+
}
|
|
55390
|
+
return names.map((name) => safeComponent(name) ? resolveAccountDir(containerDir, name) : null).filter((dir) => dir !== null);
|
|
55391
|
+
}
|
|
55392
|
+
function resolveMediaPath(media, accountIdentifier = null, containerDir = NOTES_CONTAINER_DIR) {
|
|
55393
|
+
const id2 = safeComponent(media?.identifier);
|
|
55394
|
+
const filename = safeComponent(media?.filename);
|
|
55395
|
+
if (!id2 || !filename) return null;
|
|
55396
|
+
const generation = safeComponent(media?.generation);
|
|
55397
|
+
for (const dir of accountDirsFor(containerDir, accountIdentifier)) {
|
|
55398
|
+
const base = join25(dir, "Media", id2);
|
|
55399
|
+
const candidates = generation ? [join25(base, generation, filename), join25(base, filename)] : [join25(base, filename)];
|
|
55400
|
+
for (const candidate of candidates) {
|
|
55401
|
+
const real = realInside(candidate, dir);
|
|
55402
|
+
if (real && isFile2(real)) return real;
|
|
55403
|
+
}
|
|
55404
|
+
}
|
|
55405
|
+
return null;
|
|
55406
|
+
}
|
|
55407
|
+
function isFile2(path10) {
|
|
55408
|
+
try {
|
|
55409
|
+
return statSync4(path10).isFile();
|
|
55410
|
+
} catch {
|
|
55411
|
+
return false;
|
|
55412
|
+
}
|
|
55413
|
+
}
|
|
55414
|
+
var positive = (value) => typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
55415
|
+
function readAudioAssets(noteId3, options = {}) {
|
|
55416
|
+
const { store, pk } = parseNoteId2(noteId3);
|
|
55417
|
+
const dbPath2 = options.dbPath ?? NOTES_DB_PATH7;
|
|
55418
|
+
const columns = readColumns(dbPath2);
|
|
55419
|
+
const missing = ["ZMEDIA", "ZPARENTATTACHMENT", "ZFILENAME"].filter((c) => !columns.has(c));
|
|
55420
|
+
if (missing.length)
|
|
55421
|
+
throw new NoteStoreError(
|
|
55422
|
+
`This macOS version's Notes database lacks ${missing.join(", ")}; audio files cannot be located.`,
|
|
55423
|
+
"schema"
|
|
55424
|
+
);
|
|
55425
|
+
const generation = columns.has("ZGENERATION1") ? "ZGENERATION1" : columns.has("ZGENERATION") ? "ZGENERATION" : null;
|
|
55426
|
+
const [stateLine, rowsLine, accountLine] = queryNoteScoped(
|
|
55427
|
+
audioRowsSql(columns, generation),
|
|
55428
|
+
pk,
|
|
55429
|
+
dbPath2
|
|
55430
|
+
);
|
|
55431
|
+
assertNoteReadable(stateLine, noteId3);
|
|
55432
|
+
const rows = JSON.parse(rowsLine || "[]");
|
|
55433
|
+
const account = accountLine?.trim() || null;
|
|
55434
|
+
const attachmentId = (rowPk) => `x-coredata://${store}/ICAttachment/p${rowPk}`;
|
|
55435
|
+
return rows.map((row) => {
|
|
55436
|
+
const sources = row.children?.length ? row.children : [{ pk: row.pk, identifier: row.identifier, duration: row.duration, media: row.media }];
|
|
55437
|
+
const takes = sources.map((take) => ({
|
|
55438
|
+
attachmentId: attachmentId(take.pk),
|
|
55439
|
+
identifier: take.identifier ?? "",
|
|
55440
|
+
durationSeconds: positive(take.duration),
|
|
55441
|
+
path: resolveMediaPath(take.media, account, options.containerDir)
|
|
55442
|
+
}));
|
|
55443
|
+
const total = takes.reduce((sum, t) => sum + (t.durationSeconds ?? 0), 0);
|
|
55444
|
+
return {
|
|
55445
|
+
pk: row.pk,
|
|
55446
|
+
attachmentId: attachmentId(row.pk),
|
|
55447
|
+
identifier: row.identifier ?? "",
|
|
55448
|
+
typeUti: row.uti,
|
|
55449
|
+
durationSeconds: positive(row.duration) ?? positive(total),
|
|
55450
|
+
takes
|
|
55451
|
+
};
|
|
55452
|
+
});
|
|
55453
|
+
}
|
|
55454
|
+
function countWords2(text2) {
|
|
55455
|
+
return text2.split(/\s+/u).filter((w) => /[\p{L}\p{N}]/u.test(w)).length;
|
|
55456
|
+
}
|
|
55457
|
+
|
|
55458
|
+
// src/services/noteTranscription.ts
|
|
55459
|
+
var DEFAULT_TRANSCRIPTION_LOCALE = "en-US";
|
|
55460
|
+
var LOCALE_PATTERN = /^[A-Za-z]{2,3}(?:[-_][A-Za-z0-9]{2,8}){0,3}$/;
|
|
55461
|
+
var TRANSCRIBE_MAX_SECONDS = { default: 900, min: 30, max: 3600 };
|
|
55462
|
+
function takeDeadlineSeconds(durationSeconds) {
|
|
55463
|
+
const duration3 = durationSeconds ?? 600;
|
|
55464
|
+
return Math.min(1800, Math.max(60, Math.ceil(duration3 * 1.5) + 60));
|
|
55465
|
+
}
|
|
55466
|
+
var KILL_GRACE_MS = 3e4;
|
|
55467
|
+
var BUDGET_MARGIN_MS = 1e4;
|
|
55468
|
+
var MIN_TAKE_SECONDS = 5;
|
|
55469
|
+
var CALL_WIDE_CODES = /* @__PURE__ */ new Set([
|
|
55470
|
+
"permission_required",
|
|
55471
|
+
"asset_unavailable",
|
|
55472
|
+
"unsupported_locale",
|
|
55473
|
+
"speech_unavailable"
|
|
55474
|
+
]);
|
|
55475
|
+
var helperTranscriptSchema = external_exports.object({
|
|
55476
|
+
status: external_exports.literal("ok"),
|
|
55477
|
+
transcript: external_exports.string(),
|
|
55478
|
+
complete: external_exports.boolean(),
|
|
55479
|
+
stopReason: external_exports.string().optional(),
|
|
55480
|
+
engine: external_exports.string().optional(),
|
|
55481
|
+
durationSeconds: external_exports.number().optional()
|
|
55482
|
+
});
|
|
55483
|
+
async function transcribeTake(take, ctx) {
|
|
55484
|
+
const base = {
|
|
55485
|
+
attachmentId: take.attachmentId,
|
|
55486
|
+
identifier: take.identifier,
|
|
55487
|
+
status: "error",
|
|
55488
|
+
...take.durationSeconds !== null ? { durationSeconds: Math.round(take.durationSeconds) } : {}
|
|
55489
|
+
};
|
|
55490
|
+
const failed = (code, message) => ({
|
|
55491
|
+
take: { ...base, code, message },
|
|
55492
|
+
text: ""
|
|
55493
|
+
});
|
|
55494
|
+
if (!take.path)
|
|
55495
|
+
return failed(
|
|
55496
|
+
"asset_unavailable",
|
|
55497
|
+
"The audio file is not on this Mac (it may not have downloaded from iCloud yet)."
|
|
55498
|
+
);
|
|
55499
|
+
if (ctx.callWide) return failed(ctx.callWide.code, ctx.callWide.message);
|
|
55500
|
+
const remainingMs = ctx.deadlineAt - ctx.now();
|
|
55501
|
+
const deadline = Math.min(
|
|
55502
|
+
takeDeadlineSeconds(take.durationSeconds),
|
|
55503
|
+
Math.floor((remainingMs - BUDGET_MARGIN_MS) / 1e3)
|
|
55504
|
+
);
|
|
55505
|
+
if (deadline < MIN_TAKE_SECONDS)
|
|
55506
|
+
return failed(
|
|
55507
|
+
"time_limit",
|
|
55508
|
+
`Not started: the call's ${ctx.maxSeconds}-second limit was reached. Transcribe this recording on its own with attachmentId, or raise maxSeconds.`
|
|
55509
|
+
);
|
|
55510
|
+
let raw;
|
|
55511
|
+
try {
|
|
55512
|
+
raw = await callPublicHelperAsync(
|
|
55513
|
+
"transcribe",
|
|
55514
|
+
{
|
|
55515
|
+
path: take.path,
|
|
55516
|
+
locale: ctx.locale,
|
|
55517
|
+
timeoutSeconds: deadline,
|
|
55518
|
+
downloadAssets: ctx.downloadAssets
|
|
55519
|
+
},
|
|
55520
|
+
ctx.deps,
|
|
55521
|
+
{ timeoutMs: Math.min(deadline * 1e3 + KILL_GRACE_MS, remainingMs), signal: ctx.signal }
|
|
55522
|
+
);
|
|
55523
|
+
} catch (error2) {
|
|
55524
|
+
const code = error2 instanceof PublicHelperError ? error2.code : "internal_error";
|
|
55525
|
+
if (code === "aborted") throw error2;
|
|
55526
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
55527
|
+
if (CALL_WIDE_CODES.has(code)) ctx.callWide = { code, message };
|
|
55528
|
+
return {
|
|
55529
|
+
take: {
|
|
55530
|
+
...base,
|
|
55531
|
+
// A timeout means the helper was killed mid-run: the outcome is unknown.
|
|
55532
|
+
status: code === "timeout" ? "indeterminate" : "error",
|
|
55533
|
+
code,
|
|
55534
|
+
message
|
|
55535
|
+
},
|
|
55536
|
+
text: ""
|
|
55537
|
+
};
|
|
55538
|
+
}
|
|
55539
|
+
const parsed = helperTranscriptSchema.safeParse(raw);
|
|
55540
|
+
if (!parsed.success) return failed("invalid_response", "Unexpected helper response.");
|
|
55541
|
+
const { transcript, complete, stopReason, engine, durationSeconds } = parsed.data;
|
|
55542
|
+
const text2 = transcript.trim();
|
|
55543
|
+
return {
|
|
55544
|
+
take: {
|
|
55545
|
+
...base,
|
|
55546
|
+
status: complete ? "ok" : text2 ? "partial" : "indeterminate",
|
|
55547
|
+
...complete ? {} : { code: "incomplete", message: stopReason ?? "stopped early" },
|
|
55548
|
+
...durationSeconds !== void 0 ? { durationSeconds } : {},
|
|
55549
|
+
wordCount: countWords2(text2),
|
|
55550
|
+
...engine ? { engine } : {}
|
|
55551
|
+
},
|
|
55552
|
+
text: text2
|
|
55553
|
+
};
|
|
55554
|
+
}
|
|
55555
|
+
function combineStatus(parts) {
|
|
55556
|
+
if (parts.length && parts.every((p) => p.status === "ok")) return "ok";
|
|
55557
|
+
if (parts.some((p) => p.hasText)) return "partial";
|
|
55558
|
+
if (parts.some((p) => p.status === "indeterminate")) return "indeterminate";
|
|
55559
|
+
return "error";
|
|
55560
|
+
}
|
|
55561
|
+
async function transcribeRecording(asset, includeText, ctx) {
|
|
55562
|
+
const outcomes = [];
|
|
55563
|
+
for (const take of asset.takes) outcomes.push(await transcribeTake(take, ctx));
|
|
55564
|
+
const transcript = outcomes.map((o) => o.text).filter(Boolean).join("\n\n");
|
|
55565
|
+
const status = combineStatus(
|
|
55566
|
+
outcomes.map((o) => ({ status: o.take.status, hasText: o.text.length > 0 }))
|
|
55567
|
+
);
|
|
55568
|
+
const firstProblem = outcomes.find((o) => o.take.status !== "ok")?.take;
|
|
55569
|
+
return {
|
|
55570
|
+
attachmentId: asset.attachmentId,
|
|
55571
|
+
identifier: asset.identifier,
|
|
55572
|
+
typeUti: asset.typeUti,
|
|
55573
|
+
status,
|
|
55574
|
+
...status !== "ok" && firstProblem?.code ? { code: firstProblem.code, message: firstProblem.message } : {},
|
|
55575
|
+
...asset.durationSeconds !== null ? { durationSeconds: Math.round(asset.durationSeconds) } : {},
|
|
55576
|
+
wordCount: countWords2(transcript),
|
|
55577
|
+
...includeText ? { transcript } : {},
|
|
55578
|
+
takes: outcomes.map((o) => o.take)
|
|
55579
|
+
};
|
|
55580
|
+
}
|
|
55581
|
+
async function transcribeNoteAudio(noteId3, options = {}) {
|
|
55582
|
+
const locale = options.locale ?? DEFAULT_TRANSCRIPTION_LOCALE;
|
|
55583
|
+
if (!LOCALE_PATTERN.test(locale))
|
|
55584
|
+
throw new PublicHelperError("invalid_request", `"${locale}" is not a BCP-47 locale.`);
|
|
55585
|
+
const maxSeconds = options.maxSeconds ?? TRANSCRIBE_MAX_SECONDS.default;
|
|
55586
|
+
if (!Number.isInteger(maxSeconds) || maxSeconds < TRANSCRIBE_MAX_SECONDS.min || maxSeconds > TRANSCRIBE_MAX_SECONDS.max)
|
|
55587
|
+
throw new PublicHelperError(
|
|
55588
|
+
"invalid_request",
|
|
55589
|
+
`maxSeconds must be a whole number from ${TRANSCRIBE_MAX_SECONDS.min} to ${TRANSCRIBE_MAX_SECONDS.max}.`
|
|
55590
|
+
);
|
|
55591
|
+
const deps = options.deps ?? defaultPublicHelperDeps();
|
|
55592
|
+
let assets = (options.readAssets ?? ((id2) => readAudioAssets(id2)))(noteId3);
|
|
55593
|
+
if (options.attachmentId) {
|
|
55594
|
+
assets = assets.filter((a) => a.attachmentId === options.attachmentId);
|
|
55595
|
+
if (assets.length === 0)
|
|
55596
|
+
throw new PublicHelperError(
|
|
55597
|
+
"attachment_not_found",
|
|
55598
|
+
`No audio attachment ${options.attachmentId} in note ${noteId3}.`
|
|
55599
|
+
);
|
|
55600
|
+
}
|
|
55601
|
+
const empty = { id: noteId3, locale, recordingCount: 0, recordings: [] };
|
|
55602
|
+
if (assets.length === 0) return { ...empty, status: "none" };
|
|
55603
|
+
const install = inspectPublicHelper(deps);
|
|
55604
|
+
if (!install.ready)
|
|
55605
|
+
throw new PublicHelperError(install.reason ?? "helper_not_installed", install.detail ?? "");
|
|
55606
|
+
const now = options.now ?? Date.now;
|
|
55607
|
+
const ctx = {
|
|
55608
|
+
locale,
|
|
55609
|
+
downloadAssets: options.downloadAssets ?? false,
|
|
55610
|
+
deps,
|
|
55611
|
+
signal: options.signal,
|
|
55612
|
+
now,
|
|
55613
|
+
deadlineAt: now() + maxSeconds * 1e3,
|
|
55614
|
+
maxSeconds,
|
|
55615
|
+
callWide: null
|
|
55616
|
+
};
|
|
55617
|
+
const recordings = [];
|
|
55618
|
+
for (const asset of assets)
|
|
55619
|
+
recordings.push(await transcribeRecording(asset, options.includeText ?? true, ctx));
|
|
55620
|
+
return {
|
|
55621
|
+
...empty,
|
|
55622
|
+
status: combineStatus(
|
|
55623
|
+
recordings.map((r) => ({ status: r.status, hasText: (r.wordCount ?? 0) > 0 }))
|
|
55624
|
+
),
|
|
55625
|
+
recordingCount: recordings.length,
|
|
55626
|
+
recordings
|
|
55627
|
+
};
|
|
55628
|
+
}
|
|
55629
|
+
function fitTranscriptions(result, maxBytes, measure = (r) => Buffer.byteLength(JSON.stringify(r))) {
|
|
55630
|
+
if (measure(result) <= maxBytes) return result;
|
|
55631
|
+
let next = result;
|
|
55632
|
+
for (let share = 0.5; measure(next) > maxBytes && share > 1e-4; share /= 2) {
|
|
55633
|
+
next = {
|
|
55634
|
+
...result,
|
|
55635
|
+
recordings: result.recordings.map(
|
|
55636
|
+
(r) => r.transcript ? {
|
|
55637
|
+
...r,
|
|
55638
|
+
transcript: r.transcript.slice(0, Math.floor(r.transcript.length * share)),
|
|
55639
|
+
transcriptTruncated: true
|
|
55640
|
+
} : r
|
|
55641
|
+
)
|
|
55642
|
+
};
|
|
55643
|
+
}
|
|
55644
|
+
return next;
|
|
55645
|
+
}
|
|
55646
|
+
function formatTranscription(result) {
|
|
55647
|
+
if (result.recordingCount === 0) return `No audio attachments in note ${result.id}.`;
|
|
55648
|
+
const lines = [
|
|
55649
|
+
`${result.recordingCount} audio attachment${result.recordingCount === 1 ? "" : "s"} in note ${result.id} (${result.status}, locale ${result.locale}):`
|
|
55650
|
+
];
|
|
55651
|
+
result.recordings.forEach((r, i) => {
|
|
55652
|
+
const facts = [
|
|
55653
|
+
`status ${r.status}`,
|
|
55654
|
+
...r.durationSeconds !== void 0 ? [`${r.durationSeconds} s`] : [],
|
|
55655
|
+
`${r.wordCount ?? 0} words`,
|
|
55656
|
+
...r.takes.length > 1 ? [`${r.takes.length} takes`] : [],
|
|
55657
|
+
...r.code ? [`${r.code}: ${r.message}`] : []
|
|
55658
|
+
];
|
|
55659
|
+
lines.push("", `[${i + 1}] ${r.attachmentId} (${facts.join(", ")})`);
|
|
55660
|
+
if (r.transcript)
|
|
55661
|
+
lines.push(r.transcriptTruncated ? `${r.transcript} [truncated]` : r.transcript);
|
|
55662
|
+
});
|
|
55663
|
+
return lines.join("\n");
|
|
55664
|
+
}
|
|
55665
|
+
|
|
55239
55666
|
// src/services/privateHelperBuild.ts
|
|
55240
55667
|
import { spawnSync as spawnSync5 } from "node:child_process";
|
|
55241
55668
|
import {
|
|
@@ -55248,14 +55675,14 @@ import {
|
|
|
55248
55675
|
writeFileSync as writeFileSync5
|
|
55249
55676
|
} from "node:fs";
|
|
55250
55677
|
import { release as release4 } from "node:os";
|
|
55251
|
-
import { join as
|
|
55678
|
+
import { join as join27 } from "node:path";
|
|
55252
55679
|
|
|
55253
55680
|
// src/services/privateHelper.ts
|
|
55254
55681
|
import { spawnSync as spawnSync4 } from "node:child_process";
|
|
55255
55682
|
import { createHash as createHash6 } from "node:crypto";
|
|
55256
55683
|
import { existsSync as existsSync15, readFileSync as readFileSync5 } from "node:fs";
|
|
55257
55684
|
import { homedir as homedir20 } from "node:os";
|
|
55258
|
-
import { dirname as dirname9, join as
|
|
55685
|
+
import { dirname as dirname9, join as join26, resolve as resolve6 } from "node:path";
|
|
55259
55686
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
55260
55687
|
var PRIVATE_HELPER_PROTOCOL = 1;
|
|
55261
55688
|
var ENABLE_ENV = "APPLE_NOTES_MCP_ENABLE_PRIVATE";
|
|
@@ -55283,7 +55710,7 @@ var manifestSchema = external_exports.object({
|
|
|
55283
55710
|
function packageRoot2(fromDir = dirname9(fileURLToPath3(import.meta.url))) {
|
|
55284
55711
|
let dir = fromDir;
|
|
55285
55712
|
for (; ; ) {
|
|
55286
|
-
const candidate =
|
|
55713
|
+
const candidate = join26(dir, "package.json");
|
|
55287
55714
|
if (existsSync15(candidate)) {
|
|
55288
55715
|
try {
|
|
55289
55716
|
const pkg = JSON.parse(readFileSync5(candidate, "utf8"));
|
|
@@ -55300,7 +55727,7 @@ function defaultDeps2(overrides = {}) {
|
|
|
55300
55727
|
return {
|
|
55301
55728
|
env: process.env,
|
|
55302
55729
|
platform: process.platform,
|
|
55303
|
-
sourcePath:
|
|
55730
|
+
sourcePath: join26(packageRoot2(), HELPER_SOURCE_RELATIVE),
|
|
55304
55731
|
exists: existsSync15,
|
|
55305
55732
|
readFile: (path10) => readFileSync5(path10),
|
|
55306
55733
|
spawn: spawnSync4,
|
|
@@ -55313,14 +55740,14 @@ function privateHelperEnabled(env = process.env) {
|
|
|
55313
55740
|
function helperInstallDir(env = process.env) {
|
|
55314
55741
|
const override = env[HELPER_DIR_ENV]?.trim();
|
|
55315
55742
|
if (override) return override;
|
|
55316
|
-
return
|
|
55743
|
+
return join26(homedir20(), "Library", "Application Support", "apple-notes-mcp", "private-helper");
|
|
55317
55744
|
}
|
|
55318
55745
|
function sha256Hex2(data) {
|
|
55319
55746
|
return createHash6("sha256").update(data).digest("hex");
|
|
55320
55747
|
}
|
|
55321
55748
|
function inspectInstallation(deps = defaultDeps2()) {
|
|
55322
55749
|
const installDir = helperInstallDir(deps.env);
|
|
55323
|
-
const binaryPath =
|
|
55750
|
+
const binaryPath = join26(installDir, HELPER_BINARY_NAME);
|
|
55324
55751
|
const base = {
|
|
55325
55752
|
installDir,
|
|
55326
55753
|
binaryPath,
|
|
@@ -55338,7 +55765,7 @@ function inspectInstallation(deps = defaultDeps2()) {
|
|
|
55338
55765
|
if (!deps.exists(deps.sourcePath))
|
|
55339
55766
|
return fail("helper_not_installed", `Packaged helper source is missing: ${deps.sourcePath}`);
|
|
55340
55767
|
base.expectedSourceSha256 = sha256Hex2(deps.readFile(deps.sourcePath));
|
|
55341
|
-
const manifestPath =
|
|
55768
|
+
const manifestPath = join26(installDir, MANIFEST_NAME);
|
|
55342
55769
|
if (!deps.exists(binaryPath) || !deps.exists(manifestPath))
|
|
55343
55770
|
return fail(
|
|
55344
55771
|
"helper_not_installed",
|
|
@@ -55653,9 +56080,9 @@ function buildPrivateHelper(checkOnly, deps = defaultBuildDeps()) {
|
|
|
55653
56080
|
steps.push({ step: "find compiler", ok: true, detail: compiler });
|
|
55654
56081
|
const installDir = helperInstallDir(deps.env);
|
|
55655
56082
|
mkdirSync7(installDir, { recursive: true, mode: 448 });
|
|
55656
|
-
const staging = mkdtempSync6(
|
|
56083
|
+
const staging = mkdtempSync6(join27(installDir, ".staging-"));
|
|
55657
56084
|
try {
|
|
55658
|
-
const stagedBinary =
|
|
56085
|
+
const stagedBinary = join27(staging, HELPER_BINARY_NAME);
|
|
55659
56086
|
const compile = deps.spawn(
|
|
55660
56087
|
"/usr/bin/xcrun",
|
|
55661
56088
|
compileArguments(deps.sourcePath, stagedBinary, sourceSha),
|
|
@@ -55732,8 +56159,8 @@ function buildPrivateHelper(checkOnly, deps = defaultBuildDeps()) {
|
|
|
55732
56159
|
compiler
|
|
55733
56160
|
};
|
|
55734
56161
|
chmodSync2(stagedBinary, 448);
|
|
55735
|
-
renameSync2(stagedBinary,
|
|
55736
|
-
writeFileSync5(
|
|
56162
|
+
renameSync2(stagedBinary, join27(installDir, HELPER_BINARY_NAME));
|
|
56163
|
+
writeFileSync5(join27(installDir, MANIFEST_NAME), JSON.stringify(manifest, null, 2) + "\n", {
|
|
55737
56164
|
mode: 384
|
|
55738
56165
|
});
|
|
55739
56166
|
steps.push({ step: "install", ok: true, detail: installDir });
|
|
@@ -55927,6 +56354,16 @@ function withErrorHandling(handler, errorPrefix) {
|
|
|
55927
56354
|
}
|
|
55928
56355
|
};
|
|
55929
56356
|
}
|
|
56357
|
+
function withAsyncErrorHandling(handler, errorPrefix) {
|
|
56358
|
+
return async (params, extra) => {
|
|
56359
|
+
try {
|
|
56360
|
+
return await handler(params, extra?.signal);
|
|
56361
|
+
} catch (error2) {
|
|
56362
|
+
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
56363
|
+
return errorResponse(`${errorPrefix}: ${message}`, error2);
|
|
56364
|
+
}
|
|
56365
|
+
};
|
|
56366
|
+
}
|
|
55930
56367
|
var MAX = {
|
|
55931
56368
|
TITLE: 2e3,
|
|
55932
56369
|
CONTENT: 5 * 1024 * 1024,
|
|
@@ -56010,11 +56447,25 @@ function readExactNoteSnapshot(id2) {
|
|
|
56010
56447
|
error: `Note "${note.title}" is password-protected and cannot be changed. Unlock it in Notes.app first.`
|
|
56011
56448
|
};
|
|
56012
56449
|
}
|
|
56013
|
-
const body = notesManager.
|
|
56014
|
-
if (!body)
|
|
56450
|
+
const { body, error: error2 } = notesManager.readNoteBodyById(id2);
|
|
56451
|
+
if (!body) {
|
|
56452
|
+
const reason = bodyReadFailureMessage(id2, note.title, error2);
|
|
56453
|
+
return { error: `${reason}${error2 ? "\n\nNothing was changed." : ""}` };
|
|
56454
|
+
}
|
|
56015
56455
|
const rich = enrichNoteRead(id2, body);
|
|
56016
56456
|
return { note, body, rich, contentHash: richContentHash(body, rich) };
|
|
56017
56457
|
}
|
|
56458
|
+
function bodyReadFailureMessage(id2, title, error2) {
|
|
56459
|
+
let attachments;
|
|
56460
|
+
if (classifyBodyReadError(error2) !== "other") {
|
|
56461
|
+
try {
|
|
56462
|
+
attachments = readNoteStructure(id2, { includeText: false }).attachments;
|
|
56463
|
+
} catch {
|
|
56464
|
+
attachments = void 0;
|
|
56465
|
+
}
|
|
56466
|
+
}
|
|
56467
|
+
return describeBodyReadFailure(title, error2, attachments);
|
|
56468
|
+
}
|
|
56018
56469
|
var NOT_DELETED_MESSAGE = "Notes.app accepted the delete, but the note is still in its original folder, so it was not moved to Recently Deleted. Nothing was deleted; read the note again before retrying.";
|
|
56019
56470
|
function containerUnknownMessage(title) {
|
|
56020
56471
|
return `Could not read which folder note "${title}" is in, so it may already be in Recently Deleted, where deleting it would remove it permanently. Nothing was deleted. Retry in a moment; if it keeps failing, quit and reopen Notes.app, then retry.`;
|
|
@@ -56428,13 +56879,14 @@ ${lines}${footer}`,
|
|
|
56428
56879
|
registerTool(
|
|
56429
56880
|
"get-note-content",
|
|
56430
56881
|
{
|
|
56431
|
-
description: "Use when: reading the full body text of one known note, by id (preferred) or title.\nReturns: the exact note id, content, contentHash revision token, parsed hashtags, nativeTags, restored links, richContentComplete/writable, and strippedImages/truncated when the body was capped. Read the warning when writable is false.\nDo not use when: you only need metadata (get-note-details) or Markdown with checklist state (get-note-markdown).\nNote: password-protected notes must be unlocked in Notes.app first.\nSafety: inline images larger than APPLE_NOTES_MCP_MAX_INLINE_IMAGE_BYTES (default 256 KB) are replaced with '[inline image omitted: ...]' text placeholders, so the returned body is lossy whenever truncated is true. Mutations refuse attachment-bearing notes; edit those in Notes.app.",
|
|
56882
|
+
description: "Use when: reading the full body text of one known note, by id (preferred) or title.\nReturns: the exact note id, content, contentHash revision token, parsed hashtags, nativeTags, restored links, richContentComplete/writable, and strippedImages/truncated when the body was capped. Read the warning when writable is false.\nDo not use when: you only need metadata (get-note-details) or Markdown with checklist state (get-note-markdown).\nNote: password-protected notes must be unlocked in Notes.app first.\nSafety: inline images larger than APPLE_NOTES_MCP_MAX_INLINE_IMAGE_BYTES (default 256 KB) are replaced with '[inline image omitted: ...]' text placeholders, so the returned body is lossy whenever truncated is true. Mutations refuse attachment-bearing notes; edit those in Notes.app. Notes.app returns images inside the body as base64, so a note with a very large image can time out; the error then names the cause, and a larger timeoutSeconds gives the read more time.",
|
|
56432
56883
|
inputSchema: {
|
|
56433
56884
|
id: looseNoteId(external_exports.string()).optional().describe(`Note ID (preferred - more reliable than title): ${NOTE_ID_FORMS}`),
|
|
56434
56885
|
title: external_exports.string().max(MAX.TITLE).optional().describe("Note title (use id instead when available)"),
|
|
56435
56886
|
account: external_exports.string().max(MAX.ACCOUNT).optional().describe(
|
|
56436
56887
|
"Account name (defaults to Notes.app's default account; exact or unique-prefix match, ignored if id is provided)"
|
|
56437
|
-
)
|
|
56888
|
+
),
|
|
56889
|
+
timeoutSeconds: timeoutSecondsInput
|
|
56438
56890
|
},
|
|
56439
56891
|
outputSchema: {
|
|
56440
56892
|
id: external_exports.string().optional(),
|
|
@@ -56467,9 +56919,9 @@ registerTool(
|
|
|
56467
56919
|
`Note "${note2.title}" is password-protected and cannot be read. Unlock it in Notes.app first.`
|
|
56468
56920
|
);
|
|
56469
56921
|
}
|
|
56470
|
-
const rawContent2 = notesManager.
|
|
56922
|
+
const { body: rawContent2, error: readError2 } = notesManager.readNoteBodyById(id2);
|
|
56471
56923
|
if (!rawContent2) {
|
|
56472
|
-
return errorResponse(
|
|
56924
|
+
return errorResponse(bodyReadFailureMessage(id2, note2.title, readError2));
|
|
56473
56925
|
}
|
|
56474
56926
|
const rich2 = enrichNoteRead(id2, rawContent2);
|
|
56475
56927
|
const stripped2 = stripLargeInlineImages(rich2.content);
|
|
@@ -56509,9 +56961,9 @@ registerTool(
|
|
|
56509
56961
|
`Note "${title}" is password-protected and cannot be read. Unlock it in Notes.app first.`
|
|
56510
56962
|
);
|
|
56511
56963
|
}
|
|
56512
|
-
const rawContent = notesManager.
|
|
56964
|
+
const { body: rawContent, error: readError } = notesManager.readNoteBodyById(note.id);
|
|
56513
56965
|
if (!rawContent) {
|
|
56514
|
-
return errorResponse(
|
|
56966
|
+
return errorResponse(bodyReadFailureMessage(note.id, title, readError));
|
|
56515
56967
|
}
|
|
56516
56968
|
const rich = enrichNoteRead(note.id, rawContent);
|
|
56517
56969
|
const stripped = stripLargeInlineImages(rich.content);
|
|
@@ -58948,6 +59400,49 @@ registerTool(
|
|
|
58948
59400
|
});
|
|
58949
59401
|
}, "Error reading drawings")
|
|
58950
59402
|
);
|
|
59403
|
+
registerTool(
|
|
59404
|
+
"transcribe-note-audio",
|
|
59405
|
+
{
|
|
59406
|
+
description: "Use when: you need the words spoken in a note's voice recordings or audio attachments, transcribed now on this Mac, by note id.\nReturns: per audio attachment a status (ok / partial / error / indeterminate), duration, word count, per-take results, and the transcript; overall status ok / partial / error / indeterminate / none.\nDo not use when: you only need the audio file (save-attachment) or a note has no audio.\nNote: read-only; recognition runs entirely on-device (never sent to a server). Needs Full Disk Access and the public native helper built once with `apple-notes-mcp setup --public-helper`. Long recordings take time: pass attachmentId to transcribe one at a time. An indeterminate result means the helper timed out; retrying may succeed. Never prompts: code permission_required means the user must allow the host app under System Settings > Privacy & Security > Speech Recognition. asset_unavailable can mean the language's speech model is not installed; pass downloadAssets: true only if the user agrees to the download.",
|
|
59407
|
+
inputSchema: {
|
|
59408
|
+
id: noteIdInput,
|
|
59409
|
+
locale: external_exports.string().max(35).optional().describe('BCP-47 language of the speech, e.g. "en-US" (default), "it-IT", "fr-FR"'),
|
|
59410
|
+
attachmentId: external_exports.string().max(MAX.ATTACHMENT_ID).optional().describe("Only transcribe this audio attachment (x-coredata ICAttachment id)"),
|
|
59411
|
+
includeText: external_exports.boolean().optional().describe("Include transcript text (default true); false returns statuses and counts only"),
|
|
59412
|
+
downloadAssets: external_exports.boolean().optional().describe(
|
|
59413
|
+
"Let macOS download the language's on-device speech model if it is missing (default false: returns asset_unavailable at once)"
|
|
59414
|
+
),
|
|
59415
|
+
maxSeconds: external_exports.number().int().min(TRANSCRIBE_MAX_SECONDS.min).max(TRANSCRIBE_MAX_SECONDS.max).optional().describe(
|
|
59416
|
+
`Total time budget for the call in seconds (default ${TRANSCRIBE_MAX_SECONDS.default}); recordings not started in time report code time_limit`
|
|
59417
|
+
)
|
|
59418
|
+
},
|
|
59419
|
+
outputSchema: {
|
|
59420
|
+
id: external_exports.string().optional(),
|
|
59421
|
+
locale: external_exports.string().optional(),
|
|
59422
|
+
status: external_exports.string().optional(),
|
|
59423
|
+
recordingCount: external_exports.number().optional(),
|
|
59424
|
+
recordings: external_exports.array(external_exports.object({}).passthrough()).optional()
|
|
59425
|
+
}
|
|
59426
|
+
},
|
|
59427
|
+
withAsyncErrorHandling(async (params, signal) => {
|
|
59428
|
+
const { id: id2, locale, attachmentId, includeText, downloadAssets, maxSeconds } = params;
|
|
59429
|
+
const result = fitTranscriptions(
|
|
59430
|
+
await transcribeNoteAudio(id2, {
|
|
59431
|
+
locale,
|
|
59432
|
+
attachmentId,
|
|
59433
|
+
includeText,
|
|
59434
|
+
downloadAssets,
|
|
59435
|
+
maxSeconds,
|
|
59436
|
+
signal
|
|
59437
|
+
}),
|
|
59438
|
+
exportMaxResponseBytes()
|
|
59439
|
+
);
|
|
59440
|
+
return successResponse(
|
|
59441
|
+
formatTranscription(result),
|
|
59442
|
+
result
|
|
59443
|
+
);
|
|
59444
|
+
}, "Error transcribing audio")
|
|
59445
|
+
);
|
|
58951
59446
|
registerTool(
|
|
58952
59447
|
"list-recent-notes",
|
|
58953
59448
|
{
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
// A small command-line helper for apple-notes-mcp that uses PUBLIC Apple
|
|
4
4
|
// frameworks only. It never opens the Notes database and never writes to the
|
|
5
5
|
// Notes group container: the TypeScript server reads the bytes it needs
|
|
6
|
-
// (read-only) and hands them over on stdin
|
|
6
|
+
// (read-only) and hands them over on stdin, or names an audio file that the
|
|
7
|
+
// helper opens for reading only.
|
|
7
8
|
//
|
|
8
9
|
// Protocol: one JSON object on stdin, one JSON object on stdout.
|
|
9
10
|
// request: {"protocol": 1, "action": "<name>", ...fields}
|
|
@@ -15,6 +16,7 @@
|
|
|
15
16
|
// decode_drawing PencilKit drawing bytes (base64) -> strokes
|
|
16
17
|
// encode_drawing strokes -> PencilKit drawing bytes (base64); used to build
|
|
17
18
|
// synthetic test fixtures, never to write to Notes
|
|
19
|
+
// transcribe on-device Speech transcription of one audio file
|
|
18
20
|
//
|
|
19
21
|
// Build (done by `apple-notes-mcp setup --public-helper`, which also generates
|
|
20
22
|
// the one-line source-digest file that defines helperSourceSHA256 and the
|
|
@@ -23,9 +25,11 @@
|
|
|
23
25
|
// -Xlinker -sectcreate -Xlinker __TEXT -Xlinker __info_plist -Xlinker Info.plist \
|
|
24
26
|
// apple-notes-public-helper.swift source-digest.swift -o apple-notes-public-helper
|
|
25
27
|
|
|
28
|
+
import AVFoundation
|
|
26
29
|
import AppKit
|
|
27
30
|
import Foundation
|
|
28
31
|
import PencilKit
|
|
32
|
+
import Speech
|
|
29
33
|
|
|
30
34
|
let protocolVersion = 1
|
|
31
35
|
let maxInputBytes = 96 * 1024 * 1024
|
|
@@ -211,11 +215,331 @@ func encodeDrawing(_ request: [String: Any]) throws -> [String: Any] {
|
|
|
211
215
|
return ["status": "ok", "strokeCount": strokes.count, "dataBase64": data.base64EncodedString()]
|
|
212
216
|
}
|
|
213
217
|
|
|
218
|
+
// MARK: - Transcription
|
|
219
|
+
//
|
|
220
|
+
// On-device only. On macOS 26 and later, SpeechAnalyzer with SpeechTranscriber
|
|
221
|
+
// runs entirely on the Mac (its language model is a local asset). On older
|
|
222
|
+
// systems, SFSpeechRecognizer is used with requiresOnDeviceRecognition = true,
|
|
223
|
+
// and a locale without on-device support is refused rather than sent to a
|
|
224
|
+
// server. The helper stops at its own deadline and returns what it has with
|
|
225
|
+
// complete = false, so the caller can report a partial transcript.
|
|
226
|
+
|
|
227
|
+
/// Final transcript segments gathered so far; an actor so the deadline path can read them.
|
|
228
|
+
actor TranscriptCollector {
|
|
229
|
+
private var segments: [String] = []
|
|
230
|
+
func add(_ segment: String) { segments.append(segment) }
|
|
231
|
+
func text() -> String { segments.joined(separator: " ") }
|
|
232
|
+
func isEmpty() -> Bool { segments.isEmpty }
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/// The newest partial result (legacy recognizer), guarded by a lock because the
|
|
236
|
+
/// recognizer calls back on its own queue.
|
|
237
|
+
final class LatestText: @unchecked Sendable {
|
|
238
|
+
private let lock = NSLock()
|
|
239
|
+
private var value = ""
|
|
240
|
+
func set(_ text: String) { lock.withLock { value = text } }
|
|
241
|
+
func get() -> String { lock.withLock { value } }
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
enum RaceOutcome {
|
|
245
|
+
case completed
|
|
246
|
+
case failed(Error)
|
|
247
|
+
case timedOut
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/// Resumes a continuation exactly once, whichever side of a race finishes first.
|
|
251
|
+
final class RaceGate: @unchecked Sendable {
|
|
252
|
+
private let lock = NSLock()
|
|
253
|
+
private var continuation: CheckedContinuation<RaceOutcome, Never>?
|
|
254
|
+
private var expired = false
|
|
255
|
+
init(_ continuation: CheckedContinuation<RaceOutcome, Never>) { self.continuation = continuation }
|
|
256
|
+
/// Called when the deadline passes, before `stop`: whatever `work` reports
|
|
257
|
+
/// afterwards (usually a cancellation error) is a timeout, not a failure.
|
|
258
|
+
func expire() { lock.withLock { expired = true } }
|
|
259
|
+
func resume(_ outcome: RaceOutcome) {
|
|
260
|
+
let (pending, final): (CheckedContinuation<RaceOutcome, Never>?, RaceOutcome) = lock.withLock {
|
|
261
|
+
defer { continuation = nil }
|
|
262
|
+
return (continuation, expired ? .timedOut : outcome)
|
|
263
|
+
}
|
|
264
|
+
pending?.resume(returning: final)
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/// Runs `work` against a deadline without waiting for it after the deadline:
|
|
269
|
+
/// some framework calls (asset downloads) ignore cancellation, and the process
|
|
270
|
+
/// exits right after it answers anyway. On timeout, `stop` runs first so a
|
|
271
|
+
/// cooperative `work` can hand back what it has.
|
|
272
|
+
func race(
|
|
273
|
+
seconds: Double,
|
|
274
|
+
work: @escaping @Sendable () async throws -> Void,
|
|
275
|
+
stop: @escaping @Sendable () async -> Void
|
|
276
|
+
) async -> RaceOutcome {
|
|
277
|
+
await withCheckedContinuation { continuation in
|
|
278
|
+
let gate = RaceGate(continuation)
|
|
279
|
+
Task {
|
|
280
|
+
do {
|
|
281
|
+
try await work()
|
|
282
|
+
gate.resume(.completed)
|
|
283
|
+
} catch {
|
|
284
|
+
gate.resume(.failed(error))
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
Task {
|
|
288
|
+
try? await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000))
|
|
289
|
+
gate.expire()
|
|
290
|
+
await stop()
|
|
291
|
+
gate.resume(.timedOut)
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
func authorizationName(_ status: SFSpeechRecognizerAuthorizationStatus) -> String {
|
|
297
|
+
switch status {
|
|
298
|
+
case .authorized: return "authorized"
|
|
299
|
+
case .denied: return "denied"
|
|
300
|
+
case .restricted: return "restricted"
|
|
301
|
+
case .notDetermined: return "notDetermined"
|
|
302
|
+
@unknown default: return "unknown"
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// Speech Recognition access. The helper runs under an MCP server that nobody
|
|
307
|
+
// may be watching, so it never calls SFSpeechRecognizer.requestAuthorization:
|
|
308
|
+
// a system prompt would block the helper until the server killed it. It reads
|
|
309
|
+
// the current status instead and stops with `permission_required` when the
|
|
310
|
+
// engine it is about to use needs a grant that is missing.
|
|
311
|
+
|
|
312
|
+
/// The failure returned instead of prompting. The grant belongs to the app that
|
|
313
|
+
/// runs the MCP server (Claude, a terminal, ...), which macOS treats as responsible.
|
|
314
|
+
func permissionRequired(_ status: SFSpeechRecognizerAuthorizationStatus) -> HelperFailure {
|
|
315
|
+
let settings = "System Settings > Privacy & Security > Speech Recognition"
|
|
316
|
+
let detail: String
|
|
317
|
+
switch status {
|
|
318
|
+
case .denied:
|
|
319
|
+
detail = "Speech Recognition access was denied for the app running this server. Turn it on in \(settings), then try again."
|
|
320
|
+
case .restricted:
|
|
321
|
+
detail = "Speech Recognition is restricted on this Mac (for example by a device management profile), so it cannot be used."
|
|
322
|
+
default:
|
|
323
|
+
detail = "Speech Recognition access has not been granted to the app running this server, and the server never shows the permission prompt. Allow the app in \(settings), then try again."
|
|
324
|
+
}
|
|
325
|
+
return HelperFailure(code: "permission_required", message: detail)
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/// Stops before any Speech call when access is missing. The legacy recognizer
|
|
329
|
+
/// needs `.authorized`. SpeechAnalyzer (macOS 26+) runs on-device without a
|
|
330
|
+
/// grant and does not prompt, so there only an explicit refusal (`.denied` or
|
|
331
|
+
/// `.restricted`) stops it; that also covers a future macOS that gates it.
|
|
332
|
+
func checkSpeechAccess(requireGrant: Bool) throws {
|
|
333
|
+
let status = SFSpeechRecognizer.authorizationStatus()
|
|
334
|
+
if status == .authorized { return }
|
|
335
|
+
if requireGrant || status == .denied || status == .restricted {
|
|
336
|
+
throw permissionRequired(status)
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/// Maps a Speech error to a stable code. Permission is decided from the
|
|
341
|
+
/// explicit authorization status, never from the error's text.
|
|
342
|
+
func speechFailure(_ error: Error, requireGrant: Bool) -> HelperFailure {
|
|
343
|
+
if let failure = error as? HelperFailure { return failure }
|
|
344
|
+
do {
|
|
345
|
+
try checkSpeechAccess(requireGrant: requireGrant)
|
|
346
|
+
} catch let failure as HelperFailure {
|
|
347
|
+
return failure
|
|
348
|
+
} catch {}
|
|
349
|
+
return HelperFailure(code: "transcription_failed", message: error.localizedDescription)
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/// Makes sure the locale's on-device model is installed. A download starts only
|
|
353
|
+
/// when the caller opted in: otherwise a missing model is `asset_unavailable` at once.
|
|
354
|
+
@available(macOS 26, *)
|
|
355
|
+
func ensureSpeechAssets(_ transcriber: SpeechTranscriber, localeID: String, allowDownload: Bool) async throws {
|
|
356
|
+
switch await AssetInventory.status(forModules: [transcriber]) {
|
|
357
|
+
case .installed:
|
|
358
|
+
return
|
|
359
|
+
case .unsupported:
|
|
360
|
+
throw HelperFailure(code: "unsupported_locale", message: "No on-device speech model for \(localeID)")
|
|
361
|
+
case .downloading where !allowDownload:
|
|
362
|
+
throw HelperFailure(
|
|
363
|
+
code: "asset_unavailable",
|
|
364
|
+
message: "macOS is still downloading the on-device speech model for \(localeID); try again shortly"
|
|
365
|
+
)
|
|
366
|
+
default:
|
|
367
|
+
if !allowDownload {
|
|
368
|
+
throw HelperFailure(
|
|
369
|
+
code: "asset_unavailable",
|
|
370
|
+
message: "The on-device speech model for \(localeID) is not installed. Retry with downloadAssets: true to let macOS download it."
|
|
371
|
+
)
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
guard let installation = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) else {
|
|
375
|
+
return
|
|
376
|
+
}
|
|
377
|
+
let outcome = await race(seconds: 90, work: { try await installation.downloadAndInstall() }, stop: {})
|
|
378
|
+
switch outcome {
|
|
379
|
+
case .completed:
|
|
380
|
+
return
|
|
381
|
+
case .failed(let error):
|
|
382
|
+
throw HelperFailure(
|
|
383
|
+
code: "asset_unavailable",
|
|
384
|
+
message: "Could not install the on-device speech model for \(localeID): \(error.localizedDescription)"
|
|
385
|
+
)
|
|
386
|
+
case .timedOut:
|
|
387
|
+
throw HelperFailure(
|
|
388
|
+
code: "asset_unavailable",
|
|
389
|
+
message: "The on-device speech model for \(localeID) is still downloading; try again shortly"
|
|
390
|
+
)
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
@available(macOS 26, *)
|
|
395
|
+
func transcribeWithAnalyzer(
|
|
396
|
+
file: AVAudioFile, localeID: String, deadline: Double, allowDownload: Bool
|
|
397
|
+
) async throws -> [String: Any] {
|
|
398
|
+
try checkSpeechAccess(requireGrant: false)
|
|
399
|
+
guard SpeechTranscriber.isAvailable else {
|
|
400
|
+
throw HelperFailure(code: "speech_unavailable", message: "On-device transcription is not available on this Mac")
|
|
401
|
+
}
|
|
402
|
+
guard let locale = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: localeID)) else {
|
|
403
|
+
throw HelperFailure(code: "unsupported_locale", message: "No on-device transcription for \(localeID)")
|
|
404
|
+
}
|
|
405
|
+
let transcriber = SpeechTranscriber(locale: locale, preset: .transcription)
|
|
406
|
+
try await ensureSpeechAssets(transcriber, localeID: localeID, allowDownload: allowDownload)
|
|
407
|
+
let analyzer = SpeechAnalyzer(modules: [transcriber])
|
|
408
|
+
let collector = TranscriptCollector()
|
|
409
|
+
let reader = Task {
|
|
410
|
+
for try await result in transcriber.results where result.isFinal {
|
|
411
|
+
let segment = String(result.text.characters).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
412
|
+
if !segment.isEmpty { await collector.add(segment) }
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
let outcome = await race(
|
|
416
|
+
seconds: deadline,
|
|
417
|
+
work: {
|
|
418
|
+
_ = try await analyzer.analyzeSequence(from: file)
|
|
419
|
+
try await analyzer.finalizeAndFinishThroughEndOfInput()
|
|
420
|
+
try await reader.value
|
|
421
|
+
},
|
|
422
|
+
stop: {
|
|
423
|
+
await analyzer.cancelAndFinishNow()
|
|
424
|
+
reader.cancel()
|
|
425
|
+
}
|
|
426
|
+
)
|
|
427
|
+
var response: [String: Any] = ["status": "ok", "engine": "SpeechAnalyzer", "locale": locale.identifier(.bcp47)]
|
|
428
|
+
switch outcome {
|
|
429
|
+
case .completed:
|
|
430
|
+
response["complete"] = true
|
|
431
|
+
case .timedOut:
|
|
432
|
+
response["complete"] = false
|
|
433
|
+
response["stopReason"] = "deadline"
|
|
434
|
+
case .failed(let error):
|
|
435
|
+
await analyzer.cancelAndFinishNow()
|
|
436
|
+
reader.cancel()
|
|
437
|
+
let failure = speechFailure(error, requireGrant: false)
|
|
438
|
+
if await collector.isEmpty() { throw failure }
|
|
439
|
+
response["complete"] = false
|
|
440
|
+
response["stopReason"] = failure.message
|
|
441
|
+
}
|
|
442
|
+
response["transcript"] = await collector.text()
|
|
443
|
+
return response
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/// Pre-macOS 26 path: SFSpeechRecognizer, forced on-device. It needs a Speech
|
|
447
|
+
/// Recognition grant, which is checked here and never requested.
|
|
448
|
+
func transcribeWithRecognizer(url: URL, localeID: String, deadline: Double) async throws -> [String: Any] {
|
|
449
|
+
try checkSpeechAccess(requireGrant: true)
|
|
450
|
+
guard let recognizer = SFSpeechRecognizer(locale: Locale(identifier: localeID)) else {
|
|
451
|
+
throw HelperFailure(code: "unsupported_locale", message: "No speech recognizer for \(localeID)")
|
|
452
|
+
}
|
|
453
|
+
guard recognizer.supportsOnDeviceRecognition else {
|
|
454
|
+
throw HelperFailure(code: "unsupported_locale", message: "No on-device recognition for \(localeID)")
|
|
455
|
+
}
|
|
456
|
+
recognizer.queue = OperationQueue()
|
|
457
|
+
let request = SFSpeechURLRecognitionRequest(url: url)
|
|
458
|
+
request.requiresOnDeviceRecognition = true
|
|
459
|
+
request.shouldReportPartialResults = true
|
|
460
|
+
let latest = LatestText()
|
|
461
|
+
final class TaskBox: @unchecked Sendable { var task: SFSpeechRecognitionTask? }
|
|
462
|
+
let box = TaskBox()
|
|
463
|
+
let outcome = await race(
|
|
464
|
+
seconds: deadline,
|
|
465
|
+
work: {
|
|
466
|
+
try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
|
|
467
|
+
var resumed = false
|
|
468
|
+
box.task = recognizer.recognitionTask(with: request) { result, error in
|
|
469
|
+
if resumed { return }
|
|
470
|
+
if let result {
|
|
471
|
+
latest.set(result.bestTranscription.formattedString)
|
|
472
|
+
if result.isFinal {
|
|
473
|
+
resumed = true
|
|
474
|
+
continuation.resume()
|
|
475
|
+
return
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
if let error {
|
|
479
|
+
resumed = true
|
|
480
|
+
continuation.resume(throwing: error)
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
},
|
|
485
|
+
stop: { box.task?.cancel() }
|
|
486
|
+
)
|
|
487
|
+
// Partial results replace each other; the newest one is the transcript so far.
|
|
488
|
+
let transcript = await latest.get()
|
|
489
|
+
var response: [String: Any] = ["status": "ok", "engine": "SFSpeechRecognizer", "locale": localeID]
|
|
490
|
+
switch outcome {
|
|
491
|
+
case .completed:
|
|
492
|
+
response["complete"] = true
|
|
493
|
+
case .timedOut:
|
|
494
|
+
response["complete"] = false
|
|
495
|
+
response["stopReason"] = "deadline"
|
|
496
|
+
case .failed(let error):
|
|
497
|
+
let failure = speechFailure(error, requireGrant: true)
|
|
498
|
+
if transcript.isEmpty { throw failure }
|
|
499
|
+
response["complete"] = false
|
|
500
|
+
response["stopReason"] = failure.message
|
|
501
|
+
}
|
|
502
|
+
response["transcript"] = transcript
|
|
503
|
+
return response
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
func transcribe(_ request: [String: Any]) async throws -> [String: Any] {
|
|
507
|
+
guard let path = request["path"] as? String, path.hasPrefix("/") else {
|
|
508
|
+
throw HelperFailure(code: "invalid_request", message: "path must be an absolute file path")
|
|
509
|
+
}
|
|
510
|
+
let localeID = request["locale"] as? String ?? "en-US"
|
|
511
|
+
let deadline = min(max((request["timeoutSeconds"] as? NSNumber)?.doubleValue ?? 300, 5), 3600)
|
|
512
|
+
var isDirectory: ObjCBool = false
|
|
513
|
+
guard FileManager.default.fileExists(atPath: path, isDirectory: &isDirectory), !isDirectory.boolValue else {
|
|
514
|
+
throw HelperFailure(code: "file_not_found", message: "The audio file does not exist")
|
|
515
|
+
}
|
|
516
|
+
let url = URL(fileURLWithPath: path)
|
|
517
|
+
let file: AVAudioFile
|
|
518
|
+
do {
|
|
519
|
+
file = try AVAudioFile(forReading: url)
|
|
520
|
+
} catch {
|
|
521
|
+
throw HelperFailure(code: "unsupported_audio", message: "Could not read the audio: \(error.localizedDescription)")
|
|
522
|
+
}
|
|
523
|
+
let duration = file.fileFormat.sampleRate > 0 ? Double(file.length) / file.fileFormat.sampleRate : 0
|
|
524
|
+
var response: [String: Any]
|
|
525
|
+
if #available(macOS 26, *) {
|
|
526
|
+
response = try await transcribeWithAnalyzer(
|
|
527
|
+
file: file, localeID: localeID, deadline: deadline,
|
|
528
|
+
allowDownload: request["downloadAssets"] as? Bool ?? false
|
|
529
|
+
)
|
|
530
|
+
} else {
|
|
531
|
+
response = try await transcribeWithRecognizer(url: url, localeID: localeID, deadline: deadline)
|
|
532
|
+
}
|
|
533
|
+
response["durationSeconds"] = Int(duration.rounded())
|
|
534
|
+
response["speechAuthorization"] = authorizationName(SFSpeechRecognizer.authorizationStatus())
|
|
535
|
+
return response
|
|
536
|
+
}
|
|
537
|
+
|
|
214
538
|
// MARK: - Dispatch
|
|
215
539
|
|
|
216
|
-
let actions = ["hello", "decode_drawing", "encode_drawing"]
|
|
540
|
+
let actions = ["hello", "decode_drawing", "encode_drawing", "transcribe"]
|
|
217
541
|
|
|
218
|
-
func handle(_ request: [String: Any]) throws -> [String: Any] {
|
|
542
|
+
func handle(_ request: [String: Any]) async throws -> [String: Any] {
|
|
219
543
|
guard (request["protocol"] as? NSNumber)?.intValue == protocolVersion else {
|
|
220
544
|
throw HelperFailure(code: "protocol_mismatch", message: "expected protocol \(protocolVersion)")
|
|
221
545
|
}
|
|
@@ -231,6 +555,8 @@ func handle(_ request: [String: Any]) throws -> [String: Any] {
|
|
|
231
555
|
return try decodeDrawing(request)
|
|
232
556
|
case "encode_drawing":
|
|
233
557
|
return try encodeDrawing(request)
|
|
558
|
+
case "transcribe":
|
|
559
|
+
return try await transcribe(request)
|
|
234
560
|
default:
|
|
235
561
|
throw HelperFailure(code: "unknown_action", message: "unknown action")
|
|
236
562
|
}
|
|
@@ -238,7 +564,7 @@ func handle(_ request: [String: Any]) throws -> [String: Any] {
|
|
|
238
564
|
|
|
239
565
|
@main
|
|
240
566
|
struct PublicHelper {
|
|
241
|
-
static func main() {
|
|
567
|
+
static func main() async {
|
|
242
568
|
let input = FileHandle.standardInput.readDataToEndOfFile()
|
|
243
569
|
if input.count > maxInputBytes {
|
|
244
570
|
fail(HelperFailure(code: "invalid_request", message: "request exceeds \(maxInputBytes) bytes"))
|
|
@@ -247,7 +573,7 @@ struct PublicHelper {
|
|
|
247
573
|
fail(HelperFailure(code: "invalid_request", message: "stdin must be one JSON object"))
|
|
248
574
|
}
|
|
249
575
|
do {
|
|
250
|
-
finish(try handle(request))
|
|
576
|
+
finish(try await handle(request))
|
|
251
577
|
} catch let failure as HelperFailure {
|
|
252
578
|
fail(failure)
|
|
253
579
|
} catch {
|
package/package.json
CHANGED