arcane-os 0.5.16 → 0.5.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +42 -19
- package/browser-runtime/ai/browser-speech-providers.mjs +8 -4
- package/browser-runtime/speech-text.mjs +31 -0
- package/docs/architecture.md +1 -1
- package/docs/reference/README.md +8 -8
- package/docs/reference/ai/browser-speech.md +34 -13
- package/docs/reference/ai/twin-cloud.md +1 -1
- package/docs/reference/availability-and-normalization.md +2 -1
- package/docs/reference/cli.md +1 -1
- package/docs/reference/core/arcane-ai-contracts.md +1 -1
- package/docs/reference/inventory/package-api.json +60 -4
- package/docs/reference/inventory/runtime-components.json +2 -2
- package/docs/reference/inventory/runtime-modules.json +14 -12
- package/docs/reference/protocols.md +17 -13
- package/docs/reference/runtime-components.md +6 -5
- package/docs/reference/runtime-modules.md +88 -36
- package/docs/reference/sdk-api.md +181 -11
- package/package.json +2 -1
- package/runtime/arcane/components/chat.html +1 -3
- package/runtime/arcane/modules/AI.js +13 -23
- package/runtime/arcane/modules/AIProviderRuntime.js +30 -8
- package/runtime/arcane/modules/MarkdownSpeech.js +1 -50
- package/runtime/arcane/modules/SpeechPlayback.js +33 -3
- package/src/import-map.mjs +1 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
# Arcane OS SDK JavaScript API
|
|
2
2
|
|
|
3
3
|
The npm package exposes a Node.js ESM control plane, the portable
|
|
4
|
-
`arcane-os/event-manager`, `arcane-os/logging`, `arcane-os/mail`,
|
|
5
|
-
`arcane-os/speech-playback
|
|
4
|
+
`arcane-os/event-manager`, `arcane-os/logging`, `arcane-os/mail`,
|
|
5
|
+
`arcane-os/preference-store`, `arcane-os/speech-playback`, and
|
|
6
|
+
`arcane-os/speech-text` entrypoints, and the browser-only
|
|
6
7
|
`arcane-os/ai/browser-wasm` and `arcane-os/ai/browser-speech` entrypoints.
|
|
7
8
|
Those package subpaths are distinct from application-facing projection modules
|
|
8
9
|
in the managed browser map, such as `arcane/AIProviderRuntime`,
|
|
@@ -18,7 +19,7 @@ This table is the Node `package.json#exports` map: it defines package
|
|
|
18
19
|
entrypoints for SDK/tooling code. It is distinct from the generated browser
|
|
19
20
|
import map that resolves application-facing `arcane/*` modules and the focused
|
|
20
21
|
EventManager entry. See [browser runtime delivery](protocols.md#browser-runtime-delivery)
|
|
21
|
-
for the installed-inventory-derived physical-runtime contract in SDK `0.5.
|
|
22
|
+
for the installed-inventory-derived physical-runtime contract in SDK `0.5.17`.
|
|
22
23
|
|
|
23
24
|
| Specifier | Purpose |
|
|
24
25
|
| --- | --- |
|
|
@@ -36,6 +37,7 @@ for the installed-inventory-derived physical-runtime contract in SDK `0.5.16`.
|
|
|
36
37
|
| `arcane-os/logging` | Shared console diagnostics controlled by the existing `user.developer` preference. |
|
|
37
38
|
| `arcane-os/preference-store` | Portable preference records and injected storage adapters. |
|
|
38
39
|
| `arcane-os/speech-playback` | Portable speech preparation, playback state, and injected media adapters. |
|
|
40
|
+
| `arcane-os/speech-text` | Dependency-free speech-input formatting cleanup for complete text and streamed chunks. |
|
|
39
41
|
| `arcane-os/ai/browser-wasm` | Caller-selected browser-local Wllama inference, complete DBOPFS model storage, streaming, cancellation, and structural tool-call results. |
|
|
40
42
|
| `arcane-os/ai/browser-speech` | Caller-selected browser-local Whisper STT and Kokoro TTS provider mechanisms, ordinary upstream assets, materialized/native routing, Workers, and cancellation. |
|
|
41
43
|
| `arcane-os/mail` | Portable Mail runtime, durable outbox, complete transport responses, and provider-neutral acceptance contracts. |
|
|
@@ -143,6 +145,7 @@ browser map are cataloged separately in [Runtime modules](runtime-modules.md).
|
|
|
143
145
|
| `loadSdkBrowserRuntimeRelease()` | function | `arcane-os` | Runtime and app descriptors | Node |
|
|
144
146
|
| `materializeInstalledSdkRuntime()` | function | `arcane-os` | Runtime and app descriptors | Node |
|
|
145
147
|
| `materializeWorkspaceRuntimeContent()` | function | `arcane-os` | Runtime and app descriptors | Node |
|
|
148
|
+
| `MarkdownSpeech` | class | `arcane-os/speech-text` | Portable runtime modules | Node and browser |
|
|
146
149
|
| `NATIVE_BUILD_PLAN_PROTOCOL` | constant | `arcane-os` | Targets, native plans, and providers | Node; selected browser/native target or provider as documented |
|
|
147
150
|
| `NATIVE_BUILDER_PROTOCOL` | constant | `arcane-os` | Targets, native plans, and providers | Node; selected browser/native target or provider as documented |
|
|
148
151
|
| `normalizeError()` | function | `arcane-os` | Errors | Node |
|
|
@@ -187,8 +190,11 @@ browser map are cataloged separately in [Runtime modules](runtime-modules.md).
|
|
|
187
190
|
| `selectApp()` | function | `arcane-os` | Workspace, doctor, repository, and server | Node |
|
|
188
191
|
| `SpeechPlayback default export` | class | `arcane-os/speech-playback` | Portable runtime modules | Node with injected media adapters, or browser/native WebView media |
|
|
189
192
|
| `SPEECH_PLAYBACK_STATE_EVENT` | constant | `arcane-os/speech-playback` | Portable runtime modules | Node and browser |
|
|
193
|
+
| `SPEECH_VOICE_ALIASES` | constant | `arcane-os/speech-playback` | Portable runtime modules | Node and browser |
|
|
194
|
+
| `SPEECH_VOICE_OPTIONS` | constant | `arcane-os/speech-playback` | Portable runtime modules | Node and browser |
|
|
190
195
|
| `SpeechPlayback` | class | `arcane-os/speech-playback` | Portable runtime modules | Node with injected media adapters, or browser/native WebView media |
|
|
191
196
|
| `splitSpeechText()` | function | `arcane-os/speech-playback` | Portable runtime modules | Node and browser |
|
|
197
|
+
| `stripSpeechFormatting()` | function | `arcane-os/speech-text` | Portable runtime modules | Node and browser |
|
|
192
198
|
| `startDevServer()` | function | `arcane-os` | Workspace, doctor, repository, and server | Node control plane; browser data plane |
|
|
193
199
|
| `startSourceExampleServer()` | function | `arcane-os` | Workspace, doctor, repository, and server | Node control plane; browser data plane |
|
|
194
200
|
| `TARGET_ADAPTER_PROTOCOL` | constant | `arcane-os` | Targets, native plans, and providers | Node; selected browser/native target or provider as documented |
|
|
@@ -751,7 +757,7 @@ deterministic map. The package root also contains the public
|
|
|
751
757
|
{
|
|
752
758
|
schemaVersion: 1,
|
|
753
759
|
kind: 'arcane-app-runtime-projection',
|
|
754
|
-
sdkVersion: '0.5.
|
|
760
|
+
sdkVersion: '0.5.17',
|
|
755
761
|
pathPrefix: 'arcane/',
|
|
756
762
|
files: [{path}]
|
|
757
763
|
}
|
|
@@ -1507,6 +1513,16 @@ bounded FIFO queue (browser Kokoro defaults to four), while Blob URLs and
|
|
|
1507
1513
|
playback remain in exact input order. A native or custom client without that
|
|
1508
1514
|
advertised capacity stays serialized with one lookahead. Replay keeps completed
|
|
1509
1515
|
URLs and pending requests while retrying failed missing provider segments.
|
|
1516
|
+
The optional constructor `onState(detail)` callback runs synchronously after
|
|
1517
|
+
the canonical state occurrence is dispatched; callback failures are reported
|
|
1518
|
+
without replacing playback settlement.
|
|
1519
|
+
|
|
1520
|
+
`prepare()` keeps every nonblank stored part exact. At synthesis time,
|
|
1521
|
+
`SpeechPlayback` copies the part and automatically removes repeated formatting
|
|
1522
|
+
marks from only that outbound speech input. It passes SDK-internal preparation
|
|
1523
|
+
metadata so a downstream SDK speech boundary does not apply the filter again;
|
|
1524
|
+
applications do not supply that metadata. Original part objects, displayed or
|
|
1525
|
+
stored content, and every non-input payload field remain unchanged.
|
|
1510
1526
|
|
|
1511
1527
|
### Signature and result
|
|
1512
1528
|
|
|
@@ -1526,7 +1542,13 @@ runtime module; provider and media availability remain host-owned.
|
|
|
1526
1542
|
import SpeechPlayback from 'arcane-os/speech-playback';
|
|
1527
1543
|
|
|
1528
1544
|
const audio=document.body.appendChild(document.createElement('audio'));
|
|
1529
|
-
const playback=new SpeechPlayback({
|
|
1545
|
+
const playback=new SpeechPlayback({
|
|
1546
|
+
audio,
|
|
1547
|
+
speech:globalThis.ai,
|
|
1548
|
+
onState(detail) {
|
|
1549
|
+
console.log(detail.state);
|
|
1550
|
+
}
|
|
1551
|
+
});
|
|
1530
1552
|
const button=document.body.appendChild(document.createElement('button'));
|
|
1531
1553
|
button.textContent='Speak';
|
|
1532
1554
|
button.addEventListener('click',async function speakCompleteSegments(){
|
|
@@ -1561,18 +1583,91 @@ import {SPEECH_PLAYBACK_STATE_EVENT} from 'arcane-os/speech-playback';
|
|
|
1561
1583
|
console.log(SPEECH_PLAYBACK_STATE_EVENT);
|
|
1562
1584
|
```
|
|
1563
1585
|
|
|
1586
|
+
## SPEECH_VOICE_ALIASES
|
|
1587
|
+
|
|
1588
|
+
### Overview
|
|
1589
|
+
|
|
1590
|
+
Mutable compatibility membership for the ten shared voice identifiers used by
|
|
1591
|
+
existing speech controls. `SpeechPlayback` does not select a voice from this
|
|
1592
|
+
set, and membership does not promise support from the selected provider.
|
|
1593
|
+
|
|
1594
|
+
### Signature and result
|
|
1595
|
+
|
|
1596
|
+
```text
|
|
1597
|
+
const SPEECH_VOICE_ALIASES
|
|
1598
|
+
```
|
|
1599
|
+
|
|
1600
|
+
### Availability and normalization
|
|
1601
|
+
|
|
1602
|
+
**Node and browser.** Ordinary mutable `Set`, derived once in
|
|
1603
|
+
`SPEECH_VOICE_OPTIONS` order, containing `alloy`, `ash`, `ballad`, `coral`,
|
|
1604
|
+
`echo`, `fable`, `nova`, `onyx`, `sage`, and `shimmer`.
|
|
1605
|
+
|
|
1606
|
+
### Example
|
|
1607
|
+
|
|
1608
|
+
```javascript
|
|
1609
|
+
import {SPEECH_VOICE_ALIASES} from 'arcane-os/speech-playback';
|
|
1610
|
+
console.log(SPEECH_VOICE_ALIASES.has('alloy')); // true
|
|
1611
|
+
```
|
|
1612
|
+
|
|
1613
|
+
## SPEECH_VOICE_OPTIONS
|
|
1614
|
+
|
|
1615
|
+
### Overview
|
|
1616
|
+
|
|
1617
|
+
Mutable ordered compatibility records for existing speech voice controls.
|
|
1618
|
+
Each record has a lowercase `value` and matching title-case `label`. Provider
|
|
1619
|
+
support and voice selection remain caller-owned.
|
|
1620
|
+
|
|
1621
|
+
### Signature and result
|
|
1622
|
+
|
|
1623
|
+
```text
|
|
1624
|
+
const SPEECH_VOICE_OPTIONS
|
|
1625
|
+
```
|
|
1626
|
+
|
|
1627
|
+
### Availability and normalization
|
|
1628
|
+
|
|
1629
|
+
**Node and browser.** Ordinary mutable array and mutable `{value,label}`
|
|
1630
|
+
records ordered as Alloy, Ash, Ballad, Coral, Echo, Fable, Nova, Onyx, Sage,
|
|
1631
|
+
and Shimmer.
|
|
1632
|
+
|
|
1633
|
+
### Example
|
|
1634
|
+
|
|
1635
|
+
```javascript
|
|
1636
|
+
import {SPEECH_VOICE_OPTIONS} from 'arcane-os/speech-playback';
|
|
1637
|
+
console.log(SPEECH_VOICE_OPTIONS[0]); // {value: 'alloy', label: 'Alloy'}
|
|
1638
|
+
```
|
|
1639
|
+
|
|
1564
1640
|
## SpeechPlayback
|
|
1565
1641
|
|
|
1566
1642
|
### Overview
|
|
1567
1643
|
|
|
1568
1644
|
Named binding for the same capability-aware, exact-order canonical class exposed
|
|
1569
1645
|
as the speech-playback default. `prepare()` preserves each nonblank part's exact
|
|
1570
|
-
input string without trimming, splitting, or freezing that content.
|
|
1646
|
+
input string without trimming, splitting, or freezing that content. An optional
|
|
1647
|
+
`onState(detail)` constructor callback receives every state synchronously after
|
|
1648
|
+
the canonical occurrence is dispatched. Both surfaces expose the same public
|
|
1649
|
+
field values at dispatch time; object identity is not promised. Callback
|
|
1650
|
+
failures are reported through `globalThis.reportError` when available,
|
|
1651
|
+
otherwise `console.error`, and do not replace playback settlement.
|
|
1652
|
+
Only the outbound synthesis copy receives automatic repeated-formatting-mark
|
|
1653
|
+
cleanup; the original part and stored preparation record stay exact.
|
|
1571
1654
|
|
|
1572
1655
|
### Signature and result
|
|
1573
1656
|
|
|
1574
1657
|
```text
|
|
1575
|
-
new SpeechPlayback(
|
|
1658
|
+
new SpeechPlayback({
|
|
1659
|
+
audio,
|
|
1660
|
+
speech?,
|
|
1661
|
+
model?,
|
|
1662
|
+
voice?,
|
|
1663
|
+
responseFormat?,
|
|
1664
|
+
speed=1,
|
|
1665
|
+
onState=()=>{},
|
|
1666
|
+
createObjectURL?,
|
|
1667
|
+
revokeObjectURL?,
|
|
1668
|
+
delay?,
|
|
1669
|
+
messages?
|
|
1670
|
+
})
|
|
1576
1671
|
```
|
|
1577
1672
|
|
|
1578
1673
|
### Availability and normalization
|
|
@@ -1586,7 +1681,15 @@ serialized, and media availability remains host-owned.
|
|
|
1586
1681
|
|
|
1587
1682
|
```javascript
|
|
1588
1683
|
import SpeechPlayback,{SpeechPlayback as NamedSpeechPlayback} from 'arcane-os/speech-playback';
|
|
1589
|
-
|
|
1684
|
+
|
|
1685
|
+
const audio=document.body.appendChild(document.createElement('audio'));
|
|
1686
|
+
const playback=new NamedSpeechPlayback({
|
|
1687
|
+
audio,
|
|
1688
|
+
onState(detail) {
|
|
1689
|
+
console.log(detail.state);
|
|
1690
|
+
}
|
|
1691
|
+
});
|
|
1692
|
+
console.log(playback instanceof SpeechPlayback); // true
|
|
1590
1693
|
```
|
|
1591
1694
|
|
|
1592
1695
|
## splitSpeechText()
|
|
@@ -1612,6 +1715,73 @@ import {splitSpeechText} from 'arcane-os/speech-playback';
|
|
|
1612
1715
|
console.log(splitSpeechText('Hello world.'));
|
|
1613
1716
|
```
|
|
1614
1717
|
|
|
1718
|
+
## MarkdownSpeech
|
|
1719
|
+
|
|
1720
|
+
### Overview
|
|
1721
|
+
|
|
1722
|
+
Streams the SDK's dependency-free speech formatting cleanup across input chunk
|
|
1723
|
+
boundaries. `append()` removes runs of two or more identical `*`, `#`, `_`,
|
|
1724
|
+
backtick, or `~` marks while retaining single marks and every other character.
|
|
1725
|
+
It is a narrow speech filter, not a Markdown parser. Normal SDK TTS calls use
|
|
1726
|
+
this cleanup automatically; applications do not need to construct this class
|
|
1727
|
+
or opt in.
|
|
1728
|
+
|
|
1729
|
+
### Signature and result
|
|
1730
|
+
|
|
1731
|
+
```text
|
|
1732
|
+
new MarkdownSpeech()
|
|
1733
|
+
```
|
|
1734
|
+
|
|
1735
|
+
`append(text='',end=false)` returns only newly available narration. It retains
|
|
1736
|
+
one trailing candidate formatting mark across calls so a repeated run split
|
|
1737
|
+
between chunks is still removed. A terminal `end:true` append flushes any
|
|
1738
|
+
single pending mark and resets the instance. `reset()` discards pending state.
|
|
1739
|
+
|
|
1740
|
+
### Availability and normalization
|
|
1741
|
+
|
|
1742
|
+
**Node and browser.** The implementation has no runtime dependency. Caller
|
|
1743
|
+
strings remain unchanged. Non-string input throws `TypeError`.
|
|
1744
|
+
|
|
1745
|
+
### Example
|
|
1746
|
+
|
|
1747
|
+
```javascript
|
|
1748
|
+
import {MarkdownSpeech} from 'arcane-os/speech-text';
|
|
1749
|
+
|
|
1750
|
+
const speechText=new MarkdownSpeech();
|
|
1751
|
+
console.log(speechText.append('*')); // '' (held for the next chunk)
|
|
1752
|
+
console.log(speechText.append('*Hello.',true)); // 'Hello.'
|
|
1753
|
+
```
|
|
1754
|
+
|
|
1755
|
+
## stripSpeechFormatting()
|
|
1756
|
+
|
|
1757
|
+
### Overview
|
|
1758
|
+
|
|
1759
|
+
Removes repeated same formatting marks from one complete speech text value.
|
|
1760
|
+
Single marks, ordinary punctuation, links, code contents, and every other
|
|
1761
|
+
character remain literal. Normal SDK TTS calls already apply this function to
|
|
1762
|
+
an outbound copy; the public helper is available when another speech-text owner
|
|
1763
|
+
needs the same one-pass result.
|
|
1764
|
+
|
|
1765
|
+
### Signature and result
|
|
1766
|
+
|
|
1767
|
+
```text
|
|
1768
|
+
stripSpeechFormatting(text='')
|
|
1769
|
+
```
|
|
1770
|
+
|
|
1771
|
+
### Availability and normalization
|
|
1772
|
+
|
|
1773
|
+
**Node and browser.** Calls the supplied value's `replace()` method and returns
|
|
1774
|
+
its filtered result without modifying the supplied value. Pass a string;
|
|
1775
|
+
values without a callable `replace()` fail with the native `TypeError`.
|
|
1776
|
+
|
|
1777
|
+
### Example
|
|
1778
|
+
|
|
1779
|
+
```javascript
|
|
1780
|
+
import {stripSpeechFormatting} from 'arcane-os/speech-text';
|
|
1781
|
+
|
|
1782
|
+
console.log(stripSpeechFormatting('**Hello**')); // 'Hello'
|
|
1783
|
+
```
|
|
1784
|
+
|
|
1615
1785
|
## projectNativeDescriptor()
|
|
1616
1786
|
|
|
1617
1787
|
### Overview
|
|
@@ -3463,7 +3633,7 @@ workspace it additionally returns the exact installed package authority:
|
|
|
3463
3633
|
packageSource,
|
|
3464
3634
|
canonicalPackageRoot,
|
|
3465
3635
|
packageName: 'arcane-os',
|
|
3466
|
-
packageVersion: '0.5.
|
|
3636
|
+
packageVersion: '0.5.17',
|
|
3467
3637
|
runtimeRoot,
|
|
3468
3638
|
browserRuntimeRoot
|
|
3469
3639
|
}
|
|
@@ -3471,9 +3641,9 @@ workspace it additionally returns the exact installed package authority:
|
|
|
3471
3641
|
```
|
|
3472
3642
|
|
|
3473
3643
|
The dependency can be named `arcane-os` or be one exact npm alias for
|
|
3474
|
-
`npm:arcane-os@0.5.
|
|
3644
|
+
`npm:arcane-os@0.5.17`. The selected installation must still be one direct,
|
|
3475
3645
|
physical, non-link package directory whose manifest identifies exactly as
|
|
3476
|
-
`arcane-os@0.5.
|
|
3646
|
+
`arcane-os@0.5.17`; duplicate canonical/alias declarations reject.
|
|
3477
3647
|
`allowMissingManagedImportMap` is an internal packaging/development seam. An
|
|
3478
3648
|
ordinary caller should leave it `false`.
|
|
3479
3649
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "arcane-os",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.17",
|
|
4
4
|
"description": "Arcane OS JavaScript SDK, project-local CLI, browser runtime, and repository-portable application packager.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.mjs",
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
"./logging": "./browser-runtime/logging.mjs",
|
|
17
17
|
"./preference-store": "./runtime/arcane/modules/PreferenceStore.js",
|
|
18
18
|
"./speech-playback": "./runtime/arcane/modules/SpeechPlayback.js",
|
|
19
|
+
"./speech-text": "./browser-runtime/speech-text.mjs",
|
|
19
20
|
"./ai/browser-wasm": "./browser-runtime/ai/browser-wasm.mjs",
|
|
20
21
|
"./ai/browser-speech": "./browser-runtime/ai/browser-speech.mjs",
|
|
21
22
|
"./mail": "./src/mail-api.mjs",
|
|
@@ -2817,9 +2817,7 @@
|
|
|
2817
2817
|
){
|
|
2818
2818
|
const aiRuntime=boundChatAI??globalThis.ai;
|
|
2819
2819
|
if(typeof aiRuntime?.streamTTS==='function'){
|
|
2820
|
-
void Promise.resolve(aiRuntime.streamTTS(
|
|
2821
|
-
text,false,{textFormat:'markdown'}
|
|
2822
|
-
)).catch(
|
|
2820
|
+
void Promise.resolve(aiRuntime.streamTTS(text)).catch(
|
|
2823
2821
|
reportTTSError
|
|
2824
2822
|
);
|
|
2825
2823
|
}else{
|
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
} from './AIProviderRuntime.js';
|
|
14
14
|
import {normalizeOllamaModelIdentifier} from './OllamaModelIdentifier.js';
|
|
15
15
|
import {arcaneLogging} from 'arcane-os/logging';
|
|
16
|
-
import {MarkdownSpeech} from '
|
|
16
|
+
import {MarkdownSpeech,stripSpeechFormatting} from 'arcane-os/speech-text';
|
|
17
17
|
|
|
18
18
|
const completeValue=(value)=>value;
|
|
19
19
|
|
|
@@ -2030,7 +2030,6 @@ class AI {
|
|
|
2030
2030
|
}
|
|
2031
2031
|
|
|
2032
2032
|
audioMessageChunks='';
|
|
2033
|
-
#speechTextFormat='plain';
|
|
2034
2033
|
#markdownSpeech=new MarkdownSpeech();
|
|
2035
2034
|
sourceNodes=[];
|
|
2036
2035
|
isSpeaking=false;
|
|
@@ -5734,7 +5733,6 @@ class AI {
|
|
|
5734
5733
|
if(this.muted){
|
|
5735
5734
|
if(end){
|
|
5736
5735
|
this.audioMessageChunks='';
|
|
5737
|
-
this.#speechTextFormat='plain';
|
|
5738
5736
|
this.#markdownSpeech.reset();
|
|
5739
5737
|
}
|
|
5740
5738
|
this.#traceSpeech('streamTTS.result',{callId,result:false,reason:'muted'});
|
|
@@ -5747,24 +5745,12 @@ class AI {
|
|
|
5747
5745
|
throw new RangeError('Speech pauses must be a nonnegative number of milliseconds.');
|
|
5748
5746
|
}
|
|
5749
5747
|
|
|
5750
|
-
const
|
|
5751
|
-
if(textFormat!=='plain'&&textFormat!=='markdown'){
|
|
5752
|
-
throw new TypeError('Speech textFormat must be plain or markdown.');
|
|
5753
|
-
}
|
|
5754
|
-
let speechText='';
|
|
5755
|
-
if(textFormat!==this.#speechTextFormat){
|
|
5756
|
-
speechText=this.#markdownSpeech.append('',true);
|
|
5757
|
-
}
|
|
5758
|
-
const incomingText=String(text||'');
|
|
5759
|
-
speechText+=textFormat==='markdown'
|
|
5760
|
-
?this.#markdownSpeech.append(incomingText,end)
|
|
5761
|
-
:incomingText;
|
|
5762
|
-
this.#speechTextFormat=end?'plain':textFormat;
|
|
5748
|
+
const speechText=this.#markdownSpeech.append(String(text||''),end);
|
|
5763
5749
|
this.audioMessageChunks+=speechText;
|
|
5764
5750
|
const outputs=this.#extractSpeechSegments(end);
|
|
5765
5751
|
this.#traceSpeech('streamTTS.segments',{
|
|
5766
5752
|
callId,segments:outputs,remainder:this.audioMessageChunks,
|
|
5767
|
-
segmentation:this.ttsSegmentation,
|
|
5753
|
+
segmentation:this.ttsSegmentation,speechText
|
|
5768
5754
|
});
|
|
5769
5755
|
|
|
5770
5756
|
if(!outputs.length){
|
|
@@ -6040,7 +6026,8 @@ class AI {
|
|
|
6040
6026
|
this.#traceSpeechJob('generation.request',job,{payload,selection});
|
|
6041
6027
|
const response=await this.fetchTTS(
|
|
6042
6028
|
payload,
|
|
6043
|
-
job.abortController.signal
|
|
6029
|
+
job.abortController.signal,
|
|
6030
|
+
{speechInputPrepared:true}
|
|
6044
6031
|
);
|
|
6045
6032
|
return this.#normalizeProviderSpeechAudio(response);
|
|
6046
6033
|
}
|
|
@@ -6189,7 +6176,7 @@ class AI {
|
|
|
6189
6176
|
return this.audioContext;
|
|
6190
6177
|
}
|
|
6191
6178
|
|
|
6192
|
-
async fetchTTS(payload={},signal=null){
|
|
6179
|
+
async fetchTTS(payload={},signal=null,{speechInputPrepared=false}={}){
|
|
6193
6180
|
const callId=this.#traceSpeech('fetchTTS.call',{payload,aborted:signal?.aborted});
|
|
6194
6181
|
try{
|
|
6195
6182
|
this.#assertServiceConfigured(this.ttsService,'tts');
|
|
@@ -6233,12 +6220,15 @@ class AI {
|
|
|
6233
6220
|
throw normalizeAIRequestAbort(signal.reason);
|
|
6234
6221
|
}
|
|
6235
6222
|
|
|
6236
|
-
const
|
|
6237
|
-
if(typeof
|
|
6223
|
+
const suppliedInput=descriptors.input?.value;
|
|
6224
|
+
if(typeof suppliedInput!=='string'||!suppliedInput.trim()){
|
|
6238
6225
|
const error=new TypeError('AI.fetchTTS input must be nonempty text.');
|
|
6239
6226
|
error.code='ARCANE_AI_TTS_INPUT_INVALID';
|
|
6240
6227
|
throw error;
|
|
6241
6228
|
}
|
|
6229
|
+
const input=speechInputPrepared===true
|
|
6230
|
+
?suppliedInput
|
|
6231
|
+
:stripSpeechFormatting(suppliedInput);
|
|
6242
6232
|
const selection=this.#providerRuntime.selection('tts');
|
|
6243
6233
|
const requestedModel=descriptors.model?.value;
|
|
6244
6234
|
if(requestedModel!==undefined
|
|
@@ -6329,7 +6319,8 @@ class AI {
|
|
|
6329
6319
|
payload:{model,voice,input,responseFormat,speed},
|
|
6330
6320
|
localOnly:false,
|
|
6331
6321
|
signal
|
|
6332
|
-
}
|
|
6322
|
+
},
|
|
6323
|
+
{speechInputPrepared:true}
|
|
6333
6324
|
);
|
|
6334
6325
|
this.#traceSpeech('fetchTTS.providerResult',{callId,response});
|
|
6335
6326
|
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
@@ -6432,7 +6423,6 @@ class AI {
|
|
|
6432
6423
|
this.speechScheduleContext=null;
|
|
6433
6424
|
this.speechScheduleTime=0;
|
|
6434
6425
|
this.audioMessageChunks='';
|
|
6435
|
-
this.#speechTextFormat='plain';
|
|
6436
6426
|
this.#markdownSpeech.reset();
|
|
6437
6427
|
this.#clearSpeechUnlock();
|
|
6438
6428
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import {arcaneLogging} from 'arcane-os/logging';
|
|
2
|
+
import {stripSpeechFormatting} from 'arcane-os/speech-text';
|
|
2
3
|
import {
|
|
3
4
|
AI_RUNTIME_ROLES,
|
|
4
5
|
getAIRuntimeState,
|
|
@@ -3343,14 +3344,14 @@ export class AIProviderRuntime {
|
|
|
3343
3344
|
return this.#disposeAllPromise;
|
|
3344
3345
|
}
|
|
3345
3346
|
|
|
3346
|
-
request(role, options = {}) {
|
|
3347
|
+
request(role, options = {}, {speechInputPrepared = false} = {}) {
|
|
3347
3348
|
if (SPEECH_ROLES.includes(role)) {
|
|
3348
3349
|
arcaneLogging.debug('[Arcane speech runtime] request', role, options);
|
|
3349
3350
|
}
|
|
3350
|
-
return this.#requestRole(role, options, false);
|
|
3351
|
+
return this.#requestRole(role, options, false, speechInputPrepared);
|
|
3351
3352
|
}
|
|
3352
3353
|
|
|
3353
|
-
#requestRole(role, options, queued) {
|
|
3354
|
+
#requestRole(role, options, queued, speechInputPrepared = false) {
|
|
3354
3355
|
this.#assertOpen();
|
|
3355
3356
|
this.#assertNotConfiguring();
|
|
3356
3357
|
assertRole(role);
|
|
@@ -3379,6 +3380,24 @@ export class AIProviderRuntime {
|
|
|
3379
3380
|
if (options.signal?.aborted) {
|
|
3380
3381
|
return Promise.reject(normalizedAbort());
|
|
3381
3382
|
}
|
|
3383
|
+
if (role === 'tts' && !queued && speechInputPrepared !== true) {
|
|
3384
|
+
const payload = options.payload;
|
|
3385
|
+
const field = Object.hasOwn(payload ?? {}, 'input') ? 'input' : 'text';
|
|
3386
|
+
const text = Object.getOwnPropertyDescriptor(payload ?? {}, field)?.value;
|
|
3387
|
+
if (typeof text === 'string') {
|
|
3388
|
+
const input = stripSpeechFormatting(text);
|
|
3389
|
+
if (input !== text) {
|
|
3390
|
+
const payloadProperties = Object.getOwnPropertyDescriptors(payload);
|
|
3391
|
+
payloadProperties[field] = {...payloadProperties[field], value: input};
|
|
3392
|
+
const requestProperties = Object.getOwnPropertyDescriptors(options);
|
|
3393
|
+
requestProperties.payload = {
|
|
3394
|
+
...requestProperties.payload,
|
|
3395
|
+
value: Object.create(Object.getPrototypeOf(payload), payloadProperties)
|
|
3396
|
+
};
|
|
3397
|
+
options = Object.create(Object.getPrototypeOf(options), requestProperties);
|
|
3398
|
+
}
|
|
3399
|
+
}
|
|
3400
|
+
}
|
|
3382
3401
|
const slot = this.#slots[role];
|
|
3383
3402
|
if (slot.disposed) {
|
|
3384
3403
|
return Promise.reject(
|
|
@@ -4001,7 +4020,9 @@ export class AIProviderRuntime {
|
|
|
4001
4020
|
if (SPEECH_ROLES.includes(role)) {
|
|
4002
4021
|
arcaneLogging.debug('[Arcane speech runtime] provider.request', {generation, requestSequence, operationId}, providerRequest);
|
|
4003
4022
|
}
|
|
4004
|
-
const result =
|
|
4023
|
+
const result = role === 'tts'
|
|
4024
|
+
? await provider.request(providerRequest, {speechInputPrepared: true})
|
|
4025
|
+
: await provider.request(providerRequest);
|
|
4005
4026
|
if (SPEECH_ROLES.includes(role)) {
|
|
4006
4027
|
arcaneLogging.debug('[Arcane speech runtime] provider.request.result', {role, generation, requestSequence, operationId}, result);
|
|
4007
4028
|
}
|
|
@@ -4065,9 +4086,9 @@ export class AIProviderRuntime {
|
|
|
4065
4086
|
return this.#roleRequestAlias('stt', 'transcribe', payload, options);
|
|
4066
4087
|
}
|
|
4067
4088
|
|
|
4068
|
-
synthesize(payload, options = {}) {
|
|
4089
|
+
synthesize(payload, options = {}, preparation = {}) {
|
|
4069
4090
|
arcaneLogging.debug('[Arcane speech runtime] synthesize', payload, options);
|
|
4070
|
-
return this.#roleRequestAlias('tts', 'synthesize', payload, options);
|
|
4091
|
+
return this.#roleRequestAlias('tts', 'synthesize', payload, options, preparation);
|
|
4071
4092
|
}
|
|
4072
4093
|
|
|
4073
4094
|
cancel(role) {
|
|
@@ -4159,7 +4180,7 @@ export class AIProviderRuntime {
|
|
|
4159
4180
|
) ?? null;
|
|
4160
4181
|
}
|
|
4161
4182
|
|
|
4162
|
-
#roleRequestAlias(role, operation, payload, options) {
|
|
4183
|
+
#roleRequestAlias(role, operation, payload, options, preparation = {}) {
|
|
4163
4184
|
assertPlainObject(options, `AI ${operation} options`);
|
|
4164
4185
|
for (const key of Reflect.ownKeys(options)) {
|
|
4165
4186
|
if (key !== 'localOnly' && key !== 'signal') {
|
|
@@ -4179,7 +4200,8 @@ export class AIProviderRuntime {
|
|
|
4179
4200
|
payload,
|
|
4180
4201
|
localOnly,
|
|
4181
4202
|
signal
|
|
4182
|
-
}
|
|
4203
|
+
},
|
|
4204
|
+
preparation
|
|
4183
4205
|
);
|
|
4184
4206
|
}
|
|
4185
4207
|
|
|
@@ -1,50 +1 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
// Speech-only filtering; single markers and all other text remain literal.
|
|
4
|
-
class MarkdownSpeech {
|
|
5
|
-
#pendingMarker='';
|
|
6
|
-
#repeated=false;
|
|
7
|
-
|
|
8
|
-
append(text='',end=false){
|
|
9
|
-
if(typeof text!=='string'){
|
|
10
|
-
throw new TypeError('Markdown speech input must be text.');
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
let narration='';
|
|
14
|
-
|
|
15
|
-
for(const character of text){
|
|
16
|
-
if(character===this.#pendingMarker){
|
|
17
|
-
this.#repeated=true;
|
|
18
|
-
continue;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
if(this.#pendingMarker&&!this.#repeated){
|
|
22
|
-
narration+=this.#pendingMarker;
|
|
23
|
-
}
|
|
24
|
-
this.reset();
|
|
25
|
-
|
|
26
|
-
if(FORMATTING_MARKERS.has(character)){
|
|
27
|
-
// Wait for the next character to distinguish one mark from a run.
|
|
28
|
-
this.#pendingMarker=character;
|
|
29
|
-
}else{
|
|
30
|
-
narration+=character;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
if(end){
|
|
35
|
-
if(this.#pendingMarker&&!this.#repeated){
|
|
36
|
-
narration+=this.#pendingMarker;
|
|
37
|
-
}
|
|
38
|
-
this.reset();
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
return narration;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
reset(){
|
|
45
|
-
this.#pendingMarker='';
|
|
46
|
-
this.#repeated=false;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
export {MarkdownSpeech};
|
|
1
|
+
export {MarkdownSpeech} from 'arcane-os/speech-text';
|
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
import {createArcaneEventSource} from 'arcane-os/event-manager';
|
|
2
|
+
import {stripSpeechFormatting} from 'arcane-os/speech-text';
|
|
2
3
|
|
|
4
|
+
const SPEECH_VOICE_OPTIONS=[
|
|
5
|
+
{value:'alloy',label:'Alloy'},
|
|
6
|
+
{value:'ash',label:'Ash'},
|
|
7
|
+
{value:'ballad',label:'Ballad'},
|
|
8
|
+
{value:'coral',label:'Coral'},
|
|
9
|
+
{value:'echo',label:'Echo'},
|
|
10
|
+
{value:'fable',label:'Fable'},
|
|
11
|
+
{value:'nova',label:'Nova'},
|
|
12
|
+
{value:'onyx',label:'Onyx'},
|
|
13
|
+
{value:'sage',label:'Sage'},
|
|
14
|
+
{value:'shimmer',label:'Shimmer'}
|
|
15
|
+
];
|
|
3
16
|
const SUPERSEDED={superseded:true};
|
|
4
17
|
const SPEECH_PLAYBACK_STATE_EVENT='speech-playback-state';
|
|
5
18
|
const SPEECH_PLAYBACK_FAILURE_REASONS={
|
|
@@ -17,6 +30,10 @@ const WAV_AUDIO_CONTENT_TYPES=new Set([
|
|
|
17
30
|
]);
|
|
18
31
|
const queuesBySpeechClient=new WeakMap();
|
|
19
32
|
|
|
33
|
+
const SPEECH_VOICE_ALIASES=new Set(
|
|
34
|
+
SPEECH_VOICE_OPTIONS.map(function voiceAlias(option){return option.value;})
|
|
35
|
+
);
|
|
36
|
+
|
|
20
37
|
function splitSpeechText(value=''){
|
|
21
38
|
const text=String(value??'');
|
|
22
39
|
return text.trim()?[text]:[];
|
|
@@ -401,6 +418,7 @@ class SpeechPlayback{
|
|
|
401
418
|
voice=null,
|
|
402
419
|
responseFormat=null,
|
|
403
420
|
speed=1,
|
|
421
|
+
onState=function noop(){},
|
|
404
422
|
createObjectURL,
|
|
405
423
|
revokeObjectURL,
|
|
406
424
|
delay,
|
|
@@ -422,6 +440,7 @@ class SpeechPlayback{
|
|
|
422
440
|
eventTypes:[SPEECH_PLAYBACK_STATE_EVENT]
|
|
423
441
|
}
|
|
424
442
|
);
|
|
443
|
+
this.onState=onState;
|
|
425
444
|
this.createObjectURL=createObjectURL||function createAudioURL(blob){return URL.createObjectURL(blob);};
|
|
426
445
|
this.revokeObjectURL=revokeObjectURL||function revokeAudioURL(url){URL.revokeObjectURL(url);};
|
|
427
446
|
this.delay=delay||function playbackDelay(duration,signal){
|
|
@@ -515,6 +534,12 @@ class SpeechPlayback{
|
|
|
515
534
|
}
|
|
516
535
|
);
|
|
517
536
|
}
|
|
537
|
+
try{
|
|
538
|
+
this.onState(detail);
|
|
539
|
+
}catch(error){
|
|
540
|
+
if(typeof globalThis.reportError==='function')globalThis.reportError(error);
|
|
541
|
+
else console.error(error);
|
|
542
|
+
}
|
|
518
543
|
return detail;
|
|
519
544
|
}
|
|
520
545
|
|
|
@@ -569,8 +594,10 @@ class SpeechPlayback{
|
|
|
569
594
|
}
|
|
570
595
|
|
|
571
596
|
async requestSpeech(part,signal){
|
|
597
|
+
const input=stripSpeechFormatting(part.input);
|
|
598
|
+
const preparation={speechInputPrepared:true};
|
|
572
599
|
const payload={
|
|
573
|
-
input
|
|
600
|
+
input,
|
|
574
601
|
speed:part.speed,
|
|
575
602
|
...(this.model?{model:this.model}:{}),
|
|
576
603
|
...(part.voice?{voice:part.voice}:{}),
|
|
@@ -578,12 +605,12 @@ class SpeechPlayback{
|
|
|
578
605
|
};
|
|
579
606
|
if(typeof this.speech?.fetchTTS==='function'){
|
|
580
607
|
return playableSpeechBlob(
|
|
581
|
-
await this.speech.fetchTTS(payload,signal)
|
|
608
|
+
await this.speech.fetchTTS(payload,signal,preparation)
|
|
582
609
|
);
|
|
583
610
|
}
|
|
584
611
|
if(typeof this.speech?.synthesize==='function'){
|
|
585
612
|
return playableSpeechBlob(
|
|
586
|
-
await this.speech.synthesize(payload,{signal})
|
|
613
|
+
await this.speech.synthesize(payload,{signal},preparation)
|
|
587
614
|
);
|
|
588
615
|
}
|
|
589
616
|
const error=new Error(this.message('unavailable'));
|
|
@@ -1002,12 +1029,15 @@ class SpeechPlayback{
|
|
|
1002
1029
|
this.audio.removeEventListener('error',this.boundError);
|
|
1003
1030
|
this.destroyed=true;
|
|
1004
1031
|
this.events.dispose();
|
|
1032
|
+
this.onState=function destroyedSpeechPlaybackStateObserver(){};
|
|
1005
1033
|
return true;
|
|
1006
1034
|
}
|
|
1007
1035
|
}
|
|
1008
1036
|
|
|
1009
1037
|
export {
|
|
1010
1038
|
SPEECH_PLAYBACK_STATE_EVENT,
|
|
1039
|
+
SPEECH_VOICE_ALIASES,
|
|
1040
|
+
SPEECH_VOICE_OPTIONS,
|
|
1011
1041
|
SpeechPlayback,
|
|
1012
1042
|
splitSpeechText
|
|
1013
1043
|
};
|
package/src/import-map.mjs
CHANGED
|
@@ -20,6 +20,7 @@ const STATIC_RUNTIME_PACKAGE_IMPORTS=new Map([
|
|
|
20
20
|
const SDK_BROWSER_SELF_IMPORTS=new Map([
|
|
21
21
|
['arcane-os/event-manager',SDK_BROWSER_ENTRY],
|
|
22
22
|
['arcane-os/logging','sdk/logging.mjs'],
|
|
23
|
+
['arcane-os/speech-text','sdk/speech-text.mjs'],
|
|
23
24
|
['arcane-os/ai/browser-wasm',SDK_BROWSER_AI_ENTRY],
|
|
24
25
|
['arcane-os/ai/browser-speech',SDK_BROWSER_SPEECH_ENTRY]
|
|
25
26
|
]);
|