@gradio/multimodaltextbox 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/Index.svelte +60 -2
  3. package/MultimodalTextbox.stories.svelte +10 -0
  4. package/dist/Index.svelte +49 -2
  5. package/dist/Index.svelte.d.ts +23 -4
  6. package/dist/shared/MultimodalTextbox.svelte +185 -120
  7. package/dist/shared/MultimodalTextbox.svelte.d.ts +9 -1
  8. package/package.json +9 -9
  9. package/shared/MultimodalTextbox.svelte +191 -122
package/CHANGELOG.md CHANGED
@@ -1,5 +1,22 @@
1
1
  # @gradio/multimodaltextbox
2
2
 
3
+ ## 0.9.0
4
+
5
+ ### Features
6
+
7
+ - [#10186](https://github.com/gradio-app/gradio/pull/10186) [`9b17032`](https://github.com/gradio-app/gradio/commit/9b17032a6564c8bda4ab1749989a9797c670919c) - Add Microphone Input to MultimodalTextbox. Thanks @dawoodkhan82!
8
+
9
+ ### Dependency updates
10
+
11
+ - @gradio/atoms@0.13.0
12
+ - @gradio/utils@0.10.0
13
+ - @gradio/upload@0.14.4
14
+ - @gradio/client@1.9.0
15
+ - @gradio/icons@0.9.0
16
+ - @gradio/statustracker@0.10.0
17
+ - @gradio/image@0.20.0
18
+ - @gradio/video@0.13.0
19
+
3
20
  ## 0.8.0
4
21
 
5
22
  ### Features
package/Index.svelte CHANGED
@@ -12,6 +12,8 @@
12
12
  import { StatusTracker } from "@gradio/statustracker";
13
13
  import type { LoadingStatus } from "@gradio/statustracker";
14
14
  import type { FileData } from "@gradio/client";
15
+ import { onMount } from "svelte";
16
+ import type { WaveformOptions } from "../audio/shared/types";
15
17
 
16
18
  export let gradio: Gradio<{
17
19
  change: typeof value;
@@ -23,6 +25,11 @@
23
25
  focus: never;
24
26
  error: string;
25
27
  clear_status: LoadingStatus;
28
+ start_recording: never;
29
+ pause_recording: never;
30
+ stop_recording: never;
31
+ upload: FileData[] | FileData;
32
+ clear: undefined;
26
33
  }>;
27
34
  export let elem_id = "";
28
35
  export let elem_classes: string[] = [];
@@ -38,7 +45,6 @@
38
45
  export let info: string | undefined = undefined;
39
46
  export let show_label: boolean;
40
47
  export let max_lines: number;
41
- export let container = true;
42
48
  export let scale: number | null = null;
43
49
  export let min_width: number | undefined = undefined;
44
50
  export let submit_btn: string | boolean | null = null;
@@ -53,8 +59,52 @@
53
59
  export let root: string;
54
60
  export let file_count: "single" | "multiple" | "directory";
55
61
  export let max_plain_text_length: number;
62
+ export let sources: ["microphone" | "upload"] = ["upload"];
63
+ export let waveform_options: WaveformOptions = {};
56
64
 
57
65
  let dragging: boolean;
66
+ let active_source: "microphone" | null = null;
67
+ let waveform_settings: Record<string, any>;
68
+ let color_accent = "darkorange";
69
+
70
+ onMount(() => {
71
+ color_accent = getComputedStyle(document?.documentElement).getPropertyValue(
72
+ "--color-accent"
73
+ );
74
+ set_trim_region_colour();
75
+ waveform_settings.waveColor = waveform_options.waveform_color || "#9ca3af";
76
+ waveform_settings.progressColor =
77
+ waveform_options.waveform_progress_color || color_accent;
78
+ waveform_settings.mediaControls = waveform_options.show_controls;
79
+ waveform_settings.sampleRate = waveform_options.sample_rate || 44100;
80
+ });
81
+
82
+ $: waveform_settings = {
83
+ height: 50,
84
+
85
+ barWidth: 2,
86
+ barGap: 3,
87
+ cursorWidth: 2,
88
+ cursorColor: "#ddd5e9",
89
+ autoplay: false,
90
+ barRadius: 10,
91
+ dragToSeek: true,
92
+ normalize: true,
93
+ minPxPerSec: 20
94
+ };
95
+
96
+ const trim_region_settings = {
97
+ color: waveform_options.trim_region_color,
98
+ drag: true,
99
+ resize: true
100
+ };
101
+
102
+ function set_trim_region_colour(): void {
103
+ document.documentElement.style.setProperty(
104
+ "--trim-region-color",
105
+ trim_region_settings.color || color_accent
106
+ );
107
+ }
58
108
  </script>
59
109
 
60
110
  <Block
@@ -80,6 +130,7 @@
80
130
  bind:value
81
131
  bind:value_is_output
82
132
  bind:dragging
133
+ bind:active_source
83
134
  {file_types}
84
135
  {root}
85
136
  {label}
@@ -88,14 +139,16 @@
88
139
  {lines}
89
140
  {rtl}
90
141
  {text_align}
142
+ {waveform_settings}
143
+ i18n={gradio.i18n}
91
144
  max_lines={!max_lines ? lines + 1 : max_lines}
92
145
  {placeholder}
93
146
  {submit_btn}
94
147
  {stop_btn}
95
148
  {autofocus}
96
- {container}
97
149
  {autoscroll}
98
150
  {file_count}
151
+ {sources}
99
152
  max_file_size={gradio.max_file_size}
100
153
  on:change={() => gradio.dispatch("change", value)}
101
154
  on:input={() => gradio.dispatch("input")}
@@ -107,6 +160,11 @@
107
160
  on:error={({ detail }) => {
108
161
  gradio.dispatch("error", detail);
109
162
  }}
163
+ on:start_recording={() => gradio.dispatch("start_recording")}
164
+ on:pause_recording={() => gradio.dispatch("pause_recording")}
165
+ on:stop_recording={() => gradio.dispatch("stop_recording")}
166
+ on:upload={(e) => gradio.dispatch("upload", e.detail)}
167
+ on:clear={() => gradio.dispatch("clear")}
110
168
  disabled={!interactive}
111
169
  upload={(...args) => gradio.client.upload(...args)}
112
170
  stream_handler={(...args) => gradio.client.stream(...args)}
package/MultimodalTextbox.stories.svelte CHANGED
@@ -42,6 +42,12 @@
42
42
  description: "Whether to render right-to-left",
43
43
  control: { type: "boolean" },
44
44
  defaultValue: false
45
+ },
46
+ sources: {
47
+ options: ["upload", "microphone"],
48
+ description: "The sources to enable",
49
+ control: { type: "select" },
50
+ defaultValue: ["upload", "microphone"]
45
51
  }
46
52
  }}
47
53
  />
@@ -87,3 +93,7 @@
87
93
  }
88
94
  }}
89
95
  />
96
+ <Story
97
+ name="MultimodalTextbox with microphone input"
98
+ args={{ sources: ["microphone"] }}
99
+ />
package/dist/Index.svelte CHANGED
@@ -7,6 +7,7 @@ export { default as BaseExample } from "./Example.svelte";
7
7
  <script>import MultimodalTextbox from "./shared/MultimodalTextbox.svelte";
8
8
  import { Block } from "@gradio/atoms";
9
9
  import { StatusTracker } from "@gradio/statustracker";
10
+ import { onMount } from "svelte";
10
11
  export let gradio;
11
12
  export let elem_id = "";
12
13
  export let elem_classes = [];
@@ -22,7 +23,6 @@ export let label = "MultimodalTextbox";
22
23
  export let info = void 0;
23
24
  export let show_label;
24
25
  export let max_lines;
25
- export let container = true;
26
26
  export let scale = null;
27
27
  export let min_width = void 0;
28
28
  export let submit_btn = null;
@@ -37,7 +37,46 @@ export let interactive;
37
37
  export let root;
38
38
  export let file_count;
39
39
  export let max_plain_text_length;
40
+ export let sources = ["upload"];
41
+ export let waveform_options = {};
40
42
  let dragging;
43
+ let active_source = null;
44
+ let waveform_settings;
45
+ let color_accent = "darkorange";
46
+ onMount(() => {
47
+ color_accent = getComputedStyle(document?.documentElement).getPropertyValue(
48
+ "--color-accent"
49
+ );
50
+ set_trim_region_colour();
51
+ waveform_settings.waveColor = waveform_options.waveform_color || "#9ca3af";
52
+ waveform_settings.progressColor = waveform_options.waveform_progress_color || color_accent;
53
+ waveform_settings.mediaControls = waveform_options.show_controls;
54
+ waveform_settings.sampleRate = waveform_options.sample_rate || 44100;
55
+ });
56
+ $:
57
+ waveform_settings = {
58
+ height: 50,
59
+ barWidth: 2,
60
+ barGap: 3,
61
+ cursorWidth: 2,
62
+ cursorColor: "#ddd5e9",
63
+ autoplay: false,
64
+ barRadius: 10,
65
+ dragToSeek: true,
66
+ normalize: true,
67
+ minPxPerSec: 20
68
+ };
69
+ const trim_region_settings = {
70
+ color: waveform_options.trim_region_color,
71
+ drag: true,
72
+ resize: true
73
+ };
74
+ function set_trim_region_colour() {
75
+ document.documentElement.style.setProperty(
76
+ "--trim-region-color",
77
+ trim_region_settings.color || color_accent
78
+ );
79
+ }
41
80
  </script>
42
81
 
43
82
  <Block
@@ -63,6 +102,7 @@ let dragging;
63
102
  bind:value
64
103
  bind:value_is_output
65
104
  bind:dragging
105
+ bind:active_source
66
106
  {file_types}
67
107
  {root}
68
108
  {label}
@@ -71,14 +111,16 @@ let dragging;
71
111
  {lines}
72
112
  {rtl}
73
113
  {text_align}
114
+ {waveform_settings}
115
+ i18n={gradio.i18n}
74
116
  max_lines={!max_lines ? lines + 1 : max_lines}
75
117
  {placeholder}
76
118
  {submit_btn}
77
119
  {stop_btn}
78
120
  {autofocus}
79
- {container}
80
121
  {autoscroll}
81
122
  {file_count}
123
+ {sources}
82
124
  max_file_size={gradio.max_file_size}
83
125
  on:change={() => gradio.dispatch("change", value)}
84
126
  on:input={() => gradio.dispatch("input")}
@@ -90,6 +132,11 @@ let dragging;
90
132
  on:error={({ detail }) => {
91
133
  gradio.dispatch("error", detail);
92
134
  }}
135
+ on:start_recording={() => gradio.dispatch("start_recording")}
136
+ on:pause_recording={() => gradio.dispatch("pause_recording")}
137
+ on:stop_recording={() => gradio.dispatch("stop_recording")}
138
+ on:upload={(e) => gradio.dispatch("upload", e.detail)}
139
+ on:clear={() => gradio.dispatch("clear")}
93
140
  disabled={!interactive}
94
141
  upload={(...args) => gradio.client.upload(...args)}
95
142
  stream_handler={(...args) => gradio.client.stream(...args)}
package/dist/Index.svelte.d.ts CHANGED
@@ -19,6 +19,11 @@ declare const __propDef: {
19
19
  focus: never;
20
20
  error: string;
21
21
  clear_status: LoadingStatus;
22
+ start_recording: never;
23
+ pause_recording: never;
24
+ stop_recording: never;
25
+ upload: FileData[] | FileData;
26
+ clear: undefined;
22
27
  }>;
23
28
  elem_id?: string | undefined;
24
29
  elem_classes?: string[] | undefined;
@@ -34,7 +39,6 @@ declare const __propDef: {
34
39
  info?: string | undefined;
35
40
  show_label: boolean;
36
41
  max_lines: number;
37
- container?: boolean | undefined;
38
42
  scale?: (number | null) | undefined;
39
43
  min_width?: number | undefined;
40
44
  submit_btn?: (string | boolean | null) | undefined;
@@ -49,6 +53,8 @@ declare const __propDef: {
49
53
  root: string;
50
54
  file_count: "single" | "multiple" | "directory";
51
55
  max_plain_text_length: number;
56
+ sources?: ["microphone" | "upload"] | undefined;
57
+ waveform_options?: any;
52
58
  };
53
59
  events: {
54
60
  [evt: string]: CustomEvent<any>;
@@ -72,6 +78,11 @@ export default class Index extends SvelteComponent<IndexProps, IndexEvents, Inde
72
78
  focus: never;
73
79
  error: string;
74
80
  clear_status: LoadingStatus;
81
+ start_recording: never;
82
+ pause_recording: never;
83
+ stop_recording: never;
84
+ upload: FileData[] | FileData;
85
+ clear: undefined;
75
86
  }>;
76
87
  /**accessor*/
77
88
  set gradio(_: Gradio<{
@@ -87,6 +98,11 @@ export default class Index extends SvelteComponent<IndexProps, IndexEvents, Inde
87
98
  focus: never;
88
99
  error: string;
89
100
  clear_status: LoadingStatus;
101
+ start_recording: never;
102
+ pause_recording: never;
103
+ stop_recording: never;
104
+ upload: FileData[] | FileData;
105
+ clear: undefined;
90
106
  }>);
91
107
  get elem_id(): string | undefined;
92
108
  /**accessor*/
@@ -127,9 +143,6 @@ export default class Index extends SvelteComponent<IndexProps, IndexEvents, Inde
127
143
  get max_lines(): number;
128
144
  /**accessor*/
129
145
  set max_lines(_: number);
130
- get container(): boolean | undefined;
131
- /**accessor*/
132
- set container(_: boolean | undefined);
133
146
  get scale(): number | null | undefined;
134
147
  /**accessor*/
135
148
  set scale(_: number | null | undefined);
@@ -172,4 +185,10 @@ export default class Index extends SvelteComponent<IndexProps, IndexEvents, Inde
172
185
  get max_plain_text_length(): number;
173
186
  /**accessor*/
174
187
  set max_plain_text_length(_: number);
188
+ get sources(): ["upload" | "microphone"] | undefined;
189
+ /**accessor*/
190
+ set sources(_: ["upload" | "microphone"] | undefined);
191
+ get waveform_options(): any;
192
+ /**accessor*/
193
+ set waveform_options(_: any);
175
194
  }
package/dist/shared/MultimodalTextbox.svelte CHANGED
@@ -16,20 +16,22 @@ import {
16
16
  Paperclip,
17
17
  Video,
18
18
  Send,
19
- Square
19
+ Square,
20
+ Microphone
20
21
  } from "@gradio/icons";
22
+ import InteractiveAudio from "../../audio/interactive/InteractiveAudio.svelte";
21
23
  export let value = {
22
24
  text: "",
23
25
  files: []
24
26
  };
25
27
  export let value_is_output = false;
26
28
  export let lines = 1;
29
+ export let i18n;
27
30
  export let placeholder = "Type here...";
28
31
  export let disabled = false;
29
32
  export let label;
30
33
  export let info = void 0;
31
34
  export let show_label = true;
32
- export let container = true;
33
35
  export let max_lines;
34
36
  export let submit_btn = null;
35
37
  export let stop_btn = null;
@@ -44,6 +46,10 @@ export let upload;
44
46
  export let stream_handler;
45
47
  export let file_count = "multiple";
46
48
  export let max_plain_text_length = 1e3;
49
+ export let waveform_settings;
50
+ export let waveform_options = {};
51
+ export let sources = ["upload"];
52
+ export let active_source = null;
47
53
  let upload_component;
48
54
  let hidden_upload;
49
55
  let el;
@@ -53,8 +59,10 @@ let user_has_scrolled_up = false;
53
59
  export let dragging = false;
54
60
  let uploading = false;
55
61
  let oldValue = value.text;
62
+ let recording = false;
56
63
  $:
57
64
  dispatch("drag", dragging);
65
+ let mic_audio = null;
58
66
  let full_container;
59
67
  $:
60
68
  if (oldValue !== value.text) {
@@ -109,6 +117,11 @@ async function handle_keypress(e) {
109
117
  } else if (e.key === "Enter" && !e.shiftKey && lines === 1 && max_lines >= 1) {
110
118
  e.preventDefault();
111
119
  dispatch("submit");
120
+ active_source = null;
121
+ if (mic_audio) {
122
+ value.files.push(mic_audio);
123
+ value = value;
124
+ }
112
125
  }
113
126
  }
114
127
  function handle_scroll(event) {
@@ -158,6 +171,11 @@ function handle_stop() {
158
171
  }
159
172
  function handle_submit() {
160
173
  dispatch("submit");
174
+ active_source = null;
175
+ if (mic_audio) {
176
+ value.files.push(mic_audio);
177
+ value = value;
178
+ }
161
179
  }
162
180
  async function handle_paste(event) {
163
181
  if (!event.clipboardData)
@@ -238,127 +256,167 @@ function handle_drop(event) {
238
256
  role="group"
239
257
  aria-label="Multimedia input field"
240
258
  >
241
- <!-- svelte-ignore a11y-autofocus -->
242
- <label class:container>
243
- <BlockTitle {root} {show_label} {info}>{label}</BlockTitle>
244
- {#if value.files.length > 0 || uploading}
245
- <div
246
- class="thumbnails scroll-hide"
247
- aria-label="Uploaded files"
248
- data-testid="container_el"
249
- style="display: {value.files.length > 0 || uploading
250
- ? 'flex'
251
- : 'none'};"
259
+ <BlockTitle {root} {show_label} {info}>{label}</BlockTitle>
260
+ {#if value.files.length > 0 || uploading}
261
+ <div
262
+ class="thumbnails scroll-hide"
263
+ aria-label="Uploaded files"
264
+ data-testid="container_el"
265
+ style="display: {value.files.length > 0 || uploading ? 'flex' : 'none'};"
266
+ >
267
+ {#each value.files as file, index}
268
+ <span role="listitem" aria-label="File thumbnail">
269
+ <button class="thumbnail-item thumbnail-small">
270
+ <button
271
+ class:disabled
272
+ class="delete-button"
273
+ on:click={(event) => remove_thumbnail(event, index)}
274
+ ><Clear /></button
275
+ >
276
+ {#if file.mime_type && file.mime_type.includes("image")}
277
+ <Image
278
+ src={file.url}
279
+ title={null}
280
+ alt=""
281
+ loading="lazy"
282
+ class={"thumbnail-image"}
283
+ />
284
+ {:else if file.mime_type && file.mime_type.includes("audio")}
285
+ <Music />
286
+ {:else if file.mime_type && file.mime_type.includes("video")}
287
+ <Video />
288
+ {:else}
289
+ <File />
290
+ {/if}
291
+ </button>
292
+ </span>
293
+ {/each}
294
+ {#if uploading}
295
+ <div class="loader" role="status" aria-label="Uploading"></div>
296
+ {/if}
297
+ </div>
298
+ {/if}
299
+ {#if sources && sources.includes("microphone") && active_source === "microphone"}
300
+ <InteractiveAudio
301
+ on:change={({ detail }) => {
302
+ if (detail !== null) {
303
+ mic_audio = detail;
304
+ }
305
+ }}
306
+ on:clear={() => {
307
+ active_source = null;
308
+ }}
309
+ on:start_recording={() => dispatch("start_recording")}
310
+ on:pause_recording={() => dispatch("pause_recording")}
311
+ on:stop_recording={() => dispatch("stop_recording")}
312
+ sources={["microphone"]}
313
+ class_name="compact-audio"
314
+ {recording}
315
+ {waveform_settings}
316
+ {waveform_options}
317
+ {i18n}
318
+ {active_source}
319
+ {upload}
320
+ {stream_handler}
321
+ stream_every={1}
322
+ editable={true}
323
+ {label}
324
+ {root}
325
+ loop={false}
326
+ show_label={false}
327
+ show_download_button={false}
328
+ dragging={false}
329
+ />
330
+ {/if}
331
+ <div class="input-container">
332
+ {#if sources && sources.includes("upload") && !disabled && !(file_count === "single" && value.files.length > 0)}
333
+ <Upload
334
+ bind:this={upload_component}
335
+ on:load={handle_upload}
336
+ {file_count}
337
+ filetype={file_types}
338
+ {root}
339
+ {max_file_size}
340
+ bind:dragging
341
+ bind:uploading
342
+ show_progress={false}
343
+ disable_click={true}
344
+ bind:hidden_upload
345
+ on:error
346
+ hidden={true}
347
+ {upload}
348
+ {stream_handler}
349
+ />
350
+ <button
351
+ data-testid="upload-button"
352
+ class="upload-button"
353
+ on:click={handle_upload_click}><Paperclip /></button
252
354
  >
253
- {#each value.files as file, index}
254
- <span role="listitem" aria-label="File thumbnail">
255
- <button class="thumbnail-item thumbnail-small">
256
- <button
257
- class:disabled
258
- class="delete-button"
259
- on:click={(event) => remove_thumbnail(event, index)}
260
- ><Clear /></button
261
- >
262
- {#if file.mime_type && file.mime_type.includes("image")}
263
- <Image
264
- src={file.url}
265
- title={null}
266
- alt=""
267
- loading="lazy"
268
- class={"thumbnail-image"}
269
- />
270
- {:else if file.mime_type && file.mime_type.includes("audio")}
271
- <Music />
272
- {:else if file.mime_type && file.mime_type.includes("video")}
273
- <Video />
274
- {:else}
275
- <File />
276
- {/if}
277
- </button>
278
- </span>
279
- {/each}
280
- {#if uploading}
281
- <div class="loader" role="status" aria-label="Uploading"></div>
282
- {/if}
283
- </div>
284
355
  {/if}
285
- <div class="input-container">
286
- {#if !disabled && !(file_count === "single" && value.files.length > 0)}
287
- <Upload
288
- bind:this={upload_component}
289
- on:load={handle_upload}
290
- {file_count}
291
- filetype={file_types}
292
- {root}
293
- {max_file_size}
294
- bind:dragging
295
- bind:uploading
296
- show_progress={false}
297
- disable_click={true}
298
- bind:hidden_upload
299
- on:error
300
- hidden={true}
301
- {upload}
302
- {stream_handler}
303
- ></Upload>
304
- <button
305
- data-testid="upload-button"
306
- class="upload-button"
307
- on:click={handle_upload_click}><Paperclip /></button
308
- >
309
- {/if}
310
- <textarea
311
- data-testid="textbox"
312
- use:text_area_resize={{
313
- text: value.text,
314
- lines: lines,
315
- max_lines: max_lines
356
+ {#if sources && sources.includes("microphone")}
357
+ <button
358
+ data-testid="microphone-button"
359
+ class="microphone-button"
360
+ class:recording
361
+ on:click={() => {
362
+ active_source = active_source !== "microphone" ? "microphone" : null;
316
363
  }}
317
- class="scroll-hide"
318
- class:no-label={!show_label}
319
- dir={rtl ? "rtl" : "ltr"}
320
- bind:value={value.text}
321
- bind:this={el}
322
- {placeholder}
323
- rows={lines}
324
- {disabled}
325
- {autofocus}
326
- on:keypress={handle_keypress}
327
- on:blur
328
- on:select={handle_select}
329
- on:focus
330
- on:scroll={handle_scroll}
331
- on:paste={handle_paste}
332
- style={text_align ? "text-align: " + text_align : ""}
333
- />
334
- {#if submit_btn}
335
- <button
336
- class="submit-button"
337
- class:padded-button={submit_btn !== true}
338
- on:click={handle_submit}
339
- >
340
- {#if submit_btn === true}
341
- <Send />
342
- {:else}
343
- {submit_btn}
344
- {/if}
345
- </button>
346
- {/if}
347
- {#if stop_btn}
348
- <button
349
- class="stop-button"
350
- class:padded-button={stop_btn !== true}
351
- on:click={handle_stop}
352
- >
353
- {#if stop_btn === true}
354
- <Square fill={"none"} stroke_width={2.5} />
355
- {:else}
356
- {stop_btn}
357
- {/if}
358
- </button>
359
- {/if}
360
- </div>
361
- </label>
364
+ >
365
+ <Microphone />
366
+ </button>
367
+ {/if}
368
+ <!-- svelte-ignore a11y-autofocus -->
369
+ <textarea
370
+ data-testid="textbox"
371
+ use:text_area_resize={{
372
+ text: value.text,
373
+ lines: lines,
374
+ max_lines: max_lines
375
+ }}
376
+ class="scroll-hide"
377
+ class:no-label={!show_label}
378
+ dir={rtl ? "rtl" : "ltr"}
379
+ bind:value={value.text}
380
+ bind:this={el}
381
+ {placeholder}
382
+ rows={lines}
383
+ {disabled}
384
+ {autofocus}
385
+ on:keypress={handle_keypress}
386
+ on:blur
387
+ on:select={handle_select}
388
+ on:focus
389
+ on:scroll={handle_scroll}
390
+ on:paste={handle_paste}
391
+ style={text_align ? "text-align: " + text_align : ""}
392
+ />
393
+ {#if submit_btn}
394
+ <button
395
+ class="submit-button"
396
+ class:padded-button={submit_btn !== true}
397
+ on:click={handle_submit}
398
+ >
399
+ {#if submit_btn === true}
400
+ <Send />
401
+ {:else}
402
+ {submit_btn}
403
+ {/if}
404
+ </button>
405
+ {/if}
406
+ {#if stop_btn}
407
+ <button
408
+ class="stop-button"
409
+ class:padded-button={stop_btn !== true}
410
+ on:click={handle_stop}
411
+ >
412
+ {#if stop_btn === true}
413
+ <Square fill={"none"} stroke_width={2.5} />
414
+ {:else}
415
+ {stop_btn}
416
+ {/if}
417
+ </button>
418
+ {/if}
419
+ </div>
362
420
  </div>
363
421
 
364
422
  <style>
@@ -420,6 +478,7 @@ function handle_drop(event) {
420
478
  color: var(--input-placeholder-color);
421
479
  }
422
480
 
481
+ .microphone-button,
423
482
  .upload-button,
424
483
  .submit-button,
425
484
  .stop-button {
@@ -436,29 +495,34 @@ function handle_drop(event) {
436
495
  justify-content: center;
437
496
  align-items: center;
438
497
  z-index: var(--layer-1);
498
+ margin-left: var(--spacing-sm);
439
499
  }
440
500
  .padded-button {
441
501
  padding: 0 10px;
442
502
  }
443
503
 
504
+ .microphone-button,
444
505
  .stop-button,
445
506
  .upload-button,
446
507
  .submit-button {
447
508
  background: var(--button-secondary-background-fill);
448
509
  }
449
510
 
511
+ .microphone-button:hover,
450
512
  .stop-button:hover,
451
513
  .upload-button:hover,
452
514
  .submit-button:hover {
453
515
  background: var(--button-secondary-background-fill-hover);
454
516
  }
455
517
 
518
+ .microphone-button:disabled,
456
519
  .stop-button:disabled,
457
520
  .upload-button:disabled,
458
521
  .submit-button:disabled {
459
522
  background: var(--button-secondary-background-fill);
460
523
  cursor: initial;
461
524
  }
525
+ .microphone-button:active,
462
526
  .stop-button:active,
463
527
  .upload-button:active,
464
528
  .submit-button:active {
@@ -469,6 +533,7 @@ function handle_drop(event) {
469
533
  height: 22px;
470
534
  width: 22px;
471
535
  }
536
+ .microphone-button :global(svg),
472
537
  .upload-button :global(svg) {
473
538
  height: 17px;
474
539
  width: 17px;
package/dist/shared/MultimodalTextbox.svelte.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { SvelteComponent } from "svelte";
2
+ import type { I18nFormatter } from "js/core/src/gradio_helper";
2
3
  import type { FileData, Client } from "@gradio/client";
3
4
  import type { SelectData } from "@gradio/utils";
4
5
  declare const __propDef: {
@@ -9,12 +10,12 @@ declare const __propDef: {
9
10
  } | undefined;
10
11
  value_is_output?: boolean | undefined;
11
12
  lines?: number | undefined;
13
+ i18n: I18nFormatter;
12
14
  placeholder?: string | undefined;
13
15
  disabled?: boolean | undefined;
14
16
  label: string;
15
17
  info?: string | undefined;
16
18
  show_label?: boolean | undefined;
17
- container?: boolean | undefined;
18
19
  max_lines: number;
19
20
  submit_btn?: (string | boolean | null) | undefined;
20
21
  stop_btn?: (string | boolean | null) | undefined;
@@ -29,6 +30,10 @@ declare const __propDef: {
29
30
  stream_handler: Client["stream"];
30
31
  file_count?: ("single" | "multiple" | "directory") | undefined;
31
32
  max_plain_text_length?: number | undefined;
33
+ waveform_settings: Record<string, any>;
34
+ waveform_options?: any;
35
+ sources?: ["microphone" | "upload"] | undefined;
36
+ active_source?: ("microphone" | null) | undefined;
32
37
  dragging?: boolean | undefined;
33
38
  };
34
39
  events: {
@@ -48,6 +53,9 @@ declare const __propDef: {
48
53
  upload: CustomEvent<any>;
49
54
  clear: CustomEvent<undefined>;
50
55
  load: CustomEvent<any>;
56
+ start_recording: CustomEvent<undefined>;
57
+ pause_recording: CustomEvent<undefined>;
58
+ stop_recording: CustomEvent<undefined>;
51
59
  } & {
52
60
  [evt: string]: CustomEvent<any>;
53
61
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gradio/multimodaltextbox",
3
- "version": "0.8.0",
3
+ "version": "0.9.0",
4
4
  "description": "Gradio UI packages",
5
5
  "type": "module",
6
6
  "author": "",
@@ -22,14 +22,14 @@
22
22
  "./package.json": "./package.json"
23
23
  },
24
24
  "dependencies": {
25
- "@gradio/atoms": "^0.12.0",
26
- "@gradio/statustracker": "^0.9.7",
27
- "@gradio/utils": "^0.9.0",
28
- "@gradio/upload": "^0.14.3",
29
- "@gradio/video": "^0.12.1",
30
- "@gradio/image": "^0.19.0",
31
- "@gradio/icons": "^0.8.1",
32
- "@gradio/client": "^1.8.0"
25
+ "@gradio/icons": "^0.9.0",
26
+ "@gradio/atoms": "^0.13.0",
27
+ "@gradio/statustracker": "^0.10.0",
28
+ "@gradio/upload": "^0.14.4",
29
+ "@gradio/utils": "^0.10.0",
30
+ "@gradio/image": "^0.20.0",
31
+ "@gradio/video": "^0.13.0",
32
+ "@gradio/client": "^1.9.0"
33
33
  },
34
34
  "devDependencies": {
35
35
  "@gradio/preview": "^0.13.0"
package/shared/MultimodalTextbox.svelte CHANGED
@@ -10,7 +10,9 @@
10
10
  import { BlockTitle } from "@gradio/atoms";
11
11
  import { Upload } from "@gradio/upload";
12
12
  import { Image } from "@gradio/image/shared";
13
+ import type { I18nFormatter } from "js/core/src/gradio_helper";
13
14
  import type { FileData, Client } from "@gradio/client";
15
+ import type { WaveformOptions } from "../../audio/shared/types";
14
16
  import {
15
17
  Clear,
16
18
  File,
@@ -18,9 +20,11 @@
18
20
  Paperclip,
19
21
  Video,
20
22
  Send,
21
- Square
23
+ Square,
24
+ Microphone
22
25
  } from "@gradio/icons";
23
26
  import type { SelectData } from "@gradio/utils";
27
+ import InteractiveAudio from "../../audio/interactive/InteractiveAudio.svelte";
24
28
 
25
29
  export let value: { text: string; files: FileData[] } = {
26
30
  text: "",
@@ -29,12 +33,12 @@
29
33
 
30
34
  export let value_is_output = false;
31
35
  export let lines = 1;
36
+ export let i18n: I18nFormatter;
32
37
  export let placeholder = "Type here...";
33
38
  export let disabled = false;
34
39
  export let label: string;
35
40
  export let info: string | undefined = undefined;
36
41
  export let show_label = true;
37
- export let container = true;
38
42
  export let max_lines: number;
39
43
  export let submit_btn: string | boolean | null = null;
40
44
  export let stop_btn: string | boolean | null = null;
@@ -49,7 +53,10 @@
49
53
  export let stream_handler: Client["stream"];
50
54
  export let file_count: "single" | "multiple" | "directory" = "multiple";
51
55
  export let max_plain_text_length = 1000;
52
-
56
+ export let waveform_settings: Record<string, any>;
57
+ export let waveform_options: WaveformOptions = {};
58
+ export let sources: ["microphone" | "upload"] = ["upload"];
59
+ export let active_source: "microphone" | null = null;
53
60
  let upload_component: Upload;
54
61
  let hidden_upload: HTMLInputElement;
55
62
  let el: HTMLTextAreaElement | HTMLInputElement;
@@ -59,7 +66,9 @@
59
66
  export let dragging = false;
60
67
  let uploading = false;
61
68
  let oldValue = value.text;
69
+ let recording = false;
62
70
  $: dispatch("drag", dragging);
71
+ let mic_audio: FileData | null = null;
63
72
 
64
73
  let full_container: HTMLDivElement;
65
74
 
@@ -84,6 +93,9 @@
84
93
  clear: undefined;
85
94
  load: FileData[] | FileData;
86
95
  error: string;
96
+ start_recording: undefined;
97
+ pause_recording: undefined;
98
+ stop_recording: undefined;
87
99
  }>();
88
100
 
89
101
  beforeUpdate(() => {
@@ -141,6 +153,11 @@
141
153
  ) {
142
154
  e.preventDefault();
143
155
  dispatch("submit");
156
+ active_source = null;
157
+ if (mic_audio) {
158
+ value.files.push(mic_audio);
159
+ value = value;
160
+ }
144
161
  }
145
162
  }
146
163
 
@@ -161,7 +178,7 @@
161
178
 
162
179
  async function handle_upload({
163
180
  detail
164
- }: CustomEvent<FileData | FileData[]>): Promise<void> {
181
+ }: CustomEvent<FileData>): Promise<void> {
165
182
  handle_change();
166
183
  if (Array.isArray(detail)) {
167
184
  for (let file of detail) {
@@ -197,6 +214,11 @@
197
214
 
198
215
  function handle_submit(): void {
199
216
  dispatch("submit");
217
+ active_source = null;
218
+ if (mic_audio) {
219
+ value.files.push(mic_audio);
220
+ value = value;
221
+ }
200
222
  }
201
223
 
202
224
  async function handle_paste(event: ClipboardEvent): Promise<void> {
@@ -289,127 +311,167 @@
289
311
  role="group"
290
312
  aria-label="Multimedia input field"
291
313
  >
292
- <!-- svelte-ignore a11y-autofocus -->
293
- <label class:container>
294
- <BlockTitle {root} {show_label} {info}>{label}</BlockTitle>
295
- {#if value.files.length > 0 || uploading}
296
- <div
297
- class="thumbnails scroll-hide"
298
- aria-label="Uploaded files"
299
- data-testid="container_el"
300
- style="display: {value.files.length > 0 || uploading
301
- ? 'flex'
302
- : 'none'};"
314
+ <BlockTitle {root} {show_label} {info}>{label}</BlockTitle>
315
+ {#if value.files.length > 0 || uploading}
316
+ <div
317
+ class="thumbnails scroll-hide"
318
+ aria-label="Uploaded files"
319
+ data-testid="container_el"
320
+ style="display: {value.files.length > 0 || uploading ? 'flex' : 'none'};"
321
+ >
322
+ {#each value.files as file, index}
323
+ <span role="listitem" aria-label="File thumbnail">
324
+ <button class="thumbnail-item thumbnail-small">
325
+ <button
326
+ class:disabled
327
+ class="delete-button"
328
+ on:click={(event) => remove_thumbnail(event, index)}
329
+ ><Clear /></button
330
+ >
331
+ {#if file.mime_type && file.mime_type.includes("image")}
332
+ <Image
333
+ src={file.url}
334
+ title={null}
335
+ alt=""
336
+ loading="lazy"
337
+ class={"thumbnail-image"}
338
+ />
339
+ {:else if file.mime_type && file.mime_type.includes("audio")}
340
+ <Music />
341
+ {:else if file.mime_type && file.mime_type.includes("video")}
342
+ <Video />
343
+ {:else}
344
+ <File />
345
+ {/if}
346
+ </button>
347
+ </span>
348
+ {/each}
349
+ {#if uploading}
350
+ <div class="loader" role="status" aria-label="Uploading"></div>
351
+ {/if}
352
+ </div>
353
+ {/if}
354
+ {#if sources && sources.includes("microphone") && active_source === "microphone"}
355
+ <InteractiveAudio
356
+ on:change={({ detail }) => {
357
+ if (detail !== null) {
358
+ mic_audio = detail;
359
+ }
360
+ }}
361
+ on:clear={() => {
362
+ active_source = null;
363
+ }}
364
+ on:start_recording={() => dispatch("start_recording")}
365
+ on:pause_recording={() => dispatch("pause_recording")}
366
+ on:stop_recording={() => dispatch("stop_recording")}
367
+ sources={["microphone"]}
368
+ class_name="compact-audio"
369
+ {recording}
370
+ {waveform_settings}
371
+ {waveform_options}
372
+ {i18n}
373
+ {active_source}
374
+ {upload}
375
+ {stream_handler}
376
+ stream_every={1}
377
+ editable={true}
378
+ {label}
379
+ {root}
380
+ loop={false}
381
+ show_label={false}
382
+ show_download_button={false}
383
+ dragging={false}
384
+ />
385
+ {/if}
386
+ <div class="input-container">
387
+ {#if sources && sources.includes("upload") && !disabled && !(file_count === "single" && value.files.length > 0)}
388
+ <Upload
389
+ bind:this={upload_component}
390
+ on:load={handle_upload}
391
+ {file_count}
392
+ filetype={file_types}
393
+ {root}
394
+ {max_file_size}
395
+ bind:dragging
396
+ bind:uploading
397
+ show_progress={false}
398
+ disable_click={true}
399
+ bind:hidden_upload
400
+ on:error
401
+ hidden={true}
402
+ {upload}
403
+ {stream_handler}
404
+ />
405
+ <button
406
+ data-testid="upload-button"
407
+ class="upload-button"
408
+ on:click={handle_upload_click}><Paperclip /></button
303
409
  >
304
- {#each value.files as file, index}
305
- <span role="listitem" aria-label="File thumbnail">
306
- <button class="thumbnail-item thumbnail-small">
307
- <button
308
- class:disabled
309
- class="delete-button"
310
- on:click={(event) => remove_thumbnail(event, index)}
311
- ><Clear /></button
312
- >
313
- {#if file.mime_type && file.mime_type.includes("image")}
314
- <Image
315
- src={file.url}
316
- title={null}
317
- alt=""
318
- loading="lazy"
319
- class={"thumbnail-image"}
320
- />
321
- {:else if file.mime_type && file.mime_type.includes("audio")}
322
- <Music />
323
- {:else if file.mime_type && file.mime_type.includes("video")}
324
- <Video />
325
- {:else}
326
- <File />
327
- {/if}
328
- </button>
329
- </span>
330
- {/each}
331
- {#if uploading}
332
- <div class="loader" role="status" aria-label="Uploading"></div>
333
- {/if}
334
- </div>
335
410
  {/if}
336
- <div class="input-container">
337
- {#if !disabled && !(file_count === "single" && value.files.length > 0)}
338
- <Upload
339
- bind:this={upload_component}
340
- on:load={handle_upload}
341
- {file_count}
342
- filetype={file_types}
343
- {root}
344
- {max_file_size}
345
- bind:dragging
346
- bind:uploading
347
- show_progress={false}
348
- disable_click={true}
349
- bind:hidden_upload
350
- on:error
351
- hidden={true}
352
- {upload}
353
- {stream_handler}
354
- ></Upload>
355
- <button
356
- data-testid="upload-button"
357
- class="upload-button"
358
- on:click={handle_upload_click}><Paperclip /></button
359
- >
360
- {/if}
361
- <textarea
362
- data-testid="textbox"
363
- use:text_area_resize={{
364
- text: value.text,
365
- lines: lines,
366
- max_lines: max_lines
411
+ {#if sources && sources.includes("microphone")}
412
+ <button
413
+ data-testid="microphone-button"
414
+ class="microphone-button"
415
+ class:recording
416
+ on:click={() => {
417
+ active_source = active_source !== "microphone" ? "microphone" : null;
367
418
  }}
368
- class="scroll-hide"
369
- class:no-label={!show_label}
370
- dir={rtl ? "rtl" : "ltr"}
371
- bind:value={value.text}
372
- bind:this={el}
373
- {placeholder}
374
- rows={lines}
375
- {disabled}
376
- {autofocus}
377
- on:keypress={handle_keypress}
378
- on:blur
379
- on:select={handle_select}
380
- on:focus
381
- on:scroll={handle_scroll}
382
- on:paste={handle_paste}
383
- style={text_align ? "text-align: " + text_align : ""}
384
- />
385
- {#if submit_btn}
386
- <button
387
- class="submit-button"
388
- class:padded-button={submit_btn !== true}
389
- on:click={handle_submit}
390
- >
391
- {#if submit_btn === true}
392
- <Send />
393
- {:else}
394
- {submit_btn}
395
- {/if}
396
- </button>
397
- {/if}
398
- {#if stop_btn}
399
- <button
400
- class="stop-button"
401
- class:padded-button={stop_btn !== true}
402
- on:click={handle_stop}
403
- >
404
- {#if stop_btn === true}
405
- <Square fill={"none"} stroke_width={2.5} />
406
- {:else}
407
- {stop_btn}
408
- {/if}
409
- </button>
410
- {/if}
411
- </div>
412
- </label>
419
+ >
420
+ <Microphone />
421
+ </button>
422
+ {/if}
423
+ <!-- svelte-ignore a11y-autofocus -->
424
+ <textarea
425
+ data-testid="textbox"
426
+ use:text_area_resize={{
427
+ text: value.text,
428
+ lines: lines,
429
+ max_lines: max_lines
430
+ }}
431
+ class="scroll-hide"
432
+ class:no-label={!show_label}
433
+ dir={rtl ? "rtl" : "ltr"}
434
+ bind:value={value.text}
435
+ bind:this={el}
436
+ {placeholder}
437
+ rows={lines}
438
+ {disabled}
439
+ {autofocus}
440
+ on:keypress={handle_keypress}
441
+ on:blur
442
+ on:select={handle_select}
443
+ on:focus
444
+ on:scroll={handle_scroll}
445
+ on:paste={handle_paste}
446
+ style={text_align ? "text-align: " + text_align : ""}
447
+ />
448
+ {#if submit_btn}
449
+ <button
450
+ class="submit-button"
451
+ class:padded-button={submit_btn !== true}
452
+ on:click={handle_submit}
453
+ >
454
+ {#if submit_btn === true}
455
+ <Send />
456
+ {:else}
457
+ {submit_btn}
458
+ {/if}
459
+ </button>
460
+ {/if}
461
+ {#if stop_btn}
462
+ <button
463
+ class="stop-button"
464
+ class:padded-button={stop_btn !== true}
465
+ on:click={handle_stop}
466
+ >
467
+ {#if stop_btn === true}
468
+ <Square fill={"none"} stroke_width={2.5} />
469
+ {:else}
470
+ {stop_btn}
471
+ {/if}
472
+ </button>
473
+ {/if}
474
+ </div>
413
475
  </div>
414
476
 
415
477
  <style>
@@ -471,6 +533,7 @@
471
533
  color: var(--input-placeholder-color);
472
534
  }
473
535
 
536
+ .microphone-button,
474
537
  .upload-button,
475
538
  .submit-button,
476
539
  .stop-button {
@@ -487,29 +550,34 @@
487
550
  justify-content: center;
488
551
  align-items: center;
489
552
  z-index: var(--layer-1);
553
+ margin-left: var(--spacing-sm);
490
554
  }
491
555
  .padded-button {
492
556
  padding: 0 10px;
493
557
  }
494
558
 
559
+ .microphone-button,
495
560
  .stop-button,
496
561
  .upload-button,
497
562
  .submit-button {
498
563
  background: var(--button-secondary-background-fill);
499
564
  }
500
565
 
566
+ .microphone-button:hover,
501
567
  .stop-button:hover,
502
568
  .upload-button:hover,
503
569
  .submit-button:hover {
504
570
  background: var(--button-secondary-background-fill-hover);
505
571
  }
506
572
 
573
+ .microphone-button:disabled,
507
574
  .stop-button:disabled,
508
575
  .upload-button:disabled,
509
576
  .submit-button:disabled {
510
577
  background: var(--button-secondary-background-fill);
511
578
  cursor: initial;
512
579
  }
580
+ .microphone-button:active,
513
581
  .stop-button:active,
514
582
  .upload-button:active,
515
583
  .submit-button:active {
@@ -520,6 +588,7 @@
520
588
  height: 22px;
521
589
  width: 22px;
522
590
  }
591
+ .microphone-button :global(svg),
523
592
  .upload-button :global(svg) {
524
593
  height: 17px;
525
594
  width: 17px;