convex-rust 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +137 -2
- package/dist/cli/bin.d.ts +2 -0
- package/dist/cli/bin.js +200 -0
- package/dist/cli/build.d.ts +146 -0
- package/dist/cli/build.js +902 -0
- package/dist/cli/demangle.d.ts +7 -0
- package/dist/cli/demangle.js +328 -0
- package/dist/cli/diagnostics.d.ts +34 -0
- package/dist/cli/diagnostics.js +296 -0
- package/dist/cli/generate.d.ts +29 -0
- package/dist/cli/generate.js +82 -0
- package/dist/cli/manifest.d.ts +24 -0
- package/dist/cli/manifest.js +75 -0
- package/dist/cli/rustApi.d.ts +78 -0
- package/dist/cli/rustApi.js +609 -0
- package/dist/cli/scaffold.d.ts +22 -0
- package/dist/cli/scaffold.js +114 -0
- package/dist/cli/shim.d.ts +30 -0
- package/dist/cli/shim.js +140 -0
- package/dist/cli/size.d.ts +17 -0
- package/dist/cli/size.js +73 -0
- package/dist/cli/split.d.ts +20 -0
- package/dist/cli/split.js +140 -0
- package/dist/cli/symbols.d.ts +30 -0
- package/dist/cli/symbols.js +133 -0
- package/dist/cli/toolchain.d.ts +53 -0
- package/dist/cli/toolchain.js +167 -0
- package/dist/cli/validators.d.ts +6 -0
- package/dist/cli/validators.js +66 -0
- package/dist/cli/wasisdk.d.ts +82 -0
- package/dist/cli/wasisdk.js +220 -0
- package/dist/cli/wasmHints.d.ts +29 -0
- package/dist/cli/wasmHints.js +172 -0
- package/dist/runtime/base64.d.ts +4 -0
- package/dist/runtime/base64.js +77 -0
- package/dist/runtime/crypto.d.ts +7 -0
- package/dist/runtime/crypto.js +421 -0
- package/dist/runtime/fastjson.d.ts +3 -0
- package/dist/runtime/fastjson.js +170 -0
- package/dist/runtime/host.d.ts +73 -0
- package/dist/runtime/host.js +1230 -0
- package/dist/runtime/http.d.ts +40 -0
- package/dist/runtime/http.js +157 -0
- package/dist/runtime/index.d.ts +2 -0
- package/dist/runtime/index.js +1 -0
- package/dist/runtime/intl.d.ts +5 -0
- package/dist/runtime/intl.js +69 -0
- package/dist/runtime/tmpfs.d.ts +129 -0
- package/dist/runtime/tmpfs.js +624 -0
- package/dist/runtime/wasi.d.ts +33 -0
- package/dist/runtime/wasi.js +216 -0
- package/package.json +57 -4
- package/rust/convex_fn/Cargo.toml +32 -0
- package/rust/convex_fn/LICENSE +201 -0
- package/rust/convex_fn/build.rs +3 -0
- package/rust/convex_fn/src/args.rs +416 -0
- package/rust/convex_fn/src/crypto.rs +237 -0
- package/rust/convex_fn/src/ctx.rs +1044 -0
- package/rust/convex_fn/src/dns.rs +356 -0
- package/rust/convex_fn/src/error.rs +179 -0
- package/rust/convex_fn/src/executor.rs +577 -0
- package/rust/convex_fn/src/fanout.rs +106 -0
- package/rust/convex_fn/src/fs.rs +19 -0
- package/rust/convex_fn/src/function_ref.rs +219 -0
- package/rust/convex_fn/src/http.rs +441 -0
- package/rust/convex_fn/src/intl.rs +75 -0
- package/rust/convex_fn/src/lib.rs +242 -0
- package/rust/convex_fn/src/logging.rs +222 -0
- package/rust/convex_fn/src/native.rs +360 -0
- package/rust/convex_fn/src/net.rs +122 -0
- package/rust/convex_fn/src/sys.rs +178 -0
- package/rust/convex_fn/src/values.rs +121 -0
- package/rust/convex_fn/src/wire.rs +579 -0
- package/rust/convex_fn_macros/Cargo.toml +13 -0
- package/rust/convex_fn_macros/LICENSE +201 -0
- package/rust/convex_fn_macros/src/derive.rs +632 -0
- package/rust/convex_fn_macros/src/lib.rs +338 -0
- package/rust/convex_reqwest/Cargo.toml +69 -0
- package/rust/convex_reqwest/LICENSE +201 -0
- package/rust/convex_reqwest/src/blocking.rs +287 -0
- package/rust/convex_reqwest/src/body.rs +152 -0
- package/rust/convex_reqwest/src/client.rs +359 -0
- package/rust/convex_reqwest/src/config_types.rs +150 -0
- package/rust/convex_reqwest/src/error.rs +241 -0
- package/rust/convex_reqwest/src/into_url.rs +57 -0
- package/rust/convex_reqwest/src/lib.rs +42 -0
- package/rust/convex_reqwest/src/multipart.rs +140 -0
- package/rust/convex_reqwest/src/request.rs +358 -0
- package/rust/convex_reqwest/src/response.rs +310 -0
|
@@ -0,0 +1,577 @@
|
|
|
1
|
+
//! Single-threaded executor driven by `task_poll` (contracts/ABI.md).
|
|
2
|
+
//!
|
|
3
|
+
//! Every pending future is re-polled after each host event, which is simple
|
|
4
|
+
//! and correct for the one-call-per-instance model. DX-06a replaces this with
|
|
5
|
+
//! a tokio current_thread runtime.
|
|
6
|
+
|
|
7
|
+
use std::{
|
|
8
|
+
cell::RefCell,
|
|
9
|
+
collections::{HashMap, HashSet, VecDeque},
|
|
10
|
+
future::Future,
|
|
11
|
+
pin::Pin,
|
|
12
|
+
task::{Context, Poll, Waker},
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
use serde_json::Value;
|
|
16
|
+
|
|
17
|
+
use crate::{error::Error, net, sys, wire::Incoming, Result};
|
|
18
|
+
|
|
19
|
+
#[derive(Default)]
|
|
20
|
+
struct Stream {
|
|
21
|
+
chunks: VecDeque<Vec<u8>>,
|
|
22
|
+
done: Option<std::result::Result<(), String>>,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
#[derive(Default)]
|
|
26
|
+
struct State {
|
|
27
|
+
results: HashMap<i64, Result<Value>>,
|
|
28
|
+
streams: HashMap<String, Stream>,
|
|
29
|
+
/// Task ids whose future was dropped before completion; late results are discarded.
|
|
30
|
+
abandoned_tasks: HashSet<i64>,
|
|
31
|
+
/// Stream ids whose reader was dropped before the stream ended; late chunks are discarded.
|
|
32
|
+
abandoned_streams: HashSet<String>,
|
|
33
|
+
/// `tokio-rt`: wakers of futures awaiting a task result / stream event.
|
|
34
|
+
#[cfg(feature = "tokio-rt")]
|
|
35
|
+
wakers: HashMap<WakeKey, Waker>,
|
|
36
|
+
/// `tokio-rt`: the host pump task, woken when a waiter registers.
|
|
37
|
+
#[cfg(feature = "tokio-rt")]
|
|
38
|
+
pump: Option<Waker>,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
#[cfg(feature = "tokio-rt")]
|
|
42
|
+
#[derive(PartialEq, Eq, Hash)]
|
|
43
|
+
enum WakeKey {
|
|
44
|
+
Task(i64),
|
|
45
|
+
Stream(String),
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
#[cfg(feature = "tokio-rt")]
|
|
49
|
+
fn register(s: &mut State, key: WakeKey, cx: &Context<'_>) {
|
|
50
|
+
s.wakers.insert(key, cx.waker().clone());
|
|
51
|
+
if let Some(p) = s.pump.take() {
|
|
52
|
+
p.wake();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
thread_local! {
|
|
57
|
+
static STATE: RefCell<State> = RefCell::new(State::default());
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
|
|
61
|
+
pub(crate) fn reset() {
|
|
62
|
+
reset_state();
|
|
63
|
+
net::reset();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/// Forgets every task result, stream buffer and abandoned id. The wasm entry
|
|
67
|
+
/// point calls this at the start of each call: the host restarts task and
|
|
68
|
+
/// stream ids at 0 per call, so on a reused instance anything left over from
|
|
69
|
+
/// the previous call (a dropped stream, a late result) must not be matched by
|
|
70
|
+
/// a new call's ids. The `net/*` capabilities stay probed (they are per instance).
|
|
71
|
+
#[cfg_attr(not(target_arch = "wasm32"), allow(dead_code))]
|
|
72
|
+
pub(crate) fn reset_state() {
|
|
73
|
+
// Take the old state out first so its wakers drop outside the borrow.
|
|
74
|
+
let old = STATE.with(|s| std::mem::take(&mut *s.borrow_mut()));
|
|
75
|
+
drop(old);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/// Submits a task request and returns a future for its result.
|
|
79
|
+
pub(crate) fn submit(request: &Value) -> TaskFuture {
|
|
80
|
+
let id = sys::submit(request.to_string().as_bytes());
|
|
81
|
+
TaskFuture {
|
|
82
|
+
id,
|
|
83
|
+
finished: false,
|
|
84
|
+
cancel: false,
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/// Like [`submit`], but dropping the future before it completes also cancels
|
|
89
|
+
/// the work on the host (`net/cancel`) when the host supports it.
|
|
90
|
+
pub(crate) fn submit_cancellable(request: &Value) -> TaskFuture {
|
|
91
|
+
let mut t = submit(request);
|
|
92
|
+
t.cancel = net::caps().cancel;
|
|
93
|
+
t
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/// A submitted task. Dropping it before it completes abandons the task: the
|
|
97
|
+
/// host result, when it arrives, is discarded instead of piling up.
|
|
98
|
+
pub(crate) struct TaskFuture {
|
|
99
|
+
id: i64,
|
|
100
|
+
finished: bool,
|
|
101
|
+
/// Cancel the host task on drop.
|
|
102
|
+
cancel: bool,
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
impl TaskFuture {
|
|
106
|
+
pub(crate) fn id(&self) -> i64 {
|
|
107
|
+
self.id
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
impl Drop for TaskFuture {
|
|
112
|
+
fn drop(&mut self) {
|
|
113
|
+
if self.finished || self.id < 0 {
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
let _ = STATE.try_with(|s| {
|
|
117
|
+
let mut s = s.borrow_mut();
|
|
118
|
+
#[cfg(feature = "tokio-rt")]
|
|
119
|
+
s.wakers.remove(&WakeKey::Task(self.id));
|
|
120
|
+
if s.results.remove(&self.id).is_none() {
|
|
121
|
+
s.abandoned_tasks.insert(self.id);
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
if self.cancel {
|
|
125
|
+
net::cancel_task(self.id);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
impl Future for TaskFuture {
|
|
131
|
+
type Output = Result<Value>;
|
|
132
|
+
|
|
133
|
+
fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
|
|
134
|
+
if self.id < 0 {
|
|
135
|
+
self.finished = true;
|
|
136
|
+
return Poll::Ready(Err(Error::msg("host rejected the task request")));
|
|
137
|
+
}
|
|
138
|
+
let id = self.id;
|
|
139
|
+
match STATE.with(|s| {
|
|
140
|
+
let mut s = s.borrow_mut();
|
|
141
|
+
let r = s.results.remove(&id);
|
|
142
|
+
#[cfg(feature = "tokio-rt")]
|
|
143
|
+
if r.is_none() {
|
|
144
|
+
register(&mut s, WakeKey::Task(id), _cx);
|
|
145
|
+
}
|
|
146
|
+
r
|
|
147
|
+
}) {
|
|
148
|
+
Some(r) => {
|
|
149
|
+
self.finished = true;
|
|
150
|
+
Poll::Ready(r)
|
|
151
|
+
}
|
|
152
|
+
None => Poll::Pending,
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/// Resolves to the full body of a host stream once its `end` chunk arrives.
|
|
158
|
+
/// Dropping it before it completes abandons the stream (late chunks are discarded).
|
|
159
|
+
pub(crate) struct StreamFuture {
|
|
160
|
+
stream_id: String,
|
|
161
|
+
finished: bool,
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
impl StreamFuture {
|
|
165
|
+
pub(crate) fn new(stream_id: String) -> Self {
|
|
166
|
+
Self {
|
|
167
|
+
stream_id,
|
|
168
|
+
finished: false,
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
impl Drop for StreamFuture {
|
|
174
|
+
fn drop(&mut self) {
|
|
175
|
+
if !self.finished {
|
|
176
|
+
abandon_stream(&self.stream_id);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
impl Future for StreamFuture {
|
|
182
|
+
type Output = Result<Vec<u8>>;
|
|
183
|
+
|
|
184
|
+
fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
|
|
185
|
+
let r = STATE.with(|s| {
|
|
186
|
+
let mut s = s.borrow_mut();
|
|
187
|
+
let finished = s
|
|
188
|
+
.streams
|
|
189
|
+
.get(&self.stream_id)
|
|
190
|
+
.is_some_and(|st| st.done.is_some());
|
|
191
|
+
if !finished {
|
|
192
|
+
#[cfg(feature = "tokio-rt")]
|
|
193
|
+
register(&mut s, WakeKey::Stream(self.stream_id.clone()), _cx);
|
|
194
|
+
return Poll::Pending;
|
|
195
|
+
}
|
|
196
|
+
let st = s.streams.remove(&self.stream_id).unwrap_or_default();
|
|
197
|
+
Poll::Ready(match st.done {
|
|
198
|
+
Some(Err(e)) => Err(Error::msg(e)),
|
|
199
|
+
_ => {
|
|
200
|
+
let mut chunks = st.chunks;
|
|
201
|
+
if chunks.len() == 1 {
|
|
202
|
+
Ok(chunks.pop_front().unwrap_or_default())
|
|
203
|
+
} else {
|
|
204
|
+
let total: usize = chunks.iter().map(Vec::len).sum();
|
|
205
|
+
let mut out = Vec::with_capacity(total);
|
|
206
|
+
for c in &chunks {
|
|
207
|
+
out.extend_from_slice(c);
|
|
208
|
+
}
|
|
209
|
+
Ok(out)
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
})
|
|
213
|
+
});
|
|
214
|
+
if r.is_ready() {
|
|
215
|
+
self.finished = true;
|
|
216
|
+
}
|
|
217
|
+
r
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/// Resolves to the next chunk of a host stream: `Some(bytes)`, `None` at the
|
|
222
|
+
/// end, or the stream error (after all chunks that preceded it).
|
|
223
|
+
pub(crate) struct NextChunk<'a> {
|
|
224
|
+
pub(crate) stream_id: &'a str,
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
impl Future for NextChunk<'_> {
|
|
228
|
+
type Output = Result<Option<Vec<u8>>>;
|
|
229
|
+
|
|
230
|
+
fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
|
|
231
|
+
STATE.with(|s| {
|
|
232
|
+
let mut s = s.borrow_mut();
|
|
233
|
+
let Some(st) = s.streams.get_mut(self.stream_id) else {
|
|
234
|
+
#[cfg(feature = "tokio-rt")]
|
|
235
|
+
register(&mut s, WakeKey::Stream(self.stream_id.to_string()), _cx);
|
|
236
|
+
return Poll::Pending;
|
|
237
|
+
};
|
|
238
|
+
if let Some(chunk) = st.chunks.pop_front() {
|
|
239
|
+
return Poll::Ready(Ok(Some(chunk)));
|
|
240
|
+
}
|
|
241
|
+
match st.done.take() {
|
|
242
|
+
None => {
|
|
243
|
+
#[cfg(feature = "tokio-rt")]
|
|
244
|
+
register(&mut s, WakeKey::Stream(self.stream_id.to_string()), _cx);
|
|
245
|
+
Poll::Pending
|
|
246
|
+
}
|
|
247
|
+
Some(r) => {
|
|
248
|
+
s.streams.remove(self.stream_id);
|
|
249
|
+
Poll::Ready(match r {
|
|
250
|
+
Ok(()) => Ok(None),
|
|
251
|
+
Err(e) => Err(Error::msg(e)),
|
|
252
|
+
})
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
})
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/// Stops tracking a stream that will not be read any further; chunks the host
|
|
260
|
+
/// still sends for it are discarded.
|
|
261
|
+
pub(crate) fn abandon_stream(stream_id: &str) {
|
|
262
|
+
let ended = STATE
|
|
263
|
+
.try_with(|s| {
|
|
264
|
+
let mut s = s.borrow_mut();
|
|
265
|
+
let ended = s
|
|
266
|
+
.streams
|
|
267
|
+
.remove(stream_id)
|
|
268
|
+
.is_some_and(|st| st.done.is_some());
|
|
269
|
+
if !ended {
|
|
270
|
+
s.abandoned_streams.insert(stream_id.to_string());
|
|
271
|
+
}
|
|
272
|
+
ended
|
|
273
|
+
})
|
|
274
|
+
.unwrap_or(true);
|
|
275
|
+
if !ended {
|
|
276
|
+
// Stop the download on the host too (best effort).
|
|
277
|
+
net::cancel_stream(stream_id);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
fn dispatch(event: Vec<u8>) {
|
|
282
|
+
let Some(ev) = Incoming::parse_owned(event) else {
|
|
283
|
+
return;
|
|
284
|
+
};
|
|
285
|
+
STATE.with(|s| {
|
|
286
|
+
let mut s = s.borrow_mut();
|
|
287
|
+
match ev {
|
|
288
|
+
Incoming::Done {
|
|
289
|
+
task_id: id,
|
|
290
|
+
result,
|
|
291
|
+
} => {
|
|
292
|
+
if s.abandoned_tasks.remove(&id) {
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
let result = result.map_err(|err| Error::from_json(&err));
|
|
296
|
+
s.results.insert(id, result);
|
|
297
|
+
#[cfg(feature = "tokio-rt")]
|
|
298
|
+
if let Some(w) = s.wakers.remove(&WakeKey::Task(id)) {
|
|
299
|
+
w.wake();
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
Incoming::Chunk {
|
|
303
|
+
stream_id: id,
|
|
304
|
+
data,
|
|
305
|
+
error,
|
|
306
|
+
terminal,
|
|
307
|
+
} => {
|
|
308
|
+
if s.abandoned_streams.contains(&id) {
|
|
309
|
+
if terminal {
|
|
310
|
+
s.abandoned_streams.remove(&id);
|
|
311
|
+
}
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
#[cfg(feature = "tokio-rt")]
|
|
315
|
+
if let Some(w) = s.wakers.remove(&WakeKey::Stream(id.clone())) {
|
|
316
|
+
w.wake();
|
|
317
|
+
}
|
|
318
|
+
let st = s.streams.entry(id).or_default();
|
|
319
|
+
if let Some(data) = data {
|
|
320
|
+
st.chunks.push_back(data);
|
|
321
|
+
}
|
|
322
|
+
if let Some(err) = error {
|
|
323
|
+
st.done = Some(Err(err));
|
|
324
|
+
} else if terminal {
|
|
325
|
+
st.done = Some(Ok(()));
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/// Drives `fut` to completion, blocking on the host for task events.
|
|
333
|
+
#[cfg(not(feature = "tokio-rt"))]
|
|
334
|
+
pub fn block_on<F: Future>(fut: F) -> F::Output {
|
|
335
|
+
block_on_direct(fut)
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/// Polls `fut` with a no-op waker and blocks in `task_poll(1)` between polls,
|
|
339
|
+
/// dispatching every host event to its result/stream buffer (and waking any
|
|
340
|
+
/// registered waker). No runtime is involved.
|
|
341
|
+
fn block_on_direct<F: Future>(fut: F) -> F::Output {
|
|
342
|
+
let mut fut = std::pin::pin!(fut);
|
|
343
|
+
let mut cx = Context::from_waker(Waker::noop());
|
|
344
|
+
loop {
|
|
345
|
+
if let Poll::Ready(v) = fut.as_mut().poll(&mut cx) {
|
|
346
|
+
return v;
|
|
347
|
+
}
|
|
348
|
+
match sys::poll(true) {
|
|
349
|
+
Some(event) => {
|
|
350
|
+
dispatch(event);
|
|
351
|
+
// Everything already delivered, before re-polling once.
|
|
352
|
+
while let Some(event) = sys::poll(false) {
|
|
353
|
+
dispatch(event);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
None => panic!("action is waiting on a future that no host task will complete"),
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/// Runs futures concurrently and collects their outputs in order
|
|
362
|
+
/// (the equivalent of `Promise.all`).
|
|
363
|
+
pub async fn join_all<F: Future>(futs: impl IntoIterator<Item = F>) -> Vec<F::Output> {
|
|
364
|
+
let mut futs: Vec<Pin<Box<F>>> = futs.into_iter().map(Box::pin).collect();
|
|
365
|
+
let mut out: Vec<Option<F::Output>> = futs.iter().map(|_| None).collect();
|
|
366
|
+
std::future::poll_fn(move |cx| {
|
|
367
|
+
let mut pending = false;
|
|
368
|
+
for (f, slot) in futs.iter_mut().zip(out.iter_mut()) {
|
|
369
|
+
if slot.is_none() {
|
|
370
|
+
match f.as_mut().poll(cx) {
|
|
371
|
+
Poll::Ready(v) => *slot = Some(v),
|
|
372
|
+
Poll::Pending => pending = true,
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
if pending {
|
|
377
|
+
Poll::Pending
|
|
378
|
+
} else {
|
|
379
|
+
Poll::Ready(
|
|
380
|
+
out.iter_mut()
|
|
381
|
+
.map(|s| s.take().expect("polled to completion"))
|
|
382
|
+
.collect(),
|
|
383
|
+
)
|
|
384
|
+
}
|
|
385
|
+
})
|
|
386
|
+
.await
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/// The result of [`select`]: which of the two futures finished first.
|
|
390
|
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
391
|
+
pub enum Either<A, B> {
|
|
392
|
+
/// The first future finished first.
|
|
393
|
+
Left(A),
|
|
394
|
+
/// The second future finished first.
|
|
395
|
+
Right(B),
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/// Races two futures (like `Promise.race`). Resolves with the output of the
|
|
399
|
+
/// first one to finish; if both are ready on the same poll, `a` wins. The
|
|
400
|
+
/// loser is dropped; any host task it had in flight is abandoned and its late
|
|
401
|
+
/// result is discarded, so it cannot wedge or confuse later awaits.
|
|
402
|
+
pub async fn select<A: Future, B: Future>(a: A, b: B) -> Either<A::Output, B::Output> {
|
|
403
|
+
let mut a = Box::pin(a);
|
|
404
|
+
let mut b = Box::pin(b);
|
|
405
|
+
std::future::poll_fn(move |cx| {
|
|
406
|
+
if let Poll::Ready(v) = a.as_mut().poll(cx) {
|
|
407
|
+
return Poll::Ready(Either::Left(v));
|
|
408
|
+
}
|
|
409
|
+
if let Poll::Ready(v) = b.as_mut().poll(cx) {
|
|
410
|
+
return Poll::Ready(Either::Right(v));
|
|
411
|
+
}
|
|
412
|
+
Poll::Pending
|
|
413
|
+
})
|
|
414
|
+
.await
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/// Races any number of futures. Resolves with `(index, output)` of the first
|
|
418
|
+
/// one to finish (lowest index wins ties); all the others are dropped as in
|
|
419
|
+
/// [`select`].
|
|
420
|
+
///
|
|
421
|
+
/// # Panics
|
|
422
|
+
///
|
|
423
|
+
/// Panics if `futs` is empty, since no future could ever finish.
|
|
424
|
+
pub async fn race<F: Future>(futs: impl IntoIterator<Item = F>) -> (usize, F::Output) {
|
|
425
|
+
let mut futs: Vec<Pin<Box<F>>> = futs.into_iter().map(Box::pin).collect();
|
|
426
|
+
assert!(!futs.is_empty(), "race() needs at least one future");
|
|
427
|
+
std::future::poll_fn(move |cx| {
|
|
428
|
+
for (i, f) in futs.iter_mut().enumerate() {
|
|
429
|
+
if let Poll::Ready(v) = f.as_mut().poll(cx) {
|
|
430
|
+
return Poll::Ready((i, v));
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
Poll::Pending
|
|
434
|
+
})
|
|
435
|
+
.await
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/// `tokio-rt`: drives `fut` on a tokio `current_thread` runtime (+ `LocalSet`),
|
|
439
|
+
/// so `tokio::spawn`, `spawn_local`, `tokio::time::*`, `select!`, `join!` and
|
|
440
|
+
/// tokio channels work inside actions.
|
|
441
|
+
///
|
|
442
|
+
/// Host events are fed in by a "pump" future running inside the runtime: it
|
|
443
|
+
/// drains `task_poll(0)` and, while something awaits a host result, re-checks
|
|
444
|
+
/// every `tick` (1 ms backing off to 8 ms) because tokio's own idle park
|
|
445
|
+
/// (`poll_oneoff` on the WASI clock) cannot be interrupted by a host event.
|
|
446
|
+
///
|
|
447
|
+
/// # Nested (blocking) calls
|
|
448
|
+
///
|
|
449
|
+
/// Called from inside a running action (e.g. `reqwest::blocking` in an
|
|
450
|
+
/// `#[action]`), `block_on` cannot start a second tokio runtime, so it drives
|
|
451
|
+
/// `fut` directly: submit + `task_poll(block = 1)` in a loop, dispatching all
|
|
452
|
+
/// events to their buffers/wakers. The guest is single-threaded, so while a
|
|
453
|
+
/// blocking call runs, tasks spawned on the `LocalSet` (and timers) do NOT
|
|
454
|
+
/// progress; they resume when the call returns. `fut` must therefore only
|
|
455
|
+
/// depend on host tasks (fetch, run_*, sleep...), never on a spawned tokio task
|
|
456
|
+
/// or a tokio timer/channel; such a future would wait forever and the call
|
|
457
|
+
/// panics ("no host task will complete") once no host task is outstanding.
|
|
458
|
+
#[cfg(feature = "tokio-rt")]
|
|
459
|
+
pub fn block_on<F: Future>(fut: F) -> F::Output {
|
|
460
|
+
use std::time::Duration;
|
|
461
|
+
|
|
462
|
+
thread_local! {
|
|
463
|
+
static IN_RT: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
|
|
464
|
+
}
|
|
465
|
+
if IN_RT.with(|f| f.get()) {
|
|
466
|
+
return block_on_direct(fut);
|
|
467
|
+
}
|
|
468
|
+
struct Reset;
|
|
469
|
+
impl Drop for Reset {
|
|
470
|
+
fn drop(&mut self) {
|
|
471
|
+
IN_RT.with(|f| f.set(false));
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
IN_RT.with(|f| f.set(true));
|
|
475
|
+
let _reset = Reset;
|
|
476
|
+
|
|
477
|
+
const MIN_TICK: Duration = Duration::from_millis(1);
|
|
478
|
+
const MAX_TICK: Duration = Duration::from_millis(8);
|
|
479
|
+
|
|
480
|
+
/// Blocks in `task_poll(1)` until a host event arrives or a host `sleep(tick)` fires,
|
|
481
|
+
/// dispatching what arrived. `Some(true)` if an event came before the sleep, `Some(false)`
|
|
482
|
+
/// if the sleep ended the wait, `None` if the host rejected the sleep.
|
|
483
|
+
fn block_on_host(tick: Duration) -> Option<bool> {
|
|
484
|
+
let mut sleep =
|
|
485
|
+
submit_cancellable(&crate::wire::sleep_request(tick.as_millis().max(1) as u64));
|
|
486
|
+
if sleep.id() < 0 {
|
|
487
|
+
return None;
|
|
488
|
+
}
|
|
489
|
+
if let Some(event) = sys::poll(true) {
|
|
490
|
+
dispatch(event);
|
|
491
|
+
}
|
|
492
|
+
while let Some(event) = sys::poll(false) {
|
|
493
|
+
dispatch(event);
|
|
494
|
+
}
|
|
495
|
+
// Done: consumes the result. Still pending: dropping `sleep` cancels it on the host.
|
|
496
|
+
let mut cx = Context::from_waker(Waker::noop());
|
|
497
|
+
Some(Pin::new(&mut sleep).poll(&mut cx).is_pending())
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
async fn pump() {
|
|
501
|
+
let mut tick = MIN_TICK;
|
|
502
|
+
let mut yielded = false;
|
|
503
|
+
loop {
|
|
504
|
+
let mut got = false;
|
|
505
|
+
while let Some(event) = sys::poll(false) {
|
|
506
|
+
dispatch(event);
|
|
507
|
+
got = true;
|
|
508
|
+
}
|
|
509
|
+
if got {
|
|
510
|
+
tick = MIN_TICK;
|
|
511
|
+
yielded = false;
|
|
512
|
+
// Let woken tasks run before we look again.
|
|
513
|
+
tokio::task::yield_now().await;
|
|
514
|
+
continue;
|
|
515
|
+
}
|
|
516
|
+
let waiting = STATE.with(|s| !s.borrow().wakers.is_empty());
|
|
517
|
+
if waiting {
|
|
518
|
+
if net::caps().cancel {
|
|
519
|
+
// Let ready tasks run once, then block on the host: it wakes us the moment a
|
|
520
|
+
// result arrives instead of up to `tick` later. The bounded host sleep keeps
|
|
521
|
+
// tokio timers / spawned tasks served at `tick` granularity (tokio's own park
|
|
522
|
+
// cannot be interrupted by a host event, so it is not used while waiting).
|
|
523
|
+
if !yielded {
|
|
524
|
+
yielded = true;
|
|
525
|
+
tokio::task::yield_now().await;
|
|
526
|
+
continue;
|
|
527
|
+
}
|
|
528
|
+
yielded = false;
|
|
529
|
+
if let Some(early) = block_on_host(tick) {
|
|
530
|
+
tick = if early {
|
|
531
|
+
MIN_TICK
|
|
532
|
+
} else {
|
|
533
|
+
(tick * 2).min(MAX_TICK)
|
|
534
|
+
};
|
|
535
|
+
tokio::task::yield_now().await;
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
tokio::time::sleep(tick).await;
|
|
540
|
+
tick = (tick * 2).min(MAX_TICK);
|
|
541
|
+
} else {
|
|
542
|
+
// Nothing awaits the host: sleep until a future registers.
|
|
543
|
+
tick = MIN_TICK;
|
|
544
|
+
std::future::poll_fn(|cx| {
|
|
545
|
+
STATE.with(|s| {
|
|
546
|
+
let mut s = s.borrow_mut();
|
|
547
|
+
if s.wakers.is_empty() {
|
|
548
|
+
s.pump = Some(cx.waker().clone());
|
|
549
|
+
Poll::Pending
|
|
550
|
+
} else {
|
|
551
|
+
Poll::Ready(())
|
|
552
|
+
}
|
|
553
|
+
})
|
|
554
|
+
})
|
|
555
|
+
.await;
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
let rt = tokio::runtime::Builder::new_current_thread()
|
|
561
|
+
.enable_time()
|
|
562
|
+
.build()
|
|
563
|
+
.expect("tokio runtime");
|
|
564
|
+
let local = tokio::task::LocalSet::new();
|
|
565
|
+
local.block_on(&rt, async move {
|
|
566
|
+
let mut fut = std::pin::pin!(fut);
|
|
567
|
+
let mut pump = std::pin::pin!(pump());
|
|
568
|
+
std::future::poll_fn(|cx| {
|
|
569
|
+
if let Poll::Ready(v) = fut.as_mut().poll(cx) {
|
|
570
|
+
return Poll::Ready(v);
|
|
571
|
+
}
|
|
572
|
+
let _ = pump.as_mut().poll(cx);
|
|
573
|
+
Poll::Pending
|
|
574
|
+
})
|
|
575
|
+
.await
|
|
576
|
+
})
|
|
577
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
//! Fan-out: run many Convex actions (typically internal Rust actions) in parallel with a concurrency limit.
|
|
2
|
+
//!
|
|
3
|
+
//! ```ignore
|
|
4
|
+
//! // 200 chunks, at most 16 actions running at once; results come back in input order.
|
|
5
|
+
//! let sums: Vec<f64> = ctx.run_action_each(api::work::sum_chunk, chunks, 16).await?;
|
|
6
|
+
//! ```
|
|
7
|
+
//!
|
|
8
|
+
//! Each element is a separate action execution (`ctx.runAction`), so it runs in its own isolate and gets
|
|
9
|
+
//! its own 64 MiB / time budget; the calling action waits for all of them and its wall time counts. Send
|
|
10
|
+
//! small inputs as arguments (16 MiB per call at most); put large ones in storage
|
|
11
|
+
//! ([`Ctx::storage`]) and pass the storage id. See docs/rust/cloud.md "Fan-out".
|
|
12
|
+
|
|
13
|
+
use std::future::Future;
|
|
14
|
+
use std::pin::Pin;
|
|
15
|
+
use std::task::Poll;
|
|
16
|
+
|
|
17
|
+
use serde::de::DeserializeOwned;
|
|
18
|
+
use serde::Serialize;
|
|
19
|
+
|
|
20
|
+
use crate::{Ctx, FunctionTarget, Result};
|
|
21
|
+
|
|
22
|
+
/// Maps `items` through `f`, keeping at most `limit` (at least 1) futures in flight, and returns the outputs
|
|
23
|
+
/// in input order. Futures start as earlier ones finish, in input order. Single-threaded (like [`crate::join_all`]).
|
|
24
|
+
pub async fn par_map<T, R, Fut>(
|
|
25
|
+
items: impl IntoIterator<Item = T>,
|
|
26
|
+
limit: usize,
|
|
27
|
+
mut f: impl FnMut(T) -> Fut,
|
|
28
|
+
) -> Vec<R>
|
|
29
|
+
where
|
|
30
|
+
Fut: Future<Output = R>,
|
|
31
|
+
{
|
|
32
|
+
let mut iter = items.into_iter();
|
|
33
|
+
let limit = limit.max(1);
|
|
34
|
+
let mut out: Vec<Option<R>> = Vec::new();
|
|
35
|
+
let mut flight: Vec<(usize, Pin<Box<Fut>>)> = Vec::new();
|
|
36
|
+
let mut exhausted = false;
|
|
37
|
+
std::future::poll_fn(|cx| loop {
|
|
38
|
+
while !exhausted && flight.len() < limit {
|
|
39
|
+
match iter.next() {
|
|
40
|
+
Some(t) => {
|
|
41
|
+
out.push(None);
|
|
42
|
+
flight.push((out.len() - 1, Box::pin(f(t))));
|
|
43
|
+
}
|
|
44
|
+
None => exhausted = true,
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
let mut finished = false;
|
|
48
|
+
let mut i = 0;
|
|
49
|
+
while i < flight.len() {
|
|
50
|
+
match flight[i].1.as_mut().poll(cx) {
|
|
51
|
+
Poll::Ready(v) => {
|
|
52
|
+
let (idx, _) = flight.swap_remove(i);
|
|
53
|
+
out[idx] = Some(v);
|
|
54
|
+
finished = true;
|
|
55
|
+
}
|
|
56
|
+
Poll::Pending => i += 1,
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if flight.is_empty() && exhausted {
|
|
60
|
+
return Poll::Ready(());
|
|
61
|
+
}
|
|
62
|
+
if !finished {
|
|
63
|
+
return Poll::Pending;
|
|
64
|
+
}
|
|
65
|
+
})
|
|
66
|
+
.await;
|
|
67
|
+
out.into_iter()
|
|
68
|
+
.map(|o| o.expect("polled to completion"))
|
|
69
|
+
.collect()
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
impl Ctx {
|
|
73
|
+
/// Runs `target` once per element of `args`, at most `limit` at a time, and returns every outcome in input
|
|
74
|
+
/// order (one failure does not stop the others).
|
|
75
|
+
pub async fn run_action_each_settled<A, R>(
|
|
76
|
+
&self,
|
|
77
|
+
target: impl FunctionTarget<A, R> + Copy,
|
|
78
|
+
args: impl IntoIterator<Item = A>,
|
|
79
|
+
limit: usize,
|
|
80
|
+
) -> Vec<Result<R>>
|
|
81
|
+
where
|
|
82
|
+
A: Serialize,
|
|
83
|
+
R: DeserializeOwned,
|
|
84
|
+
{
|
|
85
|
+
par_map(args, limit, |a| self.run_action(target, a)).await
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/// Like [`Ctx::run_action_each_settled`], but fails with the first error (in input order) after all
|
|
89
|
+
/// started calls have finished. Calls not yet started when an error occurs are still started: use
|
|
90
|
+
/// the `settled` form and stop yourself if that matters.
|
|
91
|
+
pub async fn run_action_each<A, R>(
|
|
92
|
+
&self,
|
|
93
|
+
target: impl FunctionTarget<A, R> + Copy,
|
|
94
|
+
args: impl IntoIterator<Item = A>,
|
|
95
|
+
limit: usize,
|
|
96
|
+
) -> Result<Vec<R>>
|
|
97
|
+
where
|
|
98
|
+
A: Serialize,
|
|
99
|
+
R: DeserializeOwned,
|
|
100
|
+
{
|
|
101
|
+
self.run_action_each_settled(target, args, limit)
|
|
102
|
+
.await
|
|
103
|
+
.into_iter()
|
|
104
|
+
.collect()
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
//! The per-call in-memory `/tmp` (contracts/ABI.md "Optional WASI: /tmp").
|
|
2
|
+
//!
|
|
3
|
+
//! The V8 host preopens `/tmp` as an in-memory directory: `std::fs` works under it, it is empty at the start
|
|
4
|
+
//! of every call and discarded after it, and its size is capped (default 64 MiB, `ENOSPC` beyond). File data
|
|
5
|
+
//! lives in the guest's linear memory. Everything outside `/tmp` has no file system.
|
|
6
|
+
//!
|
|
7
|
+
//! `std::env::temp_dir()` **panics** on `wasm32-wasip1` ("not supported by WASI yet"), whatever `TMPDIR` says,
|
|
8
|
+
//! so use [`temp_dir`] here, and for crates that default to `std::env::temp_dir()` use their `*_in` forms
|
|
9
|
+
//! (for example `tempfile::Builder::new().tempfile_in(convex_fn::fs::temp_dir())`).
|
|
10
|
+
|
|
11
|
+
use std::path::PathBuf;
|
|
12
|
+
|
|
13
|
+
/// The directory the host preopens for temporary files.
|
|
14
|
+
pub const TMP_DIR: &str = "/tmp";
|
|
15
|
+
|
|
16
|
+
/// `/tmp`; the replacement for `std::env::temp_dir()`, which panics on `wasm32-wasip1`.
|
|
17
|
+
pub fn temp_dir() -> PathBuf {
|
|
18
|
+
PathBuf::from(TMP_DIR)
|
|
19
|
+
}
|