create-sia-app 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/dist/index.d.ts +1 -0
  2. package/dist/index.js +179 -0
  3. package/package.json +29 -0
  4. package/template/CLAUDE.md +160 -0
  5. package/template/README.md +102 -0
  6. package/template/_gitignore +5 -0
  7. package/template/biome.json +40 -0
  8. package/template/index.html +13 -0
  9. package/template/package.json +30 -0
  10. package/template/rust/README.md +16 -0
  11. package/template/rust/sia-sdk-rs/.changeset/added_cancel_function_to_cancel_inflight_packed_uploads.md +6 -0
  12. package/template/rust/sia-sdk-rs/.changeset/check_if_we_have_enough_hosts_prior_to_encoding_in_upload_slabs.md +16 -0
  13. package/template/rust/sia-sdk-rs/.changeset/fix_slab_length_in_packed_object.md +5 -0
  14. package/template/rust/sia-sdk-rs/.changeset/fix_upload_racing_race_conditon.md +13 -0
  15. package/template/rust/sia-sdk-rs/.changeset/improved_parallelism_of_packed_uploads.md +5 -0
  16. package/template/rust/sia-sdk-rs/.changeset/progress_callback_will_now_be_called_as_expected_for_packed_uploads.md +5 -0
  17. package/template/rust/sia-sdk-rs/.github/dependabot.yml +10 -0
  18. package/template/rust/sia-sdk-rs/.github/workflows/main.yml +36 -0
  19. package/template/rust/sia-sdk-rs/.github/workflows/prepare-release.yml +34 -0
  20. package/template/rust/sia-sdk-rs/.github/workflows/release.yml +30 -0
  21. package/template/rust/sia-sdk-rs/.rustfmt.toml +4 -0
  22. package/template/rust/sia-sdk-rs/Cargo.lock +4127 -0
  23. package/template/rust/sia-sdk-rs/Cargo.toml +3 -0
  24. package/template/rust/sia-sdk-rs/LICENSE +21 -0
  25. package/template/rust/sia-sdk-rs/README.md +30 -0
  26. package/template/rust/sia-sdk-rs/indexd/CHANGELOG.md +79 -0
  27. package/template/rust/sia-sdk-rs/indexd/Cargo.toml +79 -0
  28. package/template/rust/sia-sdk-rs/indexd/benches/upload.rs +258 -0
  29. package/template/rust/sia-sdk-rs/indexd/src/app_client.rs +1710 -0
  30. package/template/rust/sia-sdk-rs/indexd/src/builder.rs +354 -0
  31. package/template/rust/sia-sdk-rs/indexd/src/download.rs +379 -0
  32. package/template/rust/sia-sdk-rs/indexd/src/hosts.rs +659 -0
  33. package/template/rust/sia-sdk-rs/indexd/src/lib.rs +827 -0
  34. package/template/rust/sia-sdk-rs/indexd/src/mock.rs +162 -0
  35. package/template/rust/sia-sdk-rs/indexd/src/object_encryption.rs +125 -0
  36. package/template/rust/sia-sdk-rs/indexd/src/quic.rs +575 -0
  37. package/template/rust/sia-sdk-rs/indexd/src/rhp4.rs +52 -0
  38. package/template/rust/sia-sdk-rs/indexd/src/slabs.rs +497 -0
  39. package/template/rust/sia-sdk-rs/indexd/src/upload.rs +629 -0
  40. package/template/rust/sia-sdk-rs/indexd/src/wasm_time.rs +41 -0
  41. package/template/rust/sia-sdk-rs/indexd/src/web_transport.rs +398 -0
  42. package/template/rust/sia-sdk-rs/indexd_ffi/CHANGELOG.md +76 -0
  43. package/template/rust/sia-sdk-rs/indexd_ffi/Cargo.toml +47 -0
  44. package/template/rust/sia-sdk-rs/indexd_ffi/examples/python/README.md +10 -0
  45. package/template/rust/sia-sdk-rs/indexd_ffi/examples/python/example.py +130 -0
  46. package/template/rust/sia-sdk-rs/indexd_ffi/src/bin/uniffi-bindgen.rs +3 -0
  47. package/template/rust/sia-sdk-rs/indexd_ffi/src/builder.rs +377 -0
  48. package/template/rust/sia-sdk-rs/indexd_ffi/src/io.rs +155 -0
  49. package/template/rust/sia-sdk-rs/indexd_ffi/src/lib.rs +1039 -0
  50. package/template/rust/sia-sdk-rs/indexd_ffi/src/logging.rs +58 -0
  51. package/template/rust/sia-sdk-rs/indexd_ffi/src/tls.rs +23 -0
  52. package/template/rust/sia-sdk-rs/indexd_wasm/Cargo.toml +33 -0
  53. package/template/rust/sia-sdk-rs/indexd_wasm/src/lib.rs +818 -0
  54. package/template/rust/sia-sdk-rs/knope.toml +54 -0
  55. package/template/rust/sia-sdk-rs/sia_derive/CHANGELOG.md +38 -0
  56. package/template/rust/sia-sdk-rs/sia_derive/Cargo.toml +19 -0
  57. package/template/rust/sia-sdk-rs/sia_derive/src/lib.rs +278 -0
  58. package/template/rust/sia-sdk-rs/sia_sdk/CHANGELOG.md +91 -0
  59. package/template/rust/sia-sdk-rs/sia_sdk/Cargo.toml +59 -0
  60. package/template/rust/sia-sdk-rs/sia_sdk/benches/merkle_root.rs +12 -0
  61. package/template/rust/sia-sdk-rs/sia_sdk/src/blake2.rs +22 -0
  62. package/template/rust/sia-sdk-rs/sia_sdk/src/consensus.rs +767 -0
  63. package/template/rust/sia-sdk-rs/sia_sdk/src/encoding/v1.rs +257 -0
  64. package/template/rust/sia-sdk-rs/sia_sdk/src/encoding/v2.rs +291 -0
  65. package/template/rust/sia-sdk-rs/sia_sdk/src/encoding.rs +26 -0
  66. package/template/rust/sia-sdk-rs/sia_sdk/src/encoding_async/v2.rs +367 -0
  67. package/template/rust/sia-sdk-rs/sia_sdk/src/encoding_async.rs +6 -0
  68. package/template/rust/sia-sdk-rs/sia_sdk/src/encryption.rs +303 -0
  69. package/template/rust/sia-sdk-rs/sia_sdk/src/erasure_coding.rs +347 -0
  70. package/template/rust/sia-sdk-rs/sia_sdk/src/lib.rs +15 -0
  71. package/template/rust/sia-sdk-rs/sia_sdk/src/macros.rs +435 -0
  72. package/template/rust/sia-sdk-rs/sia_sdk/src/merkle.rs +112 -0
  73. package/template/rust/sia-sdk-rs/sia_sdk/src/rhp/merkle.rs +357 -0
  74. package/template/rust/sia-sdk-rs/sia_sdk/src/rhp/rpc.rs +1507 -0
  75. package/template/rust/sia-sdk-rs/sia_sdk/src/rhp/types.rs +146 -0
  76. package/template/rust/sia-sdk-rs/sia_sdk/src/rhp.rs +7 -0
  77. package/template/rust/sia-sdk-rs/sia_sdk/src/seed.rs +278 -0
  78. package/template/rust/sia-sdk-rs/sia_sdk/src/signing.rs +236 -0
  79. package/template/rust/sia-sdk-rs/sia_sdk/src/types/common.rs +677 -0
  80. package/template/rust/sia-sdk-rs/sia_sdk/src/types/currency.rs +450 -0
  81. package/template/rust/sia-sdk-rs/sia_sdk/src/types/specifier.rs +110 -0
  82. package/template/rust/sia-sdk-rs/sia_sdk/src/types/spendpolicy.rs +778 -0
  83. package/template/rust/sia-sdk-rs/sia_sdk/src/types/utils.rs +117 -0
  84. package/template/rust/sia-sdk-rs/sia_sdk/src/types/v1.rs +1737 -0
  85. package/template/rust/sia-sdk-rs/sia_sdk/src/types/v2.rs +1726 -0
  86. package/template/rust/sia-sdk-rs/sia_sdk/src/types/work.rs +59 -0
  87. package/template/rust/sia-sdk-rs/sia_sdk/src/types.rs +16 -0
  88. package/template/scripts/setup-rust.js +29 -0
  89. package/template/src/App.tsx +13 -0
  90. package/template/src/components/DevNote.tsx +21 -0
  91. package/template/src/components/auth/ApproveScreen.tsx +84 -0
  92. package/template/src/components/auth/AuthFlow.tsx +77 -0
  93. package/template/src/components/auth/ConnectScreen.tsx +214 -0
  94. package/template/src/components/auth/LoadingScreen.tsx +8 -0
  95. package/template/src/components/auth/RecoveryScreen.tsx +182 -0
  96. package/template/src/components/upload/UploadZone.tsx +314 -0
  97. package/template/src/index.css +9 -0
  98. package/template/src/lib/constants.ts +8 -0
  99. package/template/src/lib/format.ts +35 -0
  100. package/template/src/lib/hex.ts +13 -0
  101. package/template/src/lib/sdk.ts +25 -0
  102. package/template/src/lib/wasm-env.ts +5 -0
  103. package/template/src/main.tsx +12 -0
  104. package/template/src/stores/auth.ts +86 -0
  105. package/template/tsconfig.app.json +31 -0
  106. package/template/tsconfig.json +7 -0
  107. package/template/tsconfig.node.json +26 -0
  108. package/template/vite.config.ts +18 -0
  109. package/template/wasm/indexd_wasm/indexd_wasm.d.ts +309 -0
  110. package/template/wasm/indexd_wasm/indexd_wasm.js +1507 -0
  111. package/template/wasm/indexd_wasm/indexd_wasm_bg.wasm +0 -0
  112. package/template/wasm/indexd_wasm/package.json +31 -0
@@ -0,0 +1,827 @@
1
+ use std::sync::Arc;
2
+
3
+ use crate::app_client::{HostQuery, SlabPinParams};
4
+ use crate::rhp4::RHP4Client;
5
+
6
+ use chrono::{DateTime, Utc};
7
+ use sia::signing::PrivateKey;
8
+ pub use slabs::*;
9
+
10
+ mod hosts;
11
+ pub use hosts::*;
12
+
13
+ use crate::app_client::{Account, AppClient, ObjectsCursor};
14
+ use sia::rhp::Host;
15
+ use sia::types::Hash256;
16
+ use thiserror::Error;
17
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
18
+
19
+ pub use reqwest::{IntoUrl, Url};
20
+
21
+ mod rhp4;
22
+ mod upload;
23
+ pub use upload::*;
24
+
25
+ mod download;
26
+ pub use download::*;
27
+
28
+ #[cfg(any(test, feature = "mock"))]
29
+ pub mod mock;
30
+
31
+ mod object_encryption;
32
+ mod slabs;
33
+
34
+ pub mod app_client;
35
+
36
+ #[cfg(not(target_arch = "wasm32"))]
37
+ pub mod quic;
38
+
39
+ #[cfg(target_arch = "wasm32")]
40
+ pub mod web_transport;
41
+
42
+ #[cfg(target_arch = "wasm32")]
43
+ pub(crate) mod wasm_time;
44
+
45
+ mod builder;
46
+ pub use builder::*;
47
+
48
+ #[derive(Error, Debug)]
49
+ pub enum Error {
50
+ #[error("app error: {0}")]
51
+ App(String),
52
+
53
+ #[error("upload error: {0}")]
54
+ Upload(#[from] UploadError),
55
+
56
+ #[error("download error: {0}")]
57
+ Download(#[from] DownloadError),
58
+
59
+ #[error("TLS error: {0}")]
60
+ Tls(String),
61
+
62
+ #[error("sealed object: {0}")]
63
+ SealedObject(#[from] SealedObjectError),
64
+ }
65
+
66
+ #[derive(Clone)]
67
+ pub struct SDK {
68
+ app_key: Arc<PrivateKey>,
69
+ api_client: Arc<dyn AppClient>,
70
+ downloader: Downloader,
71
+ uploader: Uploader,
72
+ }
73
+
74
+ impl SDK {
75
+ /// Returns default download options with the configured concurrency settings.
76
+ #[cfg(target_arch = "wasm32")]
77
+ pub fn default_download_options(&self) -> DownloadOptions {
78
+ self.downloader.default_options()
79
+ }
80
+
81
+ /// Returns default upload options with the configured concurrency settings.
82
+ #[cfg(target_arch = "wasm32")]
83
+ pub fn default_upload_options(&self) -> UploadOptions {
84
+ self.uploader.default_options()
85
+ }
86
+ /// Creates a new SDK instance.
87
+ #[cfg(not(target_arch = "wasm32"))]
88
+ async fn new(
89
+ api_client: Arc<dyn AppClient>,
90
+ app_key: Arc<PrivateKey>,
91
+ tls_config: rustls::ClientConfig,
92
+ ) -> Result<Self, BuilderError> {
93
+ let usable_hosts = api_client.hosts(&app_key, HostQuery::default()).await?;
94
+ let hosts = Hosts::new();
95
+ hosts.update(usable_hosts);
96
+
97
+ let transport = quic::Client::new(tls_config, hosts.clone())?;
98
+
99
+ let downloader =
100
+ Downloader::new(hosts.clone(), Arc::new(transport.clone()), app_key.clone());
101
+ let uploader = Uploader::new(hosts.clone(), Arc::new(transport), app_key.clone());
102
+ Ok(Self {
103
+ app_key,
104
+ api_client,
105
+ downloader,
106
+ uploader,
107
+ })
108
+ }
109
+
110
+ /// Creates a new SDK instance.
111
+ #[cfg(target_arch = "wasm32")]
112
+ async fn new(
113
+ api_client: Arc<dyn AppClient>,
114
+ app_key: Arc<PrivateKey>,
115
+ concurrency: crate::builder::ConcurrencyConfig,
116
+ ) -> Result<Self, BuilderError> {
117
+ // Fetch all hosts - concurrency limits in price fetching and downloads prevent crashes
118
+ let usable_hosts = api_client.hosts(&app_key, HostQuery::default()).await?;
119
+ let hosts = Hosts::new();
120
+ hosts.update(usable_hosts);
121
+
122
+ let transport = web_transport::Client::new(hosts.clone(), concurrency.max_price_fetches);
123
+
124
+ let downloader = Downloader::new(
125
+ hosts.clone(),
126
+ Arc::new(transport.clone()),
127
+ app_key.clone(),
128
+ concurrency.max_downloads,
129
+ );
130
+ let uploader = Uploader::new(
131
+ hosts.clone(),
132
+ Arc::new(transport),
133
+ app_key.clone(),
134
+ concurrency.max_uploads,
135
+ );
136
+ Ok(Self {
137
+ app_key,
138
+ api_client,
139
+ downloader,
140
+ uploader,
141
+ })
142
+ }
143
+
144
+ /// Returns the application key used by the SDK.
145
+ ///
146
+ /// This should be kept secret and secure. Applications
147
+ /// should store it safely.
148
+ pub fn app_key(&self) -> &PrivateKey {
149
+ &self.app_key
150
+ }
151
+
152
+ /// Reads until EOF and uploads all slabs.
153
+ /// The data will be erasure coded, encrypted,
154
+ /// and uploaded using the uploader's parameters.
155
+ ///
156
+ /// # Arguments
157
+ /// * `r` - The reader to read the data from. It will be read until EOF.
158
+ /// * `options` - The [UploadOptions] to use for the upload.
159
+ ///
160
+ /// # Returns
161
+ /// A new object containing the metadata needed to download the object. The object can be sealed and pinned to the
162
+ /// indexer when ready.
163
+ pub async fn upload<R: AsyncReadExt + Unpin + Send + 'static>(
164
+ &self,
165
+ reader: R,
166
+ options: UploadOptions,
167
+ ) -> Result<Object, UploadError> {
168
+ let object = self.uploader.upload(reader, options).await?;
169
+ Ok(object)
170
+ }
171
+
172
+ /// Creates a new packed upload. This allows multiple objects to be packed together
173
+ /// for more efficient uploads. The returned `PackedUpload` can be used to add objects to the upload, and then finalized to get the resulting objects.
174
+ ///
175
+ /// # Arguments
176
+ /// * `options` - The [UploadOptions] to use for the upload.
177
+ ///
178
+ /// # Returns
179
+ /// A [PackedUpload] that can be used to add objects and finalize the upload.
180
+ pub fn upload_packed(&self, options: UploadOptions) -> PackedUpload {
181
+ self.uploader.upload_packed(options)
182
+ }
183
+
184
+ /// Downloads an object using the provided writer and options.
185
+ pub async fn download<W: AsyncWriteExt + Unpin>(
186
+ &self,
187
+ w: &mut W,
188
+ object: &Object,
189
+ options: DownloadOptions,
190
+ ) -> Result<(), DownloadError> {
191
+ self.downloader.download(w, object, options).await?;
192
+ Ok(())
193
+ }
194
+
195
+ /// Retrieves a list of hosts from the indexer matching the provided query
196
+ /// that can be used for uploading and downloading data.
197
+ ///
198
+ /// # Arguments
199
+ /// * `query` - Filtering criteria to select hosts.
200
+ pub async fn hosts(&self, query: HostQuery) -> Result<Vec<Host>, Error> {
201
+ self.api_client
202
+ .hosts(&self.app_key, query)
203
+ .await
204
+ .map_err(|e| Error::App(format!("{e:?}")))
205
+ }
206
+
207
+ /// Retrieves account information from the indexer.
208
+ pub async fn account(&self) -> Result<Account, Error> {
209
+ self.api_client
210
+ .account(&self.app_key)
211
+ .await
212
+ .map_err(|e| Error::App(format!("{e:?}")))
213
+ }
214
+
215
+ /// Retrieves an object from the indexer by its key.
216
+ ///
217
+ /// # Arguments
218
+ /// * `key` - The key of the object to retrieve.
219
+ pub async fn object(&self, key: &Hash256) -> Result<Object, Error> {
220
+ let sealed = self
221
+ .api_client
222
+ .object(&self.app_key, key)
223
+ .await
224
+ .map_err(|e| Error::App(format!("{e:?}")))?;
225
+
226
+ let obj = sealed.open(&self.app_key)?;
227
+ Ok(obj)
228
+ }
229
+
230
+ /// Retrieves a list of object events from the indexer. This
231
+ /// can be used to synchronize local state with the indexer.
232
+ ///
233
+ /// # Arguments
234
+ /// * `cursor` - An optional cursor to continue from a previous call.
235
+ /// * `limit` - An optional limit on the number of events to retrieve.
236
+ pub async fn object_events(
237
+ &self,
238
+ cursor: Option<ObjectsCursor>,
239
+ limit: Option<usize>,
240
+ ) -> Result<Vec<ObjectEvent>, Error> {
241
+ let events = self
242
+ .api_client
243
+ .objects(&self.app_key, cursor, limit)
244
+ .await
245
+ .map_err(|e| Error::App(format!("{e:?}")))?;
246
+
247
+ let objs = events
248
+ .into_iter()
249
+ .map(|event| {
250
+ let object = match event.object {
251
+ Some(sealed) => Some(sealed.open(&self.app_key)?),
252
+ None => None,
253
+ };
254
+ Ok(ObjectEvent {
255
+ id: event.id,
256
+ deleted: event.deleted,
257
+ updated_at: event.updated_at,
258
+ object,
259
+ })
260
+ })
261
+ .collect::<Result<_, Error>>()?;
262
+
263
+ Ok(objs)
264
+ }
265
+
266
+ /// Prunes unused slabs from the indexer. This helps to free up
267
+ /// storage space by removing slabs that are no longer
268
+ /// referenced by objects.
269
+ pub async fn prune_slabs(&self) -> Result<(), Error> {
270
+ self.api_client
271
+ .prune_slabs(&self.app_key)
272
+ .await
273
+ .map_err(|e| Error::App(format!("{e:?}")))?;
274
+ Ok(())
275
+ }
276
+
277
+ /// Updates the metadata of an object in the indexer. The object
278
+ /// must already be pinned to the indexer.
279
+ ///
280
+ /// # Arguments
281
+ /// * `object` - The object to update.
282
+ pub async fn update_object_metadata(&self, object: &Object) -> Result<(), Error> {
283
+ let sealed = object.seal(&self.app_key);
284
+ self.api_client
285
+ .save_object(&self.app_key, &sealed)
286
+ .await
287
+ .map_err(|e| Error::App(format!("{e:?}")))?;
288
+ Ok(())
289
+ }
290
+
291
+ /// Deletes the object with the given id.
292
+ ///
293
+ /// # Arguments
294
+ /// * `id` - The id of the object to delete.
295
+ pub async fn delete_object(&self, id: &Hash256) -> Result<(), Error> {
296
+ self.api_client
297
+ .delete_object(&self.app_key, id)
298
+ .await
299
+ .map_err(|e| Error::App(format!("{e:?}")))
300
+ }
301
+
302
+ /// Generates a shared URL for the given object that is valid until the specified time.
303
+ ///
304
+ /// This object should be considered public even if the URL is kept secret,
305
+ /// as anyone with the URL can access the object until the expiration time.
306
+ ///
307
+ /// # Arguments
308
+ /// * `object` - The object to share.
309
+ /// * `valid_until` - The time until which the shared URL is valid.
310
+ pub fn share_object(&self, object: &Object, valid_until: DateTime<Utc>) -> Result<Url, Error> {
311
+ self.api_client
312
+ .shared_object_url(&self.app_key, object, valid_until)
313
+ .map_err(|e| Error::App(format!("{e:?}")))
314
+ }
315
+
316
+ /// Retrieves a shared object from the given share URL.
317
+ ///
318
+ /// # Arguments
319
+ /// * `share_url` - The URL of the shared object.
320
+ pub async fn shared_object<U: IntoUrl>(&self, share_url: U) -> Result<Object, Error> {
321
+ let share_url = share_url
322
+ .into_url()
323
+ .map_err(|e| Error::App(format!("{e:?}")))?;
324
+ self.api_client
325
+ .shared_object(share_url)
326
+ .await
327
+ .map_err(|e| Error::App(format!("{e:?}")))
328
+ }
329
+
330
+ /// Pins an object to the indexer
331
+ pub async fn pin_object(&self, object: &Object) -> Result<(), Error> {
332
+ let slabs = object
333
+ .slabs()
334
+ .iter()
335
+ .map(|s| SlabPinParams {
336
+ encryption_key: s.encryption_key.clone(),
337
+ min_shards: s.min_shards,
338
+ sectors: s.sectors.clone(),
339
+ })
340
+ .collect();
341
+
342
+ self.api_client
343
+ .pin_slabs(&self.app_key, slabs)
344
+ .await
345
+ .map_err(|e| Error::App(format!("{e:?}")))?;
346
+
347
+ self.api_client
348
+ .save_object(&self.app_key, &object.seal(&self.app_key))
349
+ .await
350
+ .map_err(|e| Error::App(format!("{e:?}")))?;
351
+ Ok(())
352
+ }
353
+
354
+ /// Retrieves a pinned slab from the indexer by its id.
355
+ pub async fn slab(&self, id: &Hash256) -> Result<PinnedSlab, Error> {
356
+ self.api_client
357
+ .slab(&self.app_key, id)
358
+ .await
359
+ .map_err(|e| Error::App(format!("{e:?}")))
360
+ }
361
+ }
362
+
363
+ #[cfg(test)]
364
+ mod test {
365
+ use bytes::{Bytes, BytesMut};
366
+ use rand::Rng;
367
+ use sia::rhp::SECTOR_SIZE;
368
+ use sia::types::v2::NetAddress;
369
+ use std::io::Cursor;
370
+ use std::time::Duration;
371
+
372
+ use crate::mock::{MockDownloader, MockRHP4Client, MockUploader};
373
+
374
+ use super::*;
375
+
376
+ const SLAB_SIZE: u64 = SECTOR_SIZE as u64 * 10; // 10 sectors per slab
377
+
378
+ #[tokio::test]
379
+ async fn test_upload_download_packed() {
380
+ let app_key = Arc::new(PrivateKey::from_seed(&rand::random()));
381
+ let transport = Arc::new(MockRHP4Client::new());
382
+ let hosts = Hosts::new();
383
+
384
+ hosts.update(
385
+ (0..60)
386
+ .map(|_| Host {
387
+ public_key: PrivateKey::from_seed(&rand::random()).public_key(),
388
+ addresses: vec![NetAddress {
389
+ protocol: sia::types::v2::Protocol::QUIC,
390
+ address: "localhost:1234".to_string(),
391
+ }],
392
+ country_code: "US".to_string(),
393
+ latitude: 0.0,
394
+ longitude: 0.0,
395
+ good_for_upload: true,
396
+ })
397
+ .collect(),
398
+ );
399
+
400
+ let uploader = MockUploader::new(hosts.clone(), transport.clone(), app_key.clone());
401
+ let downloader = MockDownloader::new(hosts.clone(), transport.clone(), app_key.clone());
402
+
403
+ let input: Bytes = Bytes::from("Hello, world!");
404
+
405
+ let mut packed_upload = uploader.upload_packed(UploadOptions::default());
406
+ assert_eq!(packed_upload.remaining(), SLAB_SIZE);
407
+
408
+ packed_upload
409
+ .add(Cursor::new(input.clone()))
410
+ .await
411
+ .expect("add 1 to complete");
412
+ packed_upload
413
+ .add(Cursor::new(input.clone()))
414
+ .await
415
+ .expect("add 2 to complete");
416
+
417
+ assert_eq!(
418
+ packed_upload.remaining(),
419
+ SLAB_SIZE - (input.len() * 2) as u64
420
+ );
421
+
422
+ let objects = packed_upload.finalize().await.expect("upload to finish");
423
+ assert_eq!(objects.len(), 2);
424
+ assert_ne!(objects[0].id(), objects[1].id()); // encryption keys should be different
425
+
426
+ // Both objects should have 1 slab each, since the input is small enough to fit in a single slab.
427
+ assert_eq!(objects[0].slabs().len(), 1);
428
+ assert_eq!(objects[1].slabs().len(), 1);
429
+
430
+ // obj 0 should be the first 13 bytes
431
+ assert_eq!(objects[0].slabs()[0].offset, 0);
432
+ assert_eq!(objects[0].size(), 13);
433
+
434
+ // obj 1 should be the next 13 bytes
435
+ assert_eq!(objects[1].slabs()[0].offset, 13);
436
+ assert_eq!(objects[1].size(), 13);
437
+
438
+ let mut output = BytesMut::zeroed(13);
439
+ downloader
440
+ .download(
441
+ &mut Cursor::new(&mut output[..]),
442
+ &objects[0],
443
+ DownloadOptions::default(),
444
+ )
445
+ .await
446
+ .expect("download to complete");
447
+
448
+ assert_eq!(output.freeze(), input.clone());
449
+
450
+ let mut output = BytesMut::zeroed(13);
451
+ downloader
452
+ .download(
453
+ &mut Cursor::new(&mut output[..]),
454
+ &objects[1],
455
+ DownloadOptions::default(),
456
+ )
457
+ .await
458
+ .expect("download to complete");
459
+
460
+ assert_eq!(output.freeze(), input.clone());
461
+ }
462
+
463
+ #[tokio::test]
464
+ async fn test_upload_download_packed_spanning() {
465
+ let app_key = Arc::new(PrivateKey::from_seed(&rand::random()));
466
+ let transport = Arc::new(MockRHP4Client::new());
467
+ let hosts = Hosts::new();
468
+
469
+ hosts.update(
470
+ (0..60)
471
+ .map(|_| Host {
472
+ public_key: PrivateKey::from_seed(&rand::random()).public_key(),
473
+ addresses: vec![NetAddress {
474
+ protocol: sia::types::v2::Protocol::QUIC,
475
+ address: "localhost:1234".to_string(),
476
+ }],
477
+ country_code: "US".to_string(),
478
+ latitude: 0.0,
479
+ longitude: 0.0,
480
+ good_for_upload: true,
481
+ })
482
+ .collect(),
483
+ );
484
+
485
+ let uploader = MockUploader::new(hosts.clone(), transport.clone(), app_key.clone());
486
+ let downloader = MockDownloader::new(hosts.clone(), transport.clone(), app_key.clone());
487
+
488
+ let small_input = Bytes::from("Hello, world!");
489
+
490
+ let mut large_input = BytesMut::zeroed(SLAB_SIZE as usize + 18); // 1 full slab + 18 bytes
491
+ rand::rng().fill_bytes(&mut large_input);
492
+ let large_input = large_input.freeze();
493
+
494
+ let mut packed_upload = uploader.upload_packed(UploadOptions::default());
495
+ packed_upload
496
+ .add(Cursor::new(small_input.clone()))
497
+ .await
498
+ .expect("add 1 to complete");
499
+ packed_upload
500
+ .add(Cursor::new(large_input.clone()))
501
+ .await
502
+ .expect("add 2 to complete");
503
+
504
+ let objects = packed_upload.finalize().await.expect("upload to finish");
505
+ assert_eq!(objects.len(), 2);
506
+
507
+ // The first object should have 1 slab
508
+ assert_eq!(objects[0].slabs().len(), 1);
509
+ assert_eq!(objects[1].slabs().len(), 2);
510
+
511
+ // obj 0 should be the small input
512
+ assert_eq!(objects[0].size(), 13);
513
+ assert_eq!(objects[0].slabs()[0].offset, 0);
514
+ assert_eq!(objects[0].slabs()[0].length, 13);
515
+
516
+ // obj 1 should be the large input. The first slab starts at offset 13 so
517
+ // its length must be SLAB_SIZE - 13. The second slab has the remaining bytes.
518
+ assert_eq!(objects[1].size(), SLAB_SIZE + 18);
519
+ assert_eq!(objects[1].slabs()[0].offset, 13);
520
+ assert_eq!(objects[1].slabs()[0].length, (SLAB_SIZE - 13) as u32);
521
+ assert_eq!(objects[1].slabs()[1].offset, 0);
522
+ assert_eq!(objects[1].slabs()[1].length, 18 + 13);
523
+
524
+ let mut output = BytesMut::zeroed(objects[0].size() as usize);
525
+ downloader
526
+ .download(
527
+ &mut Cursor::new(&mut output[..]),
528
+ &objects[0],
529
+ DownloadOptions::default(),
530
+ )
531
+ .await
532
+ .expect("download to complete");
533
+
534
+ assert_eq!(output.freeze(), small_input);
535
+
536
+ let mut output = BytesMut::zeroed(objects[1].size() as usize);
537
+ downloader
538
+ .download(
539
+ &mut Cursor::new(&mut output[..]),
540
+ &objects[1],
541
+ DownloadOptions::default(),
542
+ )
543
+ .await
544
+ .expect("download to complete");
545
+
546
+ assert_eq!(output.freeze(), large_input);
547
+ }
548
+
549
+ #[tokio::test]
550
+ async fn test_upload_download_packed_exact() {
551
+ let app_key = Arc::new(PrivateKey::from_seed(&rand::random()));
552
+ let transport = Arc::new(MockRHP4Client::new());
553
+ let hosts = Hosts::new();
554
+
555
+ hosts.update(
556
+ (0..60)
557
+ .map(|_| Host {
558
+ public_key: PrivateKey::from_seed(&rand::random()).public_key(),
559
+ addresses: vec![NetAddress {
560
+ protocol: sia::types::v2::Protocol::QUIC,
561
+ address: "localhost:1234".to_string(),
562
+ }],
563
+ country_code: "US".to_string(),
564
+ latitude: 0.0,
565
+ longitude: 0.0,
566
+ good_for_upload: true,
567
+ })
568
+ .collect(),
569
+ );
570
+
571
+ let uploader = MockUploader::new(hosts.clone(), transport.clone(), app_key.clone());
572
+ let downloader = MockDownloader::new(hosts.clone(), transport.clone(), app_key.clone());
573
+
574
+ let mut exact_input = BytesMut::zeroed(SLAB_SIZE as usize); // 1 full slab
575
+ rand::rng().fill_bytes(&mut exact_input);
576
+ let exact_input = exact_input.freeze();
577
+
578
+ let mut packed_upload = uploader.upload_packed(UploadOptions::default());
579
+ packed_upload
580
+ .add(Cursor::new(exact_input.clone()))
581
+ .await
582
+ .expect("add 1 to complete");
583
+
584
+ let objects = packed_upload.finalize().await.expect("upload to finish");
585
+ assert_eq!(objects.len(), 1);
586
+
587
+ // The first object should have 1 slab, since it fits exactly
588
+ assert_eq!(objects[0].slabs().len(), 1);
589
+ // the first slab of obj[0] should be the full length. the second slab should be the remaining 18 bytes.
590
+ assert_eq!(objects[0].size(), SLAB_SIZE);
591
+ assert_eq!(objects[0].slabs()[0].offset, 0);
592
+ assert_eq!(objects[0].slabs()[0].length, SLAB_SIZE as u32);
593
+
594
+ let mut output = BytesMut::zeroed(objects[0].size() as usize);
595
+ downloader
596
+ .download(
597
+ &mut Cursor::new(&mut output[..]),
598
+ &objects[0],
599
+ DownloadOptions::default(),
600
+ )
601
+ .await
602
+ .expect("download to complete");
603
+
604
+ assert_eq!(output.freeze(), exact_input);
605
+ }
606
+
607
+ #[tokio::test]
608
+ async fn test_upload_download() {
609
+ let app_key = Arc::new(PrivateKey::from_seed(&rand::random()));
610
+ let transport = Arc::new(MockRHP4Client::new());
611
+ let hosts = Hosts::new();
612
+
613
+ hosts.update(
614
+ (0..60)
615
+ .map(|_| Host {
616
+ public_key: PrivateKey::from_seed(&rand::random()).public_key(),
617
+ addresses: vec![NetAddress {
618
+ protocol: sia::types::v2::Protocol::QUIC,
619
+ address: "localhost:1234".to_string(),
620
+ }],
621
+ country_code: "US".to_string(),
622
+ latitude: 0.0,
623
+ longitude: 0.0,
624
+ good_for_upload: true,
625
+ })
626
+ .collect(),
627
+ );
628
+
629
+ let uploader = MockUploader::new(hosts.clone(), transport.clone(), app_key.clone());
630
+ let downloader = MockDownloader::new(hosts.clone(), transport.clone(), app_key.clone());
631
+
632
+ let input: Bytes = Bytes::from("Hello, world!");
633
+
634
+ let object = uploader
635
+ .upload(Cursor::new(input.clone()), UploadOptions::default())
636
+ .await
637
+ .expect("upload to complete");
638
+
639
+ assert_eq!(object.slabs().len(), 1);
640
+ assert_eq!(object.size(), 13);
641
+
642
+ let mut output = BytesMut::zeroed(object.size() as usize);
643
+ downloader
644
+ .download(
645
+ &mut Cursor::new(&mut output[..]),
646
+ &object,
647
+ DownloadOptions::default(),
648
+ )
649
+ .await
650
+ .expect("download to complete");
651
+
652
+ assert_eq!(output.freeze(), input.clone());
653
+
654
+ let range = 7..13;
655
+ let mut output = BytesMut::zeroed(range.end - range.start);
656
+ downloader
657
+ .download(
658
+ &mut Cursor::new(&mut output[..]),
659
+ &object,
660
+ DownloadOptions {
661
+ offset: range.start as u64,
662
+ length: Some((range.end - range.start) as u64),
663
+ ..Default::default()
664
+ },
665
+ )
666
+ .await
667
+ .expect("download to complete");
668
+
669
+ assert_eq!(output.freeze(), input.slice(range));
670
+ }
671
+
672
+ #[tokio::test]
673
+ async fn test_upload_no_hosts() {
674
+ let app_key = Arc::new(PrivateKey::from_seed(&rand::random()));
675
+ let transport = Arc::new(MockRHP4Client::new());
676
+ let hosts = Hosts::new();
677
+
678
+ let uploader = MockUploader::new(hosts.clone(), transport.clone(), app_key.clone());
679
+
680
+ let input: Bytes = Bytes::from("Hello, world!");
681
+
682
+ let err = uploader
683
+ .upload(Cursor::new(input.clone()), UploadOptions::default())
684
+ .await
685
+ .expect_err("upload to fail");
686
+
687
+ match err {
688
+ UploadError::QueueError(QueueError::InsufficientHosts) => (),
689
+ _ => panic!(),
690
+ }
691
+ }
692
+
693
+ /// Tests that upload succeeds even when some hosts are slow, as long as
694
+ /// there are enough fast hosts to complete the upload.
695
+ /// This mirrors Go's TestUpload "slow" subtest.
696
+ #[tokio::test]
697
+ async fn test_upload_slow_host() {
698
+ let app_key = Arc::new(PrivateKey::from_seed(&rand::random()));
699
+ let transport = Arc::new(MockRHP4Client::new());
700
+ let hosts = Hosts::new();
701
+
702
+ // Create 30 hosts and track their public keys
703
+ let host_keys: Vec<_> = (0..30)
704
+ .map(|_| PrivateKey::from_seed(&rand::random()).public_key())
705
+ .collect();
706
+
707
+ hosts.update(
708
+ host_keys
709
+ .iter()
710
+ .map(|pk| Host {
711
+ public_key: *pk,
712
+ addresses: vec![NetAddress {
713
+ protocol: sia::types::v2::Protocol::QUIC,
714
+ address: "localhost:1234".to_string(),
715
+ }],
716
+ country_code: "US".to_string(),
717
+ latitude: 0.0,
718
+ longitude: 0.0,
719
+ good_for_upload: true,
720
+ })
721
+ .collect(),
722
+ );
723
+
724
+ // make the 1st host slow
725
+ transport.set_slow_hosts(
726
+ host_keys.iter().take(1).copied(),
727
+ tokio::time::Duration::from_secs(2),
728
+ );
729
+
730
+ let uploader = MockUploader::new(hosts.clone(), transport.clone(), app_key.clone());
731
+
732
+ let input: Bytes = Bytes::from("Hello, world!");
733
+
734
+ let object = uploader
735
+ .upload(Cursor::new(input.clone()), UploadOptions::default())
736
+ .await
737
+ .expect("upload should succeed with 1 slow host");
738
+
739
+ assert_eq!(object.slabs().len(), 1);
740
+ }
741
+
742
+ // Upload should succeed even if all initial hosts are slow
743
+ #[tokio::test]
744
+ async fn test_upload_all_hosts_slow() {
745
+ let app_key = Arc::new(PrivateKey::from_seed(&rand::random()));
746
+ let transport = Arc::new(MockRHP4Client::new());
747
+ let hosts = Hosts::new();
748
+
749
+ // Create 30 hosts and track their public keys
750
+ let host_keys: Vec<_> = (0..30)
751
+ .map(|_| PrivateKey::from_seed(&rand::random()).public_key())
752
+ .collect();
753
+
754
+ hosts.update(
755
+ host_keys
756
+ .iter()
757
+ .map(|pk| Host {
758
+ public_key: *pk,
759
+ addresses: vec![NetAddress {
760
+ protocol: sia::types::v2::Protocol::QUIC,
761
+ address: "localhost:1234".to_string(),
762
+ }],
763
+ country_code: "US".to_string(),
764
+ latitude: 0.0,
765
+ longitude: 0.0,
766
+ good_for_upload: true,
767
+ })
768
+ .collect(),
769
+ );
770
+
771
+ // Make all hosts slow
772
+ transport.set_slow_hosts(host_keys.iter().take(30).copied(), Duration::from_secs(2));
773
+
774
+ let uploader = MockUploader::new(hosts.clone(), transport.clone(), app_key.clone());
775
+
776
+ let input: Bytes = Bytes::from("Hello, world!");
777
+
778
+ let _ = uploader
779
+ .upload(Cursor::new(input.clone()), UploadOptions::default())
780
+ .await
781
+ .expect("upload to succeed");
782
+ }
783
+
784
+ #[tokio::test]
785
+ async fn test_upload_not_enough_hosts_good_for_upload() {
786
+ let app_key = Arc::new(PrivateKey::from_seed(&rand::random()));
787
+ let transport = Arc::new(MockRHP4Client::new());
788
+ let hosts = Hosts::new();
789
+
790
+ // Create 30 hosts: 10 good for upload, 20 not good for upload
791
+ let host_keys: Vec<_> = (0..30)
792
+ .map(|_| PrivateKey::from_seed(&rand::random()).public_key())
793
+ .collect();
794
+
795
+ hosts.update(
796
+ host_keys
797
+ .iter()
798
+ .enumerate()
799
+ .map(|(i, pk)| Host {
800
+ public_key: *pk,
801
+ addresses: vec![NetAddress {
802
+ protocol: sia::types::v2::Protocol::QUIC,
803
+ address: "localhost:1234".to_string(),
804
+ }],
805
+ country_code: "US".to_string(),
806
+ latitude: 0.0,
807
+ longitude: 0.0,
808
+ good_for_upload: i < 10,
809
+ })
810
+ .collect(),
811
+ );
812
+
813
+ let uploader = MockUploader::new(hosts.clone(), transport.clone(), app_key.clone());
814
+
815
+ let input: Bytes = Bytes::from("Hello, world!");
816
+
817
+ let err = uploader
818
+ .upload(Cursor::new(input.clone()), UploadOptions::default())
819
+ .await
820
+ .expect_err("upload to fail");
821
+
822
+ match err {
823
+ UploadError::QueueError(QueueError::InsufficientHosts) => (),
824
+ _ => panic!(),
825
+ }
826
+ }
827
+ }