opendal 0.1.12 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +31 -36
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +25 -35
  5. data/README.md +1 -1
  6. data/core/CHANGELOG.md +4 -0
  7. data/core/Cargo.lock +128 -187
  8. data/core/Cargo.toml +95 -95
  9. data/core/DEPENDENCIES.rust.tsv +259 -289
  10. data/core/LICENSE +16 -0
  11. data/core/benches/vs_fs/Cargo.toml +1 -1
  12. data/core/benches/vs_s3/Cargo.toml +1 -1
  13. data/core/core/CHANGELOG.md +4 -0
  14. data/core/core/Cargo.toml +1 -1
  15. data/core/core/LICENSE +16 -0
  16. data/core/core/src/docs/upgrade.md +8 -0
  17. data/core/core/src/lib.rs +1 -3
  18. data/core/core/src/raw/futures_util.rs +26 -21
  19. data/core/core/src/raw/http_util/error.rs +9 -0
  20. data/core/core/src/raw/ops.rs +1 -0
  21. data/core/core/src/types/http_transport/mod.rs +30 -2
  22. data/core/core/src/types/http_transport/uri.rs +166 -0
  23. data/core/core/src/types/metadata.rs +1 -0
  24. data/core/core/src/types/mod.rs +2 -0
  25. data/core/http-transports/reqwest/Cargo.toml +1 -1
  26. data/core/http-transports/reqwest/LICENSE +16 -0
  27. data/core/http-transports/reqwest/src/lib.rs +46 -4
  28. data/core/layers/async-backtrace/Cargo.toml +2 -2
  29. data/core/layers/async-backtrace/LICENSE +16 -0
  30. data/core/layers/await-tree/Cargo.toml +2 -2
  31. data/core/layers/await-tree/LICENSE +16 -0
  32. data/core/layers/capability-check/Cargo.toml +2 -2
  33. data/core/layers/capability-check/LICENSE +16 -0
  34. data/core/layers/chaos/Cargo.toml +2 -2
  35. data/core/layers/chaos/LICENSE +16 -0
  36. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  37. data/core/layers/concurrent-limit/LICENSE +16 -0
  38. data/core/layers/dtrace/Cargo.toml +2 -2
  39. data/core/layers/dtrace/LICENSE +16 -0
  40. data/core/layers/fastmetrics/Cargo.toml +3 -3
  41. data/core/layers/fastmetrics/LICENSE +16 -0
  42. data/core/layers/fastrace/Cargo.toml +2 -2
  43. data/core/layers/fastrace/LICENSE +16 -0
  44. data/core/layers/foyer/Cargo.toml +2 -2
  45. data/core/layers/foyer/LICENSE +16 -0
  46. data/core/layers/hotpath/Cargo.toml +2 -2
  47. data/core/layers/hotpath/LICENSE +16 -0
  48. data/core/layers/immutable-index/Cargo.toml +2 -2
  49. data/core/layers/immutable-index/LICENSE +16 -0
  50. data/core/layers/logging/Cargo.toml +2 -2
  51. data/core/layers/logging/LICENSE +16 -0
  52. data/core/layers/metrics/Cargo.toml +3 -3
  53. data/core/layers/metrics/LICENSE +16 -0
  54. data/core/layers/mime-guess/Cargo.toml +2 -2
  55. data/core/layers/mime-guess/LICENSE +16 -0
  56. data/core/layers/observe-metrics-common/Cargo.toml +1 -1
  57. data/core/layers/observe-metrics-common/LICENSE +16 -0
  58. data/core/layers/otelmetrics/Cargo.toml +3 -3
  59. data/core/layers/otelmetrics/LICENSE +16 -0
  60. data/core/layers/oteltrace/Cargo.toml +2 -2
  61. data/core/layers/oteltrace/LICENSE +16 -0
  62. data/core/layers/prometheus/Cargo.toml +3 -3
  63. data/core/layers/prometheus/LICENSE +16 -0
  64. data/core/layers/prometheus-client/Cargo.toml +3 -3
  65. data/core/layers/prometheus-client/LICENSE +16 -0
  66. data/core/layers/retry/Cargo.toml +4 -4
  67. data/core/layers/retry/LICENSE +16 -0
  68. data/core/layers/route/Cargo.toml +3 -3
  69. data/core/layers/route/LICENSE +16 -0
  70. data/core/layers/tail-cut/Cargo.toml +2 -2
  71. data/core/layers/tail-cut/LICENSE +16 -0
  72. data/core/layers/throttle/Cargo.toml +2 -2
  73. data/core/layers/throttle/LICENSE +16 -0
  74. data/core/layers/timeout/Cargo.toml +3 -3
  75. data/core/layers/timeout/LICENSE +16 -0
  76. data/core/layers/tracing/Cargo.toml +2 -2
  77. data/core/layers/tracing/LICENSE +16 -0
  78. data/core/services/aliyun-drive/Cargo.toml +2 -2
  79. data/core/services/aliyun-drive/LICENSE +16 -0
  80. data/core/services/alluxio/Cargo.toml +1 -1
  81. data/core/services/alluxio/LICENSE +16 -0
  82. data/core/services/azblob/Cargo.toml +2 -2
  83. data/core/services/azblob/LICENSE +16 -0
  84. data/core/services/azblob/src/backend.rs +21 -7
  85. data/core/services/azblob/src/config.rs +4 -1
  86. data/core/services/azdls/Cargo.toml +3 -3
  87. data/core/services/azdls/LICENSE +16 -0
  88. data/core/services/azfile/Cargo.toml +2 -2
  89. data/core/services/azfile/LICENSE +16 -0
  90. data/core/services/azure-common/Cargo.toml +1 -1
  91. data/core/services/azure-common/LICENSE +16 -0
  92. data/core/services/b2/Cargo.toml +2 -2
  93. data/core/services/b2/LICENSE +16 -0
  94. data/core/services/cacache/Cargo.toml +1 -1
  95. data/core/services/cacache/LICENSE +16 -0
  96. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  97. data/core/services/cloudflare-kv/LICENSE +16 -0
  98. data/core/services/compfs/Cargo.toml +1 -1
  99. data/core/services/compfs/LICENSE +16 -0
  100. data/core/services/compfs/src/core.rs +27 -11
  101. data/core/services/cos/Cargo.toml +1 -1
  102. data/core/services/cos/LICENSE +16 -0
  103. data/core/services/d1/Cargo.toml +1 -1
  104. data/core/services/d1/LICENSE +16 -0
  105. data/core/services/dashmap/Cargo.toml +1 -1
  106. data/core/services/dashmap/LICENSE +16 -0
  107. data/core/services/dbfs/Cargo.toml +1 -1
  108. data/core/services/dbfs/LICENSE +16 -0
  109. data/core/services/dropbox/Cargo.toml +2 -2
  110. data/core/services/dropbox/LICENSE +16 -0
  111. data/core/services/etcd/Cargo.toml +2 -2
  112. data/core/services/etcd/LICENSE +16 -0
  113. data/core/services/etcd/src/core.rs +3 -3
  114. data/core/services/foundationdb/Cargo.toml +1 -1
  115. data/core/services/foundationdb/LICENSE +16 -0
  116. data/core/services/foyer/Cargo.toml +3 -3
  117. data/core/services/foyer/LICENSE +16 -0
  118. data/core/services/fs/Cargo.toml +1 -1
  119. data/core/services/fs/LICENSE +16 -0
  120. data/core/services/fs/src/core.rs +69 -10
  121. data/core/services/ftp/Cargo.toml +2 -2
  122. data/core/services/ftp/LICENSE +16 -0
  123. data/core/services/ftp/src/core.rs +1 -1
  124. data/core/services/ftp/src/reader.rs +1 -1
  125. data/core/services/ftp/src/writer.rs +1 -1
  126. data/core/services/gcs/Cargo.toml +3 -2
  127. data/core/services/gcs/LICENSE +16 -0
  128. data/core/services/gcs/src/composer.rs +11 -21
  129. data/core/services/gcs-grpc/Cargo.toml +1 -1
  130. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +17 -18
  131. data/core/services/gdrive/Cargo.toml +3 -2
  132. data/core/services/gdrive/LICENSE +16 -0
  133. data/core/services/gdrive/src/docs.md +2 -0
  134. data/core/services/gdrive/src/path_index.rs +109 -18
  135. data/core/services/ghac/Cargo.toml +2 -2
  136. data/core/services/ghac/LICENSE +16 -0
  137. data/core/services/github/Cargo.toml +1 -1
  138. data/core/services/github/LICENSE +16 -0
  139. data/core/services/goosefs/Cargo.toml +13 -4
  140. data/core/services/goosefs/LICENSE +16 -0
  141. data/core/services/goosefs/README.md +11 -0
  142. data/core/services/goosefs/src/backend.rs +3 -5
  143. data/core/services/goosefs/src/core.rs +1 -1
  144. data/core/services/goosefs/src/docs.md +44 -3
  145. data/core/services/goosefs/src/reader.rs +22 -89
  146. data/core/services/gridfs/Cargo.toml +2 -2
  147. data/core/services/gridfs/LICENSE +16 -0
  148. data/core/services/hdfs/Cargo.toml +1 -1
  149. data/core/services/hdfs/LICENSE +16 -0
  150. data/core/services/hdfs-native/Cargo.toml +1 -1
  151. data/core/services/hdfs-native/LICENSE +16 -0
  152. data/core/services/hf/Cargo.toml +3 -3
  153. data/core/services/hf/LICENSE +16 -0
  154. data/core/services/hf/src/backend.rs +77 -9
  155. data/core/services/hf/src/config.rs +58 -0
  156. data/core/services/hf/src/core.rs +695 -318
  157. data/core/services/hf/src/docs.md +29 -0
  158. data/core/services/hf/src/lister.rs +3 -3
  159. data/core/services/hf/src/reader.rs +128 -143
  160. data/core/services/http/Cargo.toml +3 -3
  161. data/core/services/http/LICENSE +16 -0
  162. data/core/services/ipfs/Cargo.toml +1 -1
  163. data/core/services/ipfs/LICENSE +16 -0
  164. data/core/services/ipmfs/Cargo.toml +1 -1
  165. data/core/services/ipmfs/LICENSE +16 -0
  166. data/core/services/koofr/Cargo.toml +2 -2
  167. data/core/services/koofr/LICENSE +16 -0
  168. data/core/services/lakefs/Cargo.toml +1 -1
  169. data/core/services/lakefs/LICENSE +16 -0
  170. data/core/services/memcached/Cargo.toml +2 -2
  171. data/core/services/memcached/LICENSE +16 -0
  172. data/core/services/memcached/src/core.rs +3 -3
  173. data/core/services/mini_moka/Cargo.toml +1 -1
  174. data/core/services/mini_moka/LICENSE +16 -0
  175. data/core/services/moka/Cargo.toml +1 -1
  176. data/core/services/moka/LICENSE +16 -0
  177. data/core/services/mongodb/Cargo.toml +2 -2
  178. data/core/services/mongodb/LICENSE +16 -0
  179. data/core/services/monoiofs/Cargo.toml +2 -2
  180. data/core/services/monoiofs/LICENSE +16 -0
  181. data/core/services/monoiofs/src/core.rs +27 -11
  182. data/core/services/mysql/Cargo.toml +2 -2
  183. data/core/services/mysql/LICENSE +16 -0
  184. data/core/services/obs/Cargo.toml +1 -1
  185. data/core/services/obs/LICENSE +16 -0
  186. data/core/services/onedrive/Cargo.toml +2 -2
  187. data/core/services/onedrive/LICENSE +16 -0
  188. data/core/services/opfs/Cargo.toml +1 -1
  189. data/core/services/opfs/LICENSE +16 -0
  190. data/core/services/oss/Cargo.toml +1 -1
  191. data/core/services/oss/LICENSE +16 -0
  192. data/core/services/pcloud/Cargo.toml +1 -1
  193. data/core/services/pcloud/LICENSE +16 -0
  194. data/core/services/persy/Cargo.toml +1 -1
  195. data/core/services/persy/LICENSE +16 -0
  196. data/core/services/postgresql/Cargo.toml +2 -2
  197. data/core/services/postgresql/LICENSE +16 -0
  198. data/core/services/redb/Cargo.toml +1 -1
  199. data/core/services/redb/LICENSE +16 -0
  200. data/core/services/redis/Cargo.toml +2 -2
  201. data/core/services/redis/LICENSE +16 -0
  202. data/core/services/redis/src/core.rs +1 -1
  203. data/core/services/rocksdb/Cargo.toml +1 -1
  204. data/core/services/rocksdb/LICENSE +16 -0
  205. data/core/services/s3/Cargo.toml +1 -1
  206. data/core/services/s3/LICENSE +16 -0
  207. data/core/services/seafile/Cargo.toml +4 -2
  208. data/core/services/seafile/LICENSE +16 -0
  209. data/core/services/seafile/src/backend.rs +2 -3
  210. data/core/services/seafile/src/core.rs +189 -86
  211. data/core/services/seafile/src/docs.md +2 -0
  212. data/core/services/sftp/Cargo.toml +2 -2
  213. data/core/services/sftp/LICENSE +16 -0
  214. data/core/services/sftp/src/core.rs +1 -1
  215. data/core/services/sftp/src/reader.rs +1 -1
  216. data/core/services/sled/Cargo.toml +1 -1
  217. data/core/services/sled/LICENSE +16 -0
  218. data/core/services/sqlite/Cargo.toml +2 -2
  219. data/core/services/sqlite/LICENSE +16 -0
  220. data/core/services/surrealdb/Cargo.toml +2 -2
  221. data/core/services/surrealdb/LICENSE +16 -0
  222. data/core/services/swift/Cargo.toml +1 -1
  223. data/core/services/swift/LICENSE +16 -0
  224. data/core/services/tikv/Cargo.toml +2 -2
  225. data/core/services/tikv/LICENSE +16 -0
  226. data/core/services/tos/Cargo.toml +1 -1
  227. data/core/services/tos/LICENSE +16 -0
  228. data/core/services/upyun/Cargo.toml +1 -1
  229. data/core/services/upyun/LICENSE +16 -0
  230. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  231. data/core/services/vercel-artifacts/LICENSE +16 -0
  232. data/core/services/vercel-blob/Cargo.toml +1 -1
  233. data/core/services/vercel-blob/LICENSE +16 -0
  234. data/core/services/webdav/Cargo.toml +1 -2
  235. data/core/services/webdav/LICENSE +16 -0
  236. data/core/services/webhdfs/Cargo.toml +2 -2
  237. data/core/services/webhdfs/LICENSE +16 -0
  238. data/core/services/yandex-disk/Cargo.toml +1 -1
  239. data/core/services/yandex-disk/LICENSE +16 -0
  240. data/core/src/lib.rs +1 -3
  241. data/core/testkit/Cargo.toml +4 -4
  242. data/core/testkit/LICENSE +16 -0
  243. data/core/upgrade.md +8 -0
  244. metadata +3 -2
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -21,6 +21,8 @@ Google Drive allows duplicate file or directory names under the same parent.
21
21
  When multiple entries match the same path, OpenDAL resolves the most recently
22
22
  modified match, falling back to the newer creation time if needed.
23
23
 
24
+ Concurrent directory creation through an operator and its clones reuses work for shared ancestors and allows independent paths to progress concurrently.
25
+
24
26
  ## Configuration
25
27
 
26
28
  Use [`crate::GdriveConfig`] for serializable configuration and this builder's
@@ -19,6 +19,7 @@ use std::collections::HashMap;
19
19
  use std::collections::VecDeque;
20
20
 
21
21
  use asyncband::mutex::Mutex;
22
+ use asyncband::singleflight::Group;
22
23
 
23
24
  use super::core::normalize_dir_path;
24
25
  use opendal_core::raw::*;
@@ -52,7 +53,7 @@ struct GdrivePathIndexState {
52
53
  pub struct GdrivePathIndex<Q: GdrivePathQueryer> {
53
54
  query: Q,
54
55
  state: Mutex<GdrivePathIndexState>,
55
- ensure_dir_lock: Mutex<()>,
56
+ ensure_dirs: Group<(String, String), String>,
56
57
  }
57
58
 
58
59
  impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
@@ -60,7 +61,7 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
60
61
  Self {
61
62
  query,
62
63
  state: Mutex::new(GdrivePathIndexState::default()),
63
- ensure_dir_lock: Mutex::new(()),
64
+ ensure_dirs: Group::new(),
64
65
  }
65
66
  }
66
67
 
@@ -253,8 +254,6 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
253
254
 
254
255
  /// Ensure input dir exists.
255
256
  pub async fn ensure_dir(&self, ctx: &OperationContext, path: &str) -> Result<String> {
256
- let _guard = self.ensure_dir_lock.lock().await;
257
-
258
257
  let path = Self::canonical_dir_path(path);
259
258
  if path.is_empty() {
260
259
  return self.root_id(ctx).await;
@@ -285,17 +284,28 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
285
284
  }
286
285
 
287
286
  let name = get_basename(parent);
288
- let value = match self.query.query(ctx, &parent_id, name).await {
289
- Ok(Some(value)) => value,
290
- Ok(None) => match self.query.create_dir(ctx, &parent_id, name).await {
291
- Ok(value) => value,
292
- Err(err) if !refreshed && err.kind() == ErrorKind::NotFound => {
293
- refreshed = true;
294
- self.refresh_path(parent).await;
295
- continue 'retry;
287
+ // A refreshed ancestor has a different ID and must not join work for its old ID.
288
+ let result: Result<String> = self
289
+ .ensure_dirs
290
+ .try_work((parent_id.clone(), name.to_string()), async || {
291
+ if let Some(value) = {
292
+ let state = self.state.lock().await;
293
+ Self::lookup_cached_id(&state.entries, parent)
294
+ } {
295
+ return Ok(value);
296
296
  }
297
- Err(err) => return Err(err),
298
- },
297
+
298
+ let value = match self.query.query(ctx, &parent_id, name).await? {
299
+ Some(value) => value,
300
+ None => self.query.create_dir(ctx, &parent_id, name).await?,
301
+ };
302
+ // Publish before completing the flight so a later caller uses the cache.
303
+ self.upsert_dir(parent, &value).await;
304
+ Ok(value)
305
+ })
306
+ .await;
307
+ parent_id = match result {
308
+ Ok(value) => value,
299
309
  Err(err) if !refreshed && err.kind() == ErrorKind::NotFound => {
300
310
  refreshed = true;
301
311
  self.refresh_path(parent).await;
@@ -303,9 +313,6 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
303
313
  }
304
314
  Err(err) => return Err(err),
305
315
  };
306
-
307
- self.upsert_dir(parent, &value).await;
308
- parent_id = value;
309
316
  }
310
317
 
311
318
  return Ok(parent_id);
@@ -317,11 +324,15 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
317
324
  mod tests {
318
325
  use std::collections::HashMap;
319
326
  use std::collections::HashSet;
327
+ use std::future::Future;
320
328
  use std::sync::Arc;
321
329
  use std::sync::atomic::AtomicUsize;
322
330
  use std::sync::atomic::Ordering;
331
+ use std::task::Context;
332
+ use std::task::Waker;
323
333
 
324
334
  use asyncband::mutex::Mutex;
335
+ use asyncband::semaphore::Semaphore;
325
336
 
326
337
  use super::*;
327
338
 
@@ -331,6 +342,7 @@ mod tests {
331
342
  entries: Arc<Mutex<HashMap<(String, String), String>>>,
332
343
  stale_parents: Arc<Mutex<HashSet<String>>>,
333
344
  create_count: Arc<AtomicUsize>,
345
+ create_gate: Option<Arc<Semaphore>>,
334
346
  }
335
347
 
336
348
  impl TestQuery {
@@ -340,6 +352,7 @@ mod tests {
340
352
  entries: Arc::new(Mutex::new(HashMap::new())),
341
353
  stale_parents: Arc::new(Mutex::new(HashSet::new())),
342
354
  create_count: Arc::new(AtomicUsize::new(0)),
355
+ create_gate: None,
343
356
  }
344
357
  }
345
358
 
@@ -390,6 +403,9 @@ mod tests {
390
403
  return Err(Error::new(ErrorKind::NotFound, "stale parent"));
391
404
  }
392
405
  self.create_count.fetch_add(1, Ordering::SeqCst);
406
+ if let Some(gate) = &self.create_gate {
407
+ drop(gate.acquire(1).await);
408
+ }
393
409
  let id = format!("{parent_id}:{name}");
394
410
  self.insert(parent_id, name, &id).await;
395
411
  Ok(id)
@@ -453,7 +469,7 @@ mod tests {
453
469
  }
454
470
 
455
471
  #[tokio::test]
456
- async fn test_ensure_dir_is_serialized() {
472
+ async fn test_ensure_dir_reuses_existing_directory() {
457
473
  let query = TestQuery::new();
458
474
  query.insert("remote-root", "root/", "svc-root").await;
459
475
 
@@ -469,6 +485,81 @@ mod tests {
469
485
  assert_eq!(query.create_count.load(Ordering::SeqCst), 1);
470
486
  }
471
487
 
488
+ #[tokio::test]
489
+ async fn test_ensure_dir_coalesces_shared_ancestors() {
490
+ let gate = Arc::new(Semaphore::new(0));
491
+ let mut query = TestQuery::new();
492
+ query.create_gate = Some(gate.clone());
493
+ query.insert("remote-root", "root/", "svc-root").await;
494
+ let index = GdrivePathIndex::new(query.clone());
495
+ let ctx = OperationContext::new();
496
+ let mut first = Box::pin(index.ensure_dir(&ctx, "root/shared/left"));
497
+ let mut second = Box::pin(index.ensure_dir(&ctx, "root/shared/right"));
498
+ let mut cx = Context::from_waker(Waker::noop());
499
+
500
+ assert!(first.as_mut().poll(&mut cx).is_pending());
501
+ assert!(second.as_mut().poll(&mut cx).is_pending());
502
+ assert_eq!(query.create_count.load(Ordering::SeqCst), 1);
503
+
504
+ gate.release(1);
505
+ let (first, second) = tokio::join!(first, second);
506
+ assert_eq!(first.unwrap(), "svc-root:shared/:left/");
507
+ assert_eq!(second.unwrap(), "svc-root:shared/:right/");
508
+ assert_eq!(query.create_count.load(Ordering::SeqCst), 3);
509
+ }
510
+
511
+ #[tokio::test]
512
+ async fn test_ensure_dir_unrelated_paths_progress_concurrently() {
513
+ for (first_path, second_path) in [("root/left", "root/right"), ("left/dir", "right/dir")] {
514
+ let gate = Arc::new(Semaphore::new(0));
515
+ let mut query = TestQuery::new();
516
+ query.create_gate = Some(gate.clone());
517
+ for name in ["root/", "left/", "right/"] {
518
+ query.insert("remote-root", name, name).await;
519
+ }
520
+ let index = GdrivePathIndex::new(query.clone());
521
+ let ctx = OperationContext::new();
522
+ let mut first = Box::pin(index.ensure_dir(&ctx, first_path));
523
+ let mut second = Box::pin(index.ensure_dir(&ctx, second_path));
524
+ let mut cx = Context::from_waker(Waker::noop());
525
+
526
+ assert!(first.as_mut().poll(&mut cx).is_pending());
527
+ assert!(second.as_mut().poll(&mut cx).is_pending());
528
+ assert_eq!(query.create_count.load(Ordering::SeqCst), 2);
529
+
530
+ gate.release(1);
531
+ let (first, second) = tokio::join!(first, second);
532
+ assert_ne!(first.unwrap(), second.unwrap());
533
+ }
534
+ }
535
+
536
+ #[tokio::test]
537
+ async fn test_ensure_dir_retries_after_creator_is_cancelled() {
538
+ let gate = Arc::new(Semaphore::new(0));
539
+ let mut query = TestQuery::new();
540
+ query.create_gate = Some(gate.clone());
541
+ query.insert("remote-root", "root/", "svc-root").await;
542
+ let index = GdrivePathIndex::new(query.clone());
543
+ let ctx = OperationContext::new();
544
+ let mut first = Box::pin(index.ensure_dir(&ctx, "root/dir"));
545
+ let mut second = Box::pin(index.ensure_dir(&ctx, "root/dir"));
546
+ let mut cx = Context::from_waker(Waker::noop());
547
+
548
+ assert!(first.as_mut().poll(&mut cx).is_pending());
549
+ assert!(second.as_mut().poll(&mut cx).is_pending());
550
+ drop(first);
551
+ assert!(second.as_mut().poll(&mut cx).is_pending());
552
+ assert_eq!(query.create_count.load(Ordering::SeqCst), 2);
553
+
554
+ gate.release(1);
555
+ assert_eq!(second.await.unwrap(), "svc-root:dir/");
556
+ assert_eq!(
557
+ index.ensure_dir(&ctx, "root/dir").await.unwrap(),
558
+ "svc-root:dir/"
559
+ );
560
+ assert_eq!(query.create_count.load(Ordering::SeqCst), 2);
561
+ }
562
+
472
563
  #[tokio::test]
473
564
  async fn test_ensure_dir_refreshes_stale_ancestor_chain() {
474
565
  let query = TestQuery::new();
@@ -36,8 +36,8 @@ bytes = { workspace = true }
36
36
  ghac = { version = "0.3.0", default-features = false }
37
37
  http = { workspace = true }
38
38
  log = { workspace = true }
39
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
40
- opendal-service-azblob = { path = "../azblob", version = "0.59.1", default-features = false }
39
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
40
+ opendal-service-azblob = { path = "../azblob", version = "0.59.2", default-features = false }
41
41
  prost = { version = "0.14.3", default-features = false }
42
42
  reqsign-azure-storage = { version = "3.1.1", default-features = false }
43
43
  reqsign-core = { version = "3.2.0", default-features = false }
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -36,7 +36,7 @@ base64 = { workspace = true }
36
36
  bytes = { workspace = true }
37
37
  http = { workspace = true }
38
38
  log = { workspace = true }
39
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
39
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
40
40
  serde = { workspace = true, features = ["derive"] }
41
41
  serde_json = { workspace = true }
42
42
 
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -31,16 +31,25 @@ version = { workspace = true }
31
31
  [package.metadata.docs.rs]
32
32
  all-features = true
33
33
 
34
+ [features]
35
+ # Keep the default GooseFS client graph to gRPC. Cache backends are opt-in so
36
+ # applications that only need the data plane do not pull `lru`, foyer, or
37
+ # io_uring.
38
+ default = []
39
+ metadata-cache = ["goosefs-sdk/metadata-cache"]
40
+ page-cache = ["goosefs-sdk/page-cache"]
41
+ page-cache-io-uring = ["goosefs-sdk/page-cache-io-uring"]
42
+
34
43
  [dependencies]
44
+ asyncband = { workspace = true, features = ["rwlock"] }
35
45
  bytes = { workspace = true }
36
- goosefs-sdk = "0.1.9"
46
+ goosefs-sdk = { version = "0.2.1", default-features = false }
37
47
  log = { workspace = true }
38
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
48
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
39
49
  serde = { workspace = true, features = ["derive"] }
40
- tokio = { workspace = true }
41
50
 
42
51
  [dev-dependencies]
43
- opendal = { path = "../..", version = "0.59.1", features = [
52
+ opendal = { path = "../..", version = "0.59.2", features = [
44
53
  "services-goosefs",
45
54
  ] }
46
55
  tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -15,12 +15,23 @@ cargo add opendal --features services-goosefs
15
15
  The service is available as `opendal::services::GooseFs`. Configure the
16
16
  service builder, then pass it to `opendal::Operator::new`.
17
17
 
18
+ Optional client caches stay on this crate, not on the `opendal` facade. Add
19
+ this crate as a direct dependency so Cargo unifies the features into the same
20
+ `opendal-service-goosefs` build:
21
+
22
+ ```shell
23
+ cargo add opendal-service-goosefs --features metadata-cache,page-cache
24
+ # Linux io_uring page-cache backend:
25
+ cargo add opendal-service-goosefs --features page-cache-io-uring
26
+ ```
27
+
18
28
  ## Use with `opendal-core`
19
29
 
20
30
  Add the split crates directly:
21
31
 
22
32
  ```shell
23
33
  cargo add opendal-core opendal-service-goosefs
34
+ # Optional caches: cargo add opendal-service-goosefs --features metadata-cache,page-cache
24
35
  ```
25
36
 
26
37
  Pass a configured service builder to `Operator::new`:
@@ -228,11 +228,9 @@ fn detect_master_addr_source() -> MasterAddrSource {
228
228
  return MasterAddrSource::Env;
229
229
  }
230
230
 
231
- // The SDK searches `$GOOSEFS_CONFIG_FILE`, `$GOOSEFS_HOME/conf`,
232
- // `~/.goosefs`, and `/etc/goosefs` for `goosefs-site.properties`. It
233
- // documents `$GOOSEFS_CONF_DIR` as well, but 0.1.9 looks up the Java
234
- // property name `goosefs.conf.dir` as the environment variable, so that
235
- // path never matches. An unreadable file is left to
231
+ // The SDK searches `$GOOSEFS_CONFIG_FILE`, `$GOOSEFS_CONF_DIR`,
232
+ // `$GOOSEFS_HOME/conf`, `~/.goosefs`, and `/etc/goosefs` for
233
+ // `goosefs-site.properties`. An unreadable file is left to
236
234
  // `from_properties_auto()`, which already failed the build above.
237
235
  if let Some(path) = goosefs_sdk::config::discover_config_file()
238
236
  && let Ok(content) = std::fs::read_to_string(&path)
@@ -19,12 +19,12 @@ use std::fmt::Debug;
19
19
  use std::sync::Arc;
20
20
  use std::sync::atomic::{AtomicU32, Ordering};
21
21
 
22
+ use asyncband::rwlock::RwLock;
22
23
  use goosefs_sdk::client::MasterClient;
23
24
  use goosefs_sdk::config::GoosefsConfig as ClientConfig;
24
25
  use goosefs_sdk::context::FileSystemContext;
25
26
  use goosefs_sdk::io::{GoosefsFileReader, GoosefsFileWriter};
26
27
  use goosefs_sdk::proto::grpc::file::FileInfo;
27
- use tokio::sync::RwLock;
28
28
 
29
29
  use opendal_core::raw::*;
30
30
  use opendal_core::*;
@@ -23,6 +23,8 @@ which means it connects directly to GooseFS Master (port 9200) and
23
23
  Worker (port 9203) without requiring a Proxy component.
24
24
 
25
25
  Features:
26
+ - **Read metadata**: Streams expose full-file metadata from the SDK reader before
27
+ consuming the body. Bounded reads do not issue a separate stat to obtain it.
26
28
  - **HA support**: Comma-separated master addresses for automatic Primary Master discovery.
27
29
  - **Block-level I/O**: Data reads/writes go through block-level gRPC bidirectional streaming.
28
30
  - **Consistent hash routing**: Worker selection uses consistent hashing on block IDs.
@@ -32,6 +34,9 @@ Features:
32
34
  - **Rename parent directories**: `rename` creates a missing destination parent only
33
35
  when Master reports it missing. Write-via-temp publish does not call
34
36
  `CreateDirectory` for a parent that `CreateFile(recursive)` already created.
37
+ - **Optional client caches**: `goosefs-sdk` 0.2.1 keeps metadata and page caches
38
+ behind Cargo features so the default build stays a gRPC client. Enable them
39
+ on this crate; see [Optional client caches](#optional-client-caches).
35
40
 
36
41
  ## Configuration
37
42
 
@@ -48,9 +53,8 @@ first:
48
53
  list or the SDK's `gfs://h1:9200,h2:9200/root` URI form;
49
54
  2. `goosefs.master.rpc.addresses` or `goosefs.master.hostname` in
50
55
  `goosefs-site.properties`, discovered through `$GOOSEFS_CONFIG_FILE`,
51
- `$GOOSEFS_HOME/conf`, `~/.goosefs`, and `/etc/goosefs`. `goosefs-sdk` 0.1.9
52
- documents `$GOOSEFS_CONF_DIR` as a search path but does not read it, so use
53
- `$GOOSEFS_CONFIG_FILE` to point at a file outside those directories;
56
+ `$GOOSEFS_CONF_DIR`, `$GOOSEFS_HOME/conf`, `~/.goosefs`, and `/etc/goosefs`.
57
+ Use `$GOOSEFS_CONFIG_FILE` to point at a file outside those directories;
54
58
  3. the `master_addr` config key, which also receives the URI authority of
55
59
  `goosefs://host:port/path`.
56
60
 
@@ -68,6 +72,43 @@ address; it never falls back to `127.0.0.1:9200`.
68
72
  | absent, or no master keys | unset | set | `master_addr` |
69
73
  | absent, or no master keys | unset | absent | none — `ConfigInvalid` |
70
74
 
75
+ ### Optional client caches
76
+
77
+ `goosefs-sdk` 0.2.1 compiles the client metadata cache and the disk-backed page
78
+ cache only when their Cargo features are enabled. The default
79
+ `services-goosefs` / `opendal-service-goosefs` build stays a gRPC client.
80
+
81
+ Enable a cache on this crate. Applications that use the `opendal` facade still
82
+ add `opendal-service-goosefs` as a direct dependency so Cargo unifies the
83
+ features into the same build:
84
+
85
+ | Feature | Compiles |
86
+ | --- | --- |
87
+ | `metadata-cache` | Process-local status / listing LRU |
88
+ | `page-cache` | Disk-backed page cache via `tokio::fs` |
89
+ | `page-cache-io-uring` | Page cache plus the Linux io_uring backend |
90
+
91
+ ```shell
92
+ cargo add opendal --features services-goosefs
93
+ cargo add opendal-service-goosefs --features metadata-cache,page-cache-io-uring
94
+ ```
95
+
96
+ `page-cache-io-uring` already enables `page-cache`. On non-Linux targets the
97
+ SDK still builds the portable `tokio::fs` store; io_uring is Linux-only.
98
+
99
+ Compiling a cache does not replace runtime configuration. `build()` loads
100
+ `goosefs-site.properties` and `GOOSEFS_*` environment variables through
101
+ `GoosefsConfig::from_properties_auto()`:
102
+
103
+ - **Metadata cache**: on by default once `metadata-cache` is compiled. Set
104
+ `GOOSEFS_METADATA_CACHE_ENABLED=false` to disable it.
105
+ - **Page cache**: off by default. Set `GOOSEFS_USER_CLIENT_CACHE_ENABLED=true`
106
+ and configure cache directories through
107
+ `goosefs.user.client.cache.dirs` / `GOOSEFS_USER_CLIENT_CACHE_DIRS`.
108
+
109
+ See [goosefs-sdk client configuration](https://github.com/Tencent/tencent-goosefs-rust-sdk/blob/main/docs/CLIENT_CONFIGURATION.md)
110
+ for the full cache knob list.
111
+
71
112
  ## Example
72
113
 
73
114
  ### Via Builder
@@ -16,94 +16,25 @@
16
16
  // under the License.
17
17
 
18
18
  use super::backend::*;
19
- use super::core::GoosefsCore;
20
19
  use super::core::{ErrorContext, parse_error};
21
20
  use goosefs_sdk::io::GoosefsFileReader as SdkReader;
22
21
  use opendal_core::raw::*;
23
22
  use opendal_core::*;
24
- use std::sync::Arc;
25
23
 
26
24
  pub struct GoosefsReadStream {
27
- core: Arc<GoosefsCore>,
28
- path: String,
29
- range: BytesRange,
30
- content_length: Option<u64>,
31
- /// Lazily-opened SDK reader. `None` until the first `read()` call.
32
- inner: Option<SdkReader>,
33
- /// Terminal flag: once the underlying stream has returned `None`,
34
- /// subsequent `read()` calls must keep returning an empty `Buffer`
35
- /// without re-entering the SDK (which would try to open a fresh
36
- /// stream and re-read the file).
25
+ inner: SdkReader,
26
+ // The SDK may reopen its stream after EOF; keep OpenDAL's EOF terminal.
37
27
  done: bool,
38
28
  }
39
29
 
40
- impl GoosefsReadStream {
41
- pub fn new(
42
- core: Arc<GoosefsCore>,
43
- path: String,
44
- range: BytesRange,
45
- content_length: Option<u64>,
46
- ) -> Self {
47
- GoosefsReadStream {
48
- core,
49
- path,
50
- range,
51
- content_length,
52
- inner: None,
53
- done: false,
54
- }
55
- }
56
-
57
- /// Open the underlying SDK reader, picking between the full-file
58
- /// and ranged variants based on `self.range`.
59
- async fn open(&self) -> Result<SdkReader> {
60
- let offset = self.range.offset();
61
- let size = self.range.size();
62
-
63
- // Three cases:
64
- // 1. No offset and no size → full-file stream.
65
- // 2. Offset+size both set → ranged stream.
66
- // 3. Offset set, size unbounded → resolve tail length via
67
- // get_status and fall through to the ranged opener, so we
68
- // still get per-block streaming (rather than buffering the
69
- // whole tail in one call).
70
- match (offset, size) {
71
- (0, None) => self.core.open_reader(&self.path).await,
72
- (off, Some(len)) => self.core.open_range_reader(&self.path, off, len).await,
73
- (off, None) => {
74
- let content_length = self.content_length.ok_or_else(|| {
75
- Error::new(
76
- ErrorKind::Unexpected,
77
- "content length must be known for offset reads",
78
- )
79
- })?;
80
- let len = content_length.saturating_sub(off);
81
- if len == 0 {
82
- // Empty tail — short-circuit with a zero-length
83
- // ranged open so the very next `read_next_block`
84
- // call returns `None` and we EOF cleanly.
85
- self.core.open_range_reader(&self.path, off, 0).await
86
- } else {
87
- self.core.open_range_reader(&self.path, off, len).await
88
- }
89
- }
90
- }
91
- }
92
- }
93
-
94
30
  impl oio::ReadStream for GoosefsReadStream {
95
31
  async fn read(&mut self) -> Result<Buffer> {
96
32
  if self.done {
97
33
  return Ok(Buffer::new());
98
34
  }
99
35
 
100
- // Lazy open on first call.
101
- if self.inner.is_none() {
102
- self.inner = Some(self.open().await?);
103
- }
104
- let reader = self.inner.as_mut().expect("inner was just set");
105
-
106
- match reader
36
+ match self
37
+ .inner
107
38
  .read_next_block()
108
39
  .await
109
40
  .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("ReadNextBlock")), err))?
@@ -137,25 +68,27 @@ impl oio::StreamRead for GoosefsReader {
137
68
  let backend = &self.backend;
138
69
  let path = self.path.as_str();
139
70
 
140
- let content_length = if range.offset() != 0 && range.size().is_none() {
141
- let file_info = backend.core.get_status(path).await?;
142
- Some(
143
- backend
71
+ let inner = match (range.offset(), range.size()) {
72
+ (0, None) => backend.core.open_reader(path).await?,
73
+ (offset, Some(length)) => backend.core.open_range_reader(path, offset, length).await?,
74
+ (offset, None) => {
75
+ let file_info = backend.core.get_status(path).await?;
76
+ let content_length = backend
144
77
  .core
145
78
  .file_info_to_metadata(&file_info)?
146
- .content_length(),
147
- )
148
- } else {
149
- None
79
+ .content_length();
80
+ backend
81
+ .core
82
+ .open_range_reader(path, offset, content_length.saturating_sub(offset))
83
+ .await?
84
+ }
150
85
  };
151
- let rp = RpRead::default();
152
- let stream = GoosefsReadStream::new(
153
- backend.core.clone(),
154
- path.to_string(),
155
- range,
156
- content_length,
157
- );
86
+ let metadata = backend.core.file_info_to_metadata(inner.file_info())?;
87
+ let stream = GoosefsReadStream { inner, done: false };
158
88
 
159
- Ok((rp, Box::new(stream) as Box<dyn oio::ReadStreamDyn>))
89
+ Ok((
90
+ RpRead::new(metadata),
91
+ Box::new(stream) as Box<dyn oio::ReadStreamDyn>,
92
+ ))
160
93
  }
161
94
  }
@@ -32,10 +32,10 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
- asyncband = { workspace = true }
35
+ asyncband = { workspace = true, features = ["once-cell"] }
36
36
  futures = { workspace = true }
37
37
  mongodb = "3.3.0"
38
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
38
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
39
39
  serde = { workspace = true, features = ["derive"] }
40
40
 
41
41
  [dev-dependencies]
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.