opendal 0.1.12 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +31 -36
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +25 -35
- data/README.md +1 -1
- data/core/CHANGELOG.md +4 -0
- data/core/Cargo.lock +128 -187
- data/core/Cargo.toml +95 -95
- data/core/DEPENDENCIES.rust.tsv +259 -289
- data/core/LICENSE +16 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/CHANGELOG.md +4 -0
- data/core/core/Cargo.toml +1 -1
- data/core/core/LICENSE +16 -0
- data/core/core/src/docs/upgrade.md +8 -0
- data/core/core/src/lib.rs +1 -3
- data/core/core/src/raw/futures_util.rs +26 -21
- data/core/core/src/raw/http_util/error.rs +9 -0
- data/core/core/src/raw/ops.rs +1 -0
- data/core/core/src/types/http_transport/mod.rs +30 -2
- data/core/core/src/types/http_transport/uri.rs +166 -0
- data/core/core/src/types/metadata.rs +1 -0
- data/core/core/src/types/mod.rs +2 -0
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/http-transports/reqwest/LICENSE +16 -0
- data/core/http-transports/reqwest/src/lib.rs +46 -4
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/async-backtrace/LICENSE +16 -0
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/await-tree/LICENSE +16 -0
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/capability-check/LICENSE +16 -0
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/chaos/LICENSE +16 -0
- data/core/layers/concurrent-limit/Cargo.toml +3 -3
- data/core/layers/concurrent-limit/LICENSE +16 -0
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/dtrace/LICENSE +16 -0
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastmetrics/LICENSE +16 -0
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/fastrace/LICENSE +16 -0
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/foyer/LICENSE +16 -0
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/hotpath/LICENSE +16 -0
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/immutable-index/LICENSE +16 -0
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/logging/LICENSE +16 -0
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/metrics/LICENSE +16 -0
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/mime-guess/LICENSE +16 -0
- data/core/layers/observe-metrics-common/Cargo.toml +1 -1
- data/core/layers/observe-metrics-common/LICENSE +16 -0
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/otelmetrics/LICENSE +16 -0
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/oteltrace/LICENSE +16 -0
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus/LICENSE +16 -0
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/prometheus-client/LICENSE +16 -0
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/retry/LICENSE +16 -0
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/route/LICENSE +16 -0
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/tail-cut/LICENSE +16 -0
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/throttle/LICENSE +16 -0
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/timeout/LICENSE +16 -0
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/layers/tracing/LICENSE +16 -0
- data/core/services/aliyun-drive/Cargo.toml +2 -2
- data/core/services/aliyun-drive/LICENSE +16 -0
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/alluxio/LICENSE +16 -0
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azblob/LICENSE +16 -0
- data/core/services/azblob/src/backend.rs +21 -7
- data/core/services/azblob/src/config.rs +4 -1
- data/core/services/azdls/Cargo.toml +3 -3
- data/core/services/azdls/LICENSE +16 -0
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azfile/LICENSE +16 -0
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/azure-common/LICENSE +16 -0
- data/core/services/b2/Cargo.toml +2 -2
- data/core/services/b2/LICENSE +16 -0
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cacache/LICENSE +16 -0
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/LICENSE +16 -0
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/compfs/LICENSE +16 -0
- data/core/services/compfs/src/core.rs +27 -11
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/cos/LICENSE +16 -0
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/d1/LICENSE +16 -0
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dashmap/LICENSE +16 -0
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dbfs/LICENSE +16 -0
- data/core/services/dropbox/Cargo.toml +2 -2
- data/core/services/dropbox/LICENSE +16 -0
- data/core/services/etcd/Cargo.toml +2 -2
- data/core/services/etcd/LICENSE +16 -0
- data/core/services/etcd/src/core.rs +3 -3
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foundationdb/LICENSE +16 -0
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/foyer/LICENSE +16 -0
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/fs/LICENSE +16 -0
- data/core/services/fs/src/core.rs +69 -10
- data/core/services/ftp/Cargo.toml +2 -2
- data/core/services/ftp/LICENSE +16 -0
- data/core/services/ftp/src/core.rs +1 -1
- data/core/services/ftp/src/reader.rs +1 -1
- data/core/services/ftp/src/writer.rs +1 -1
- data/core/services/gcs/Cargo.toml +3 -2
- data/core/services/gcs/LICENSE +16 -0
- data/core/services/gcs/src/composer.rs +11 -21
- data/core/services/gcs-grpc/Cargo.toml +1 -1
- data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +17 -18
- data/core/services/gdrive/Cargo.toml +3 -2
- data/core/services/gdrive/LICENSE +16 -0
- data/core/services/gdrive/src/docs.md +2 -0
- data/core/services/gdrive/src/path_index.rs +109 -18
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/ghac/LICENSE +16 -0
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/github/LICENSE +16 -0
- data/core/services/goosefs/Cargo.toml +13 -4
- data/core/services/goosefs/LICENSE +16 -0
- data/core/services/goosefs/README.md +11 -0
- data/core/services/goosefs/src/backend.rs +3 -5
- data/core/services/goosefs/src/core.rs +1 -1
- data/core/services/goosefs/src/docs.md +44 -3
- data/core/services/goosefs/src/reader.rs +22 -89
- data/core/services/gridfs/Cargo.toml +2 -2
- data/core/services/gridfs/LICENSE +16 -0
- data/core/services/hdfs/Cargo.toml +1 -1
- data/core/services/hdfs/LICENSE +16 -0
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hdfs-native/LICENSE +16 -0
- data/core/services/hf/Cargo.toml +3 -3
- data/core/services/hf/LICENSE +16 -0
- data/core/services/hf/src/backend.rs +77 -9
- data/core/services/hf/src/config.rs +58 -0
- data/core/services/hf/src/core.rs +695 -318
- data/core/services/hf/src/docs.md +29 -0
- data/core/services/hf/src/lister.rs +3 -3
- data/core/services/hf/src/reader.rs +128 -143
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/http/LICENSE +16 -0
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipfs/LICENSE +16 -0
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/ipmfs/LICENSE +16 -0
- data/core/services/koofr/Cargo.toml +2 -2
- data/core/services/koofr/LICENSE +16 -0
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/lakefs/LICENSE +16 -0
- data/core/services/memcached/Cargo.toml +2 -2
- data/core/services/memcached/LICENSE +16 -0
- data/core/services/memcached/src/core.rs +3 -3
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/mini_moka/LICENSE +16 -0
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/moka/LICENSE +16 -0
- data/core/services/mongodb/Cargo.toml +2 -2
- data/core/services/mongodb/LICENSE +16 -0
- data/core/services/monoiofs/Cargo.toml +2 -2
- data/core/services/monoiofs/LICENSE +16 -0
- data/core/services/monoiofs/src/core.rs +27 -11
- data/core/services/mysql/Cargo.toml +2 -2
- data/core/services/mysql/LICENSE +16 -0
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/obs/LICENSE +16 -0
- data/core/services/onedrive/Cargo.toml +2 -2
- data/core/services/onedrive/LICENSE +16 -0
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/opfs/LICENSE +16 -0
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/LICENSE +16 -0
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/pcloud/LICENSE +16 -0
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/persy/LICENSE +16 -0
- data/core/services/postgresql/Cargo.toml +2 -2
- data/core/services/postgresql/LICENSE +16 -0
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redb/LICENSE +16 -0
- data/core/services/redis/Cargo.toml +2 -2
- data/core/services/redis/LICENSE +16 -0
- data/core/services/redis/src/core.rs +1 -1
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/rocksdb/LICENSE +16 -0
- data/core/services/s3/Cargo.toml +1 -1
- data/core/services/s3/LICENSE +16 -0
- data/core/services/seafile/Cargo.toml +4 -2
- data/core/services/seafile/LICENSE +16 -0
- data/core/services/seafile/src/backend.rs +2 -3
- data/core/services/seafile/src/core.rs +189 -86
- data/core/services/seafile/src/docs.md +2 -0
- data/core/services/sftp/Cargo.toml +2 -2
- data/core/services/sftp/LICENSE +16 -0
- data/core/services/sftp/src/core.rs +1 -1
- data/core/services/sftp/src/reader.rs +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sled/LICENSE +16 -0
- data/core/services/sqlite/Cargo.toml +2 -2
- data/core/services/sqlite/LICENSE +16 -0
- data/core/services/surrealdb/Cargo.toml +2 -2
- data/core/services/surrealdb/LICENSE +16 -0
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/swift/LICENSE +16 -0
- data/core/services/tikv/Cargo.toml +2 -2
- data/core/services/tikv/LICENSE +16 -0
- data/core/services/tos/Cargo.toml +1 -1
- data/core/services/tos/LICENSE +16 -0
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/upyun/LICENSE +16 -0
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/LICENSE +16 -0
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/vercel-blob/LICENSE +16 -0
- data/core/services/webdav/Cargo.toml +1 -2
- data/core/services/webdav/LICENSE +16 -0
- data/core/services/webhdfs/Cargo.toml +2 -2
- data/core/services/webhdfs/LICENSE +16 -0
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/services/yandex-disk/LICENSE +16 -0
- data/core/src/lib.rs +1 -3
- data/core/testkit/Cargo.toml +4 -4
- data/core/testkit/LICENSE +16 -0
- data/core/upgrade.md +8 -0
- metadata +3 -2
|
@@ -199,3 +199,19 @@
|
|
|
199
199
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
200
|
See the License for the specific language governing permissions and
|
|
201
201
|
limitations under the License.
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
------------------------------------------------------------------------------------
|
|
205
|
+
This product bundles third-party components under other open source licenses.
|
|
206
|
+
This section summarizes those components and their licenses. See the referenced
|
|
207
|
+
license files for text of these licenses.
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
Creative Commons Zero v1.0 Universal
|
|
211
|
+
------------------------------------
|
|
212
|
+
|
|
213
|
+
bindings/dotnet/.gitignore
|
|
214
|
+
|
|
215
|
+
The above file is derived from VisualStudio.gitignore in
|
|
216
|
+
https://github.com/github/gitignore. Its license text is in
|
|
217
|
+
bindings/dotnet/LICENSE-CC0-1.0.txt.
|
|
@@ -21,6 +21,8 @@ Google Drive allows duplicate file or directory names under the same parent.
|
|
|
21
21
|
When multiple entries match the same path, OpenDAL resolves the most recently
|
|
22
22
|
modified match, falling back to the newer creation time if needed.
|
|
23
23
|
|
|
24
|
+
Concurrent directory creation through an operator and its clones reuses work for shared ancestors and allows independent paths to progress concurrently.
|
|
25
|
+
|
|
24
26
|
## Configuration
|
|
25
27
|
|
|
26
28
|
Use [`crate::GdriveConfig`] for serializable configuration and this builder's
|
|
@@ -19,6 +19,7 @@ use std::collections::HashMap;
|
|
|
19
19
|
use std::collections::VecDeque;
|
|
20
20
|
|
|
21
21
|
use asyncband::mutex::Mutex;
|
|
22
|
+
use asyncband::singleflight::Group;
|
|
22
23
|
|
|
23
24
|
use super::core::normalize_dir_path;
|
|
24
25
|
use opendal_core::raw::*;
|
|
@@ -52,7 +53,7 @@ struct GdrivePathIndexState {
|
|
|
52
53
|
pub struct GdrivePathIndex<Q: GdrivePathQueryer> {
|
|
53
54
|
query: Q,
|
|
54
55
|
state: Mutex<GdrivePathIndexState>,
|
|
55
|
-
|
|
56
|
+
ensure_dirs: Group<(String, String), String>,
|
|
56
57
|
}
|
|
57
58
|
|
|
58
59
|
impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
|
|
@@ -60,7 +61,7 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
|
|
|
60
61
|
Self {
|
|
61
62
|
query,
|
|
62
63
|
state: Mutex::new(GdrivePathIndexState::default()),
|
|
63
|
-
|
|
64
|
+
ensure_dirs: Group::new(),
|
|
64
65
|
}
|
|
65
66
|
}
|
|
66
67
|
|
|
@@ -253,8 +254,6 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
|
|
|
253
254
|
|
|
254
255
|
/// Ensure input dir exists.
|
|
255
256
|
pub async fn ensure_dir(&self, ctx: &OperationContext, path: &str) -> Result<String> {
|
|
256
|
-
let _guard = self.ensure_dir_lock.lock().await;
|
|
257
|
-
|
|
258
257
|
let path = Self::canonical_dir_path(path);
|
|
259
258
|
if path.is_empty() {
|
|
260
259
|
return self.root_id(ctx).await;
|
|
@@ -285,17 +284,28 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
|
|
|
285
284
|
}
|
|
286
285
|
|
|
287
286
|
let name = get_basename(parent);
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
287
|
+
// A refreshed ancestor has a different ID and must not join work for its old ID.
|
|
288
|
+
let result: Result<String> = self
|
|
289
|
+
.ensure_dirs
|
|
290
|
+
.try_work((parent_id.clone(), name.to_string()), async || {
|
|
291
|
+
if let Some(value) = {
|
|
292
|
+
let state = self.state.lock().await;
|
|
293
|
+
Self::lookup_cached_id(&state.entries, parent)
|
|
294
|
+
} {
|
|
295
|
+
return Ok(value);
|
|
296
296
|
}
|
|
297
|
-
|
|
298
|
-
|
|
297
|
+
|
|
298
|
+
let value = match self.query.query(ctx, &parent_id, name).await? {
|
|
299
|
+
Some(value) => value,
|
|
300
|
+
None => self.query.create_dir(ctx, &parent_id, name).await?,
|
|
301
|
+
};
|
|
302
|
+
// Publish before completing the flight so a later caller uses the cache.
|
|
303
|
+
self.upsert_dir(parent, &value).await;
|
|
304
|
+
Ok(value)
|
|
305
|
+
})
|
|
306
|
+
.await;
|
|
307
|
+
parent_id = match result {
|
|
308
|
+
Ok(value) => value,
|
|
299
309
|
Err(err) if !refreshed && err.kind() == ErrorKind::NotFound => {
|
|
300
310
|
refreshed = true;
|
|
301
311
|
self.refresh_path(parent).await;
|
|
@@ -303,9 +313,6 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
|
|
|
303
313
|
}
|
|
304
314
|
Err(err) => return Err(err),
|
|
305
315
|
};
|
|
306
|
-
|
|
307
|
-
self.upsert_dir(parent, &value).await;
|
|
308
|
-
parent_id = value;
|
|
309
316
|
}
|
|
310
317
|
|
|
311
318
|
return Ok(parent_id);
|
|
@@ -317,11 +324,15 @@ impl<Q: GdrivePathQueryer> GdrivePathIndex<Q> {
|
|
|
317
324
|
mod tests {
|
|
318
325
|
use std::collections::HashMap;
|
|
319
326
|
use std::collections::HashSet;
|
|
327
|
+
use std::future::Future;
|
|
320
328
|
use std::sync::Arc;
|
|
321
329
|
use std::sync::atomic::AtomicUsize;
|
|
322
330
|
use std::sync::atomic::Ordering;
|
|
331
|
+
use std::task::Context;
|
|
332
|
+
use std::task::Waker;
|
|
323
333
|
|
|
324
334
|
use asyncband::mutex::Mutex;
|
|
335
|
+
use asyncband::semaphore::Semaphore;
|
|
325
336
|
|
|
326
337
|
use super::*;
|
|
327
338
|
|
|
@@ -331,6 +342,7 @@ mod tests {
|
|
|
331
342
|
entries: Arc<Mutex<HashMap<(String, String), String>>>,
|
|
332
343
|
stale_parents: Arc<Mutex<HashSet<String>>>,
|
|
333
344
|
create_count: Arc<AtomicUsize>,
|
|
345
|
+
create_gate: Option<Arc<Semaphore>>,
|
|
334
346
|
}
|
|
335
347
|
|
|
336
348
|
impl TestQuery {
|
|
@@ -340,6 +352,7 @@ mod tests {
|
|
|
340
352
|
entries: Arc::new(Mutex::new(HashMap::new())),
|
|
341
353
|
stale_parents: Arc::new(Mutex::new(HashSet::new())),
|
|
342
354
|
create_count: Arc::new(AtomicUsize::new(0)),
|
|
355
|
+
create_gate: None,
|
|
343
356
|
}
|
|
344
357
|
}
|
|
345
358
|
|
|
@@ -390,6 +403,9 @@ mod tests {
|
|
|
390
403
|
return Err(Error::new(ErrorKind::NotFound, "stale parent"));
|
|
391
404
|
}
|
|
392
405
|
self.create_count.fetch_add(1, Ordering::SeqCst);
|
|
406
|
+
if let Some(gate) = &self.create_gate {
|
|
407
|
+
drop(gate.acquire(1).await);
|
|
408
|
+
}
|
|
393
409
|
let id = format!("{parent_id}:{name}");
|
|
394
410
|
self.insert(parent_id, name, &id).await;
|
|
395
411
|
Ok(id)
|
|
@@ -453,7 +469,7 @@ mod tests {
|
|
|
453
469
|
}
|
|
454
470
|
|
|
455
471
|
#[tokio::test]
|
|
456
|
-
async fn
|
|
472
|
+
async fn test_ensure_dir_reuses_existing_directory() {
|
|
457
473
|
let query = TestQuery::new();
|
|
458
474
|
query.insert("remote-root", "root/", "svc-root").await;
|
|
459
475
|
|
|
@@ -469,6 +485,81 @@ mod tests {
|
|
|
469
485
|
assert_eq!(query.create_count.load(Ordering::SeqCst), 1);
|
|
470
486
|
}
|
|
471
487
|
|
|
488
|
+
#[tokio::test]
|
|
489
|
+
async fn test_ensure_dir_coalesces_shared_ancestors() {
|
|
490
|
+
let gate = Arc::new(Semaphore::new(0));
|
|
491
|
+
let mut query = TestQuery::new();
|
|
492
|
+
query.create_gate = Some(gate.clone());
|
|
493
|
+
query.insert("remote-root", "root/", "svc-root").await;
|
|
494
|
+
let index = GdrivePathIndex::new(query.clone());
|
|
495
|
+
let ctx = OperationContext::new();
|
|
496
|
+
let mut first = Box::pin(index.ensure_dir(&ctx, "root/shared/left"));
|
|
497
|
+
let mut second = Box::pin(index.ensure_dir(&ctx, "root/shared/right"));
|
|
498
|
+
let mut cx = Context::from_waker(Waker::noop());
|
|
499
|
+
|
|
500
|
+
assert!(first.as_mut().poll(&mut cx).is_pending());
|
|
501
|
+
assert!(second.as_mut().poll(&mut cx).is_pending());
|
|
502
|
+
assert_eq!(query.create_count.load(Ordering::SeqCst), 1);
|
|
503
|
+
|
|
504
|
+
gate.release(1);
|
|
505
|
+
let (first, second) = tokio::join!(first, second);
|
|
506
|
+
assert_eq!(first.unwrap(), "svc-root:shared/:left/");
|
|
507
|
+
assert_eq!(second.unwrap(), "svc-root:shared/:right/");
|
|
508
|
+
assert_eq!(query.create_count.load(Ordering::SeqCst), 3);
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
#[tokio::test]
|
|
512
|
+
async fn test_ensure_dir_unrelated_paths_progress_concurrently() {
|
|
513
|
+
for (first_path, second_path) in [("root/left", "root/right"), ("left/dir", "right/dir")] {
|
|
514
|
+
let gate = Arc::new(Semaphore::new(0));
|
|
515
|
+
let mut query = TestQuery::new();
|
|
516
|
+
query.create_gate = Some(gate.clone());
|
|
517
|
+
for name in ["root/", "left/", "right/"] {
|
|
518
|
+
query.insert("remote-root", name, name).await;
|
|
519
|
+
}
|
|
520
|
+
let index = GdrivePathIndex::new(query.clone());
|
|
521
|
+
let ctx = OperationContext::new();
|
|
522
|
+
let mut first = Box::pin(index.ensure_dir(&ctx, first_path));
|
|
523
|
+
let mut second = Box::pin(index.ensure_dir(&ctx, second_path));
|
|
524
|
+
let mut cx = Context::from_waker(Waker::noop());
|
|
525
|
+
|
|
526
|
+
assert!(first.as_mut().poll(&mut cx).is_pending());
|
|
527
|
+
assert!(second.as_mut().poll(&mut cx).is_pending());
|
|
528
|
+
assert_eq!(query.create_count.load(Ordering::SeqCst), 2);
|
|
529
|
+
|
|
530
|
+
gate.release(1);
|
|
531
|
+
let (first, second) = tokio::join!(first, second);
|
|
532
|
+
assert_ne!(first.unwrap(), second.unwrap());
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
#[tokio::test]
|
|
537
|
+
async fn test_ensure_dir_retries_after_creator_is_cancelled() {
|
|
538
|
+
let gate = Arc::new(Semaphore::new(0));
|
|
539
|
+
let mut query = TestQuery::new();
|
|
540
|
+
query.create_gate = Some(gate.clone());
|
|
541
|
+
query.insert("remote-root", "root/", "svc-root").await;
|
|
542
|
+
let index = GdrivePathIndex::new(query.clone());
|
|
543
|
+
let ctx = OperationContext::new();
|
|
544
|
+
let mut first = Box::pin(index.ensure_dir(&ctx, "root/dir"));
|
|
545
|
+
let mut second = Box::pin(index.ensure_dir(&ctx, "root/dir"));
|
|
546
|
+
let mut cx = Context::from_waker(Waker::noop());
|
|
547
|
+
|
|
548
|
+
assert!(first.as_mut().poll(&mut cx).is_pending());
|
|
549
|
+
assert!(second.as_mut().poll(&mut cx).is_pending());
|
|
550
|
+
drop(first);
|
|
551
|
+
assert!(second.as_mut().poll(&mut cx).is_pending());
|
|
552
|
+
assert_eq!(query.create_count.load(Ordering::SeqCst), 2);
|
|
553
|
+
|
|
554
|
+
gate.release(1);
|
|
555
|
+
assert_eq!(second.await.unwrap(), "svc-root:dir/");
|
|
556
|
+
assert_eq!(
|
|
557
|
+
index.ensure_dir(&ctx, "root/dir").await.unwrap(),
|
|
558
|
+
"svc-root:dir/"
|
|
559
|
+
);
|
|
560
|
+
assert_eq!(query.create_count.load(Ordering::SeqCst), 2);
|
|
561
|
+
}
|
|
562
|
+
|
|
472
563
|
#[tokio::test]
|
|
473
564
|
async fn test_ensure_dir_refreshes_stale_ancestor_chain() {
|
|
474
565
|
let query = TestQuery::new();
|
|
@@ -36,8 +36,8 @@ bytes = { workspace = true }
|
|
|
36
36
|
ghac = { version = "0.3.0", default-features = false }
|
|
37
37
|
http = { workspace = true }
|
|
38
38
|
log = { workspace = true }
|
|
39
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
40
|
-
opendal-service-azblob = { path = "../azblob", version = "0.59.
|
|
39
|
+
opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
|
|
40
|
+
opendal-service-azblob = { path = "../azblob", version = "0.59.2", default-features = false }
|
|
41
41
|
prost = { version = "0.14.3", default-features = false }
|
|
42
42
|
reqsign-azure-storage = { version = "3.1.1", default-features = false }
|
|
43
43
|
reqsign-core = { version = "3.2.0", default-features = false }
|
data/core/services/ghac/LICENSE
CHANGED
|
@@ -199,3 +199,19 @@
|
|
|
199
199
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
200
|
See the License for the specific language governing permissions and
|
|
201
201
|
limitations under the License.
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
------------------------------------------------------------------------------------
|
|
205
|
+
This product bundles third-party components under other open source licenses.
|
|
206
|
+
This section summarizes those components and their licenses. See the referenced
|
|
207
|
+
license files for text of these licenses.
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
Creative Commons Zero v1.0 Universal
|
|
211
|
+
------------------------------------
|
|
212
|
+
|
|
213
|
+
bindings/dotnet/.gitignore
|
|
214
|
+
|
|
215
|
+
The above file is derived from VisualStudio.gitignore in
|
|
216
|
+
https://github.com/github/gitignore. Its license text is in
|
|
217
|
+
bindings/dotnet/LICENSE-CC0-1.0.txt.
|
|
@@ -36,7 +36,7 @@ base64 = { workspace = true }
|
|
|
36
36
|
bytes = { workspace = true }
|
|
37
37
|
http = { workspace = true }
|
|
38
38
|
log = { workspace = true }
|
|
39
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
39
|
+
opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
|
|
40
40
|
serde = { workspace = true, features = ["derive"] }
|
|
41
41
|
serde_json = { workspace = true }
|
|
42
42
|
|
|
@@ -199,3 +199,19 @@
|
|
|
199
199
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
200
|
See the License for the specific language governing permissions and
|
|
201
201
|
limitations under the License.
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
------------------------------------------------------------------------------------
|
|
205
|
+
This product bundles third-party components under other open source licenses.
|
|
206
|
+
This section summarizes those components and their licenses. See the referenced
|
|
207
|
+
license files for text of these licenses.
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
Creative Commons Zero v1.0 Universal
|
|
211
|
+
------------------------------------
|
|
212
|
+
|
|
213
|
+
bindings/dotnet/.gitignore
|
|
214
|
+
|
|
215
|
+
The above file is derived from VisualStudio.gitignore in
|
|
216
|
+
https://github.com/github/gitignore. Its license text is in
|
|
217
|
+
bindings/dotnet/LICENSE-CC0-1.0.txt.
|
|
@@ -31,16 +31,25 @@ version = { workspace = true }
|
|
|
31
31
|
[package.metadata.docs.rs]
|
|
32
32
|
all-features = true
|
|
33
33
|
|
|
34
|
+
[features]
|
|
35
|
+
# Keep the default GooseFS client graph to gRPC. Cache backends are opt-in so
|
|
36
|
+
# applications that only need the data plane do not pull `lru`, foyer, or
|
|
37
|
+
# io_uring.
|
|
38
|
+
default = []
|
|
39
|
+
metadata-cache = ["goosefs-sdk/metadata-cache"]
|
|
40
|
+
page-cache = ["goosefs-sdk/page-cache"]
|
|
41
|
+
page-cache-io-uring = ["goosefs-sdk/page-cache-io-uring"]
|
|
42
|
+
|
|
34
43
|
[dependencies]
|
|
44
|
+
asyncband = { workspace = true, features = ["rwlock"] }
|
|
35
45
|
bytes = { workspace = true }
|
|
36
|
-
goosefs-sdk = "0.1
|
|
46
|
+
goosefs-sdk = { version = "0.2.1", default-features = false }
|
|
37
47
|
log = { workspace = true }
|
|
38
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
48
|
+
opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
|
|
39
49
|
serde = { workspace = true, features = ["derive"] }
|
|
40
|
-
tokio = { workspace = true }
|
|
41
50
|
|
|
42
51
|
[dev-dependencies]
|
|
43
|
-
opendal = { path = "../..", version = "0.59.
|
|
52
|
+
opendal = { path = "../..", version = "0.59.2", features = [
|
|
44
53
|
"services-goosefs",
|
|
45
54
|
] }
|
|
46
55
|
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
|
@@ -199,3 +199,19 @@
|
|
|
199
199
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
200
|
See the License for the specific language governing permissions and
|
|
201
201
|
limitations under the License.
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
------------------------------------------------------------------------------------
|
|
205
|
+
This product bundles third-party components under other open source licenses.
|
|
206
|
+
This section summarizes those components and their licenses. See the referenced
|
|
207
|
+
license files for text of these licenses.
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
Creative Commons Zero v1.0 Universal
|
|
211
|
+
------------------------------------
|
|
212
|
+
|
|
213
|
+
bindings/dotnet/.gitignore
|
|
214
|
+
|
|
215
|
+
The above file is derived from VisualStudio.gitignore in
|
|
216
|
+
https://github.com/github/gitignore. Its license text is in
|
|
217
|
+
bindings/dotnet/LICENSE-CC0-1.0.txt.
|
|
@@ -15,12 +15,23 @@ cargo add opendal --features services-goosefs
|
|
|
15
15
|
The service is available as `opendal::services::GooseFs`. Configure the
|
|
16
16
|
service builder, then pass it to `opendal::Operator::new`.
|
|
17
17
|
|
|
18
|
+
Optional client caches stay on this crate, not on the `opendal` facade. Add
|
|
19
|
+
this crate as a direct dependency so Cargo unifies the features into the same
|
|
20
|
+
`opendal-service-goosefs` build:
|
|
21
|
+
|
|
22
|
+
```shell
|
|
23
|
+
cargo add opendal-service-goosefs --features metadata-cache,page-cache
|
|
24
|
+
# Linux io_uring page-cache backend:
|
|
25
|
+
cargo add opendal-service-goosefs --features page-cache-io-uring
|
|
26
|
+
```
|
|
27
|
+
|
|
18
28
|
## Use with `opendal-core`
|
|
19
29
|
|
|
20
30
|
Add the split crates directly:
|
|
21
31
|
|
|
22
32
|
```shell
|
|
23
33
|
cargo add opendal-core opendal-service-goosefs
|
|
34
|
+
# Optional caches: cargo add opendal-service-goosefs --features metadata-cache,page-cache
|
|
24
35
|
```
|
|
25
36
|
|
|
26
37
|
Pass a configured service builder to `Operator::new`:
|
|
@@ -228,11 +228,9 @@ fn detect_master_addr_source() -> MasterAddrSource {
|
|
|
228
228
|
return MasterAddrSource::Env;
|
|
229
229
|
}
|
|
230
230
|
|
|
231
|
-
// The SDK searches `$GOOSEFS_CONFIG_FILE`, `$
|
|
232
|
-
// `~/.goosefs`, and `/etc/goosefs` for
|
|
233
|
-
//
|
|
234
|
-
// property name `goosefs.conf.dir` as the environment variable, so that
|
|
235
|
-
// path never matches. An unreadable file is left to
|
|
231
|
+
// The SDK searches `$GOOSEFS_CONFIG_FILE`, `$GOOSEFS_CONF_DIR`,
|
|
232
|
+
// `$GOOSEFS_HOME/conf`, `~/.goosefs`, and `/etc/goosefs` for
|
|
233
|
+
// `goosefs-site.properties`. An unreadable file is left to
|
|
236
234
|
// `from_properties_auto()`, which already failed the build above.
|
|
237
235
|
if let Some(path) = goosefs_sdk::config::discover_config_file()
|
|
238
236
|
&& let Ok(content) = std::fs::read_to_string(&path)
|
|
@@ -19,12 +19,12 @@ use std::fmt::Debug;
|
|
|
19
19
|
use std::sync::Arc;
|
|
20
20
|
use std::sync::atomic::{AtomicU32, Ordering};
|
|
21
21
|
|
|
22
|
+
use asyncband::rwlock::RwLock;
|
|
22
23
|
use goosefs_sdk::client::MasterClient;
|
|
23
24
|
use goosefs_sdk::config::GoosefsConfig as ClientConfig;
|
|
24
25
|
use goosefs_sdk::context::FileSystemContext;
|
|
25
26
|
use goosefs_sdk::io::{GoosefsFileReader, GoosefsFileWriter};
|
|
26
27
|
use goosefs_sdk::proto::grpc::file::FileInfo;
|
|
27
|
-
use tokio::sync::RwLock;
|
|
28
28
|
|
|
29
29
|
use opendal_core::raw::*;
|
|
30
30
|
use opendal_core::*;
|
|
@@ -23,6 +23,8 @@ which means it connects directly to GooseFS Master (port 9200) and
|
|
|
23
23
|
Worker (port 9203) without requiring a Proxy component.
|
|
24
24
|
|
|
25
25
|
Features:
|
|
26
|
+
- **Read metadata**: Streams expose full-file metadata from the SDK reader before
|
|
27
|
+
consuming the body. Bounded reads do not issue a separate stat to obtain it.
|
|
26
28
|
- **HA support**: Comma-separated master addresses for automatic Primary Master discovery.
|
|
27
29
|
- **Block-level I/O**: Data reads/writes go through block-level gRPC bidirectional streaming.
|
|
28
30
|
- **Consistent hash routing**: Worker selection uses consistent hashing on block IDs.
|
|
@@ -32,6 +34,9 @@ Features:
|
|
|
32
34
|
- **Rename parent directories**: `rename` creates a missing destination parent only
|
|
33
35
|
when Master reports it missing. Write-via-temp publish does not call
|
|
34
36
|
`CreateDirectory` for a parent that `CreateFile(recursive)` already created.
|
|
37
|
+
- **Optional client caches**: `goosefs-sdk` 0.2.1 keeps metadata and page caches
|
|
38
|
+
behind Cargo features so the default build stays a gRPC client. Enable them
|
|
39
|
+
on this crate; see [Optional client caches](#optional-client-caches).
|
|
35
40
|
|
|
36
41
|
## Configuration
|
|
37
42
|
|
|
@@ -48,9 +53,8 @@ first:
|
|
|
48
53
|
list or the SDK's `gfs://h1:9200,h2:9200/root` URI form;
|
|
49
54
|
2. `goosefs.master.rpc.addresses` or `goosefs.master.hostname` in
|
|
50
55
|
`goosefs-site.properties`, discovered through `$GOOSEFS_CONFIG_FILE`,
|
|
51
|
-
`$GOOSEFS_HOME/conf`, `~/.goosefs`, and `/etc/goosefs`.
|
|
52
|
-
|
|
53
|
-
`$GOOSEFS_CONFIG_FILE` to point at a file outside those directories;
|
|
56
|
+
`$GOOSEFS_CONF_DIR`, `$GOOSEFS_HOME/conf`, `~/.goosefs`, and `/etc/goosefs`.
|
|
57
|
+
Use `$GOOSEFS_CONFIG_FILE` to point at a file outside those directories;
|
|
54
58
|
3. the `master_addr` config key, which also receives the URI authority of
|
|
55
59
|
`goosefs://host:port/path`.
|
|
56
60
|
|
|
@@ -68,6 +72,43 @@ address; it never falls back to `127.0.0.1:9200`.
|
|
|
68
72
|
| absent, or no master keys | unset | set | `master_addr` |
|
|
69
73
|
| absent, or no master keys | unset | absent | none — `ConfigInvalid` |
|
|
70
74
|
|
|
75
|
+
### Optional client caches
|
|
76
|
+
|
|
77
|
+
`goosefs-sdk` 0.2.1 compiles the client metadata cache and the disk-backed page
|
|
78
|
+
cache only when their Cargo features are enabled. The default
|
|
79
|
+
`services-goosefs` / `opendal-service-goosefs` build stays a gRPC client.
|
|
80
|
+
|
|
81
|
+
Enable a cache on this crate. Applications that use the `opendal` facade still
|
|
82
|
+
add `opendal-service-goosefs` as a direct dependency so Cargo unifies the
|
|
83
|
+
features into the same build:
|
|
84
|
+
|
|
85
|
+
| Feature | Compiles |
|
|
86
|
+
| --- | --- |
|
|
87
|
+
| `metadata-cache` | Process-local status / listing LRU |
|
|
88
|
+
| `page-cache` | Disk-backed page cache via `tokio::fs` |
|
|
89
|
+
| `page-cache-io-uring` | Page cache plus the Linux io_uring backend |
|
|
90
|
+
|
|
91
|
+
```shell
|
|
92
|
+
cargo add opendal --features services-goosefs
|
|
93
|
+
cargo add opendal-service-goosefs --features metadata-cache,page-cache-io-uring
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
`page-cache-io-uring` already enables `page-cache`. On non-Linux targets the
|
|
97
|
+
SDK still builds the portable `tokio::fs` store; io_uring is Linux-only.
|
|
98
|
+
|
|
99
|
+
Compiling a cache does not replace runtime configuration. `build()` loads
|
|
100
|
+
`goosefs-site.properties` and `GOOSEFS_*` environment variables through
|
|
101
|
+
`GoosefsConfig::from_properties_auto()`:
|
|
102
|
+
|
|
103
|
+
- **Metadata cache**: on by default once `metadata-cache` is compiled. Set
|
|
104
|
+
`GOOSEFS_METADATA_CACHE_ENABLED=false` to disable it.
|
|
105
|
+
- **Page cache**: off by default. Set `GOOSEFS_USER_CLIENT_CACHE_ENABLED=true`
|
|
106
|
+
and configure cache directories through
|
|
107
|
+
`goosefs.user.client.cache.dirs` / `GOOSEFS_USER_CLIENT_CACHE_DIRS`.
|
|
108
|
+
|
|
109
|
+
See [goosefs-sdk client configuration](https://github.com/Tencent/tencent-goosefs-rust-sdk/blob/main/docs/CLIENT_CONFIGURATION.md)
|
|
110
|
+
for the full cache knob list.
|
|
111
|
+
|
|
71
112
|
## Example
|
|
72
113
|
|
|
73
114
|
### Via Builder
|
|
@@ -16,94 +16,25 @@
|
|
|
16
16
|
// under the License.
|
|
17
17
|
|
|
18
18
|
use super::backend::*;
|
|
19
|
-
use super::core::GoosefsCore;
|
|
20
19
|
use super::core::{ErrorContext, parse_error};
|
|
21
20
|
use goosefs_sdk::io::GoosefsFileReader as SdkReader;
|
|
22
21
|
use opendal_core::raw::*;
|
|
23
22
|
use opendal_core::*;
|
|
24
|
-
use std::sync::Arc;
|
|
25
23
|
|
|
26
24
|
pub struct GoosefsReadStream {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
range: BytesRange,
|
|
30
|
-
content_length: Option<u64>,
|
|
31
|
-
/// Lazily-opened SDK reader. `None` until the first `read()` call.
|
|
32
|
-
inner: Option<SdkReader>,
|
|
33
|
-
/// Terminal flag: once the underlying stream has returned `None`,
|
|
34
|
-
/// subsequent `read()` calls must keep returning an empty `Buffer`
|
|
35
|
-
/// without re-entering the SDK (which would try to open a fresh
|
|
36
|
-
/// stream and re-read the file).
|
|
25
|
+
inner: SdkReader,
|
|
26
|
+
// The SDK may reopen its stream after EOF; keep OpenDAL's EOF terminal.
|
|
37
27
|
done: bool,
|
|
38
28
|
}
|
|
39
29
|
|
|
40
|
-
impl GoosefsReadStream {
|
|
41
|
-
pub fn new(
|
|
42
|
-
core: Arc<GoosefsCore>,
|
|
43
|
-
path: String,
|
|
44
|
-
range: BytesRange,
|
|
45
|
-
content_length: Option<u64>,
|
|
46
|
-
) -> Self {
|
|
47
|
-
GoosefsReadStream {
|
|
48
|
-
core,
|
|
49
|
-
path,
|
|
50
|
-
range,
|
|
51
|
-
content_length,
|
|
52
|
-
inner: None,
|
|
53
|
-
done: false,
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/// Open the underlying SDK reader, picking between the full-file
|
|
58
|
-
/// and ranged variants based on `self.range`.
|
|
59
|
-
async fn open(&self) -> Result<SdkReader> {
|
|
60
|
-
let offset = self.range.offset();
|
|
61
|
-
let size = self.range.size();
|
|
62
|
-
|
|
63
|
-
// Three cases:
|
|
64
|
-
// 1. No offset and no size → full-file stream.
|
|
65
|
-
// 2. Offset+size both set → ranged stream.
|
|
66
|
-
// 3. Offset set, size unbounded → resolve tail length via
|
|
67
|
-
// get_status and fall through to the ranged opener, so we
|
|
68
|
-
// still get per-block streaming (rather than buffering the
|
|
69
|
-
// whole tail in one call).
|
|
70
|
-
match (offset, size) {
|
|
71
|
-
(0, None) => self.core.open_reader(&self.path).await,
|
|
72
|
-
(off, Some(len)) => self.core.open_range_reader(&self.path, off, len).await,
|
|
73
|
-
(off, None) => {
|
|
74
|
-
let content_length = self.content_length.ok_or_else(|| {
|
|
75
|
-
Error::new(
|
|
76
|
-
ErrorKind::Unexpected,
|
|
77
|
-
"content length must be known for offset reads",
|
|
78
|
-
)
|
|
79
|
-
})?;
|
|
80
|
-
let len = content_length.saturating_sub(off);
|
|
81
|
-
if len == 0 {
|
|
82
|
-
// Empty tail — short-circuit with a zero-length
|
|
83
|
-
// ranged open so the very next `read_next_block`
|
|
84
|
-
// call returns `None` and we EOF cleanly.
|
|
85
|
-
self.core.open_range_reader(&self.path, off, 0).await
|
|
86
|
-
} else {
|
|
87
|
-
self.core.open_range_reader(&self.path, off, len).await
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
30
|
impl oio::ReadStream for GoosefsReadStream {
|
|
95
31
|
async fn read(&mut self) -> Result<Buffer> {
|
|
96
32
|
if self.done {
|
|
97
33
|
return Ok(Buffer::new());
|
|
98
34
|
}
|
|
99
35
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
self.inner = Some(self.open().await?);
|
|
103
|
-
}
|
|
104
|
-
let reader = self.inner.as_mut().expect("inner was just set");
|
|
105
|
-
|
|
106
|
-
match reader
|
|
36
|
+
match self
|
|
37
|
+
.inner
|
|
107
38
|
.read_next_block()
|
|
108
39
|
.await
|
|
109
40
|
.map_err(|err| parse_error(ErrorContext::new(ServiceOperation("ReadNextBlock")), err))?
|
|
@@ -137,25 +68,27 @@ impl oio::StreamRead for GoosefsReader {
|
|
|
137
68
|
let backend = &self.backend;
|
|
138
69
|
let path = self.path.as_str();
|
|
139
70
|
|
|
140
|
-
let
|
|
141
|
-
|
|
142
|
-
Some(
|
|
143
|
-
|
|
71
|
+
let inner = match (range.offset(), range.size()) {
|
|
72
|
+
(0, None) => backend.core.open_reader(path).await?,
|
|
73
|
+
(offset, Some(length)) => backend.core.open_range_reader(path, offset, length).await?,
|
|
74
|
+
(offset, None) => {
|
|
75
|
+
let file_info = backend.core.get_status(path).await?;
|
|
76
|
+
let content_length = backend
|
|
144
77
|
.core
|
|
145
78
|
.file_info_to_metadata(&file_info)?
|
|
146
|
-
.content_length()
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
79
|
+
.content_length();
|
|
80
|
+
backend
|
|
81
|
+
.core
|
|
82
|
+
.open_range_reader(path, offset, content_length.saturating_sub(offset))
|
|
83
|
+
.await?
|
|
84
|
+
}
|
|
150
85
|
};
|
|
151
|
-
let
|
|
152
|
-
let stream = GoosefsReadStream
|
|
153
|
-
backend.core.clone(),
|
|
154
|
-
path.to_string(),
|
|
155
|
-
range,
|
|
156
|
-
content_length,
|
|
157
|
-
);
|
|
86
|
+
let metadata = backend.core.file_info_to_metadata(inner.file_info())?;
|
|
87
|
+
let stream = GoosefsReadStream { inner, done: false };
|
|
158
88
|
|
|
159
|
-
Ok((
|
|
89
|
+
Ok((
|
|
90
|
+
RpRead::new(metadata),
|
|
91
|
+
Box::new(stream) as Box<dyn oio::ReadStreamDyn>,
|
|
92
|
+
))
|
|
160
93
|
}
|
|
161
94
|
}
|
|
@@ -32,10 +32,10 @@ version = { workspace = true }
|
|
|
32
32
|
all-features = true
|
|
33
33
|
|
|
34
34
|
[dependencies]
|
|
35
|
-
asyncband = { workspace = true }
|
|
35
|
+
asyncband = { workspace = true, features = ["once-cell"] }
|
|
36
36
|
futures = { workspace = true }
|
|
37
37
|
mongodb = "3.3.0"
|
|
38
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
38
|
+
opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
|
|
39
39
|
serde = { workspace = true, features = ["derive"] }
|
|
40
40
|
|
|
41
41
|
[dev-dependencies]
|
|
@@ -199,3 +199,19 @@
|
|
|
199
199
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
200
|
See the License for the specific language governing permissions and
|
|
201
201
|
limitations under the License.
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
------------------------------------------------------------------------------------
|
|
205
|
+
This product bundles third-party components under other open source licenses.
|
|
206
|
+
This section summarizes those components and their licenses. See the referenced
|
|
207
|
+
license files for text of these licenses.
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
Creative Commons Zero v1.0 Universal
|
|
211
|
+
------------------------------------
|
|
212
|
+
|
|
213
|
+
bindings/dotnet/.gitignore
|
|
214
|
+
|
|
215
|
+
The above file is derived from VisualStudio.gitignore in
|
|
216
|
+
https://github.com/github/gitignore. Its license text is in
|
|
217
|
+
bindings/dotnet/LICENSE-CC0-1.0.txt.
|