gitx.do 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -353
- package/dist/do/logger.d.ts +50 -0
- package/dist/do/logger.d.ts.map +1 -0
- package/dist/do/logger.js +122 -0
- package/dist/do/logger.js.map +1 -0
- package/dist/{durable-object → do}/schema.d.ts +3 -3
- package/dist/do/schema.d.ts.map +1 -0
- package/dist/{durable-object → do}/schema.js +4 -3
- package/dist/do/schema.js.map +1 -0
- package/dist/do/types.d.ts +267 -0
- package/dist/do/types.d.ts.map +1 -0
- package/dist/do/types.js +62 -0
- package/dist/do/types.js.map +1 -0
- package/dist/index.d.ts +15 -415
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +31 -483
- package/dist/index.js.map +1 -1
- package/package.json +13 -21
- package/dist/cli/commands/add.d.ts +0 -174
- package/dist/cli/commands/add.d.ts.map +0 -1
- package/dist/cli/commands/add.js +0 -131
- package/dist/cli/commands/add.js.map +0 -1
- package/dist/cli/commands/blame.d.ts +0 -259
- package/dist/cli/commands/blame.d.ts.map +0 -1
- package/dist/cli/commands/blame.js +0 -609
- package/dist/cli/commands/blame.js.map +0 -1
- package/dist/cli/commands/branch.d.ts +0 -249
- package/dist/cli/commands/branch.d.ts.map +0 -1
- package/dist/cli/commands/branch.js +0 -693
- package/dist/cli/commands/branch.js.map +0 -1
- package/dist/cli/commands/commit.d.ts +0 -182
- package/dist/cli/commands/commit.d.ts.map +0 -1
- package/dist/cli/commands/commit.js +0 -437
- package/dist/cli/commands/commit.js.map +0 -1
- package/dist/cli/commands/diff.d.ts +0 -464
- package/dist/cli/commands/diff.d.ts.map +0 -1
- package/dist/cli/commands/diff.js +0 -958
- package/dist/cli/commands/diff.js.map +0 -1
- package/dist/cli/commands/log.d.ts +0 -239
- package/dist/cli/commands/log.d.ts.map +0 -1
- package/dist/cli/commands/log.js +0 -535
- package/dist/cli/commands/log.js.map +0 -1
- package/dist/cli/commands/merge.d.ts +0 -106
- package/dist/cli/commands/merge.d.ts.map +0 -1
- package/dist/cli/commands/merge.js +0 -55
- package/dist/cli/commands/merge.js.map +0 -1
- package/dist/cli/commands/review.d.ts +0 -457
- package/dist/cli/commands/review.d.ts.map +0 -1
- package/dist/cli/commands/review.js +0 -533
- package/dist/cli/commands/review.js.map +0 -1
- package/dist/cli/commands/status.d.ts +0 -269
- package/dist/cli/commands/status.d.ts.map +0 -1
- package/dist/cli/commands/status.js +0 -493
- package/dist/cli/commands/status.js.map +0 -1
- package/dist/cli/commands/web.d.ts +0 -199
- package/dist/cli/commands/web.d.ts.map +0 -1
- package/dist/cli/commands/web.js +0 -696
- package/dist/cli/commands/web.js.map +0 -1
- package/dist/cli/fs-adapter.d.ts +0 -656
- package/dist/cli/fs-adapter.d.ts.map +0 -1
- package/dist/cli/fs-adapter.js +0 -1179
- package/dist/cli/fs-adapter.js.map +0 -1
- package/dist/cli/fsx-cli-adapter.d.ts +0 -359
- package/dist/cli/fsx-cli-adapter.d.ts.map +0 -1
- package/dist/cli/fsx-cli-adapter.js +0 -619
- package/dist/cli/fsx-cli-adapter.js.map +0 -1
- package/dist/cli/index.d.ts +0 -387
- package/dist/cli/index.d.ts.map +0 -1
- package/dist/cli/index.js +0 -523
- package/dist/cli/index.js.map +0 -1
- package/dist/cli/ui/components/DiffView.d.ts +0 -7
- package/dist/cli/ui/components/DiffView.d.ts.map +0 -1
- package/dist/cli/ui/components/DiffView.js +0 -11
- package/dist/cli/ui/components/DiffView.js.map +0 -1
- package/dist/cli/ui/components/ErrorDisplay.d.ts +0 -6
- package/dist/cli/ui/components/ErrorDisplay.d.ts.map +0 -1
- package/dist/cli/ui/components/ErrorDisplay.js +0 -11
- package/dist/cli/ui/components/ErrorDisplay.js.map +0 -1
- package/dist/cli/ui/components/FuzzySearch.d.ts +0 -9
- package/dist/cli/ui/components/FuzzySearch.d.ts.map +0 -1
- package/dist/cli/ui/components/FuzzySearch.js +0 -12
- package/dist/cli/ui/components/FuzzySearch.js.map +0 -1
- package/dist/cli/ui/components/LoadingSpinner.d.ts +0 -6
- package/dist/cli/ui/components/LoadingSpinner.d.ts.map +0 -1
- package/dist/cli/ui/components/LoadingSpinner.js +0 -10
- package/dist/cli/ui/components/LoadingSpinner.js.map +0 -1
- package/dist/cli/ui/components/NavigationList.d.ts +0 -9
- package/dist/cli/ui/components/NavigationList.d.ts.map +0 -1
- package/dist/cli/ui/components/NavigationList.js +0 -11
- package/dist/cli/ui/components/NavigationList.js.map +0 -1
- package/dist/cli/ui/components/ScrollableContent.d.ts +0 -8
- package/dist/cli/ui/components/ScrollableContent.d.ts.map +0 -1
- package/dist/cli/ui/components/ScrollableContent.js +0 -11
- package/dist/cli/ui/components/ScrollableContent.js.map +0 -1
- package/dist/cli/ui/components/index.d.ts +0 -7
- package/dist/cli/ui/components/index.d.ts.map +0 -1
- package/dist/cli/ui/components/index.js +0 -9
- package/dist/cli/ui/components/index.js.map +0 -1
- package/dist/cli/ui/terminal-ui.d.ts +0 -52
- package/dist/cli/ui/terminal-ui.d.ts.map +0 -1
- package/dist/cli/ui/terminal-ui.js +0 -121
- package/dist/cli/ui/terminal-ui.js.map +0 -1
- package/dist/do/BashModule.d.ts +0 -871
- package/dist/do/BashModule.d.ts.map +0 -1
- package/dist/do/BashModule.js +0 -1143
- package/dist/do/BashModule.js.map +0 -1
- package/dist/do/FsModule.d.ts +0 -601
- package/dist/do/FsModule.d.ts.map +0 -1
- package/dist/do/FsModule.js +0 -1120
- package/dist/do/FsModule.js.map +0 -1
- package/dist/do/GitModule.d.ts +0 -635
- package/dist/do/GitModule.d.ts.map +0 -1
- package/dist/do/GitModule.js +0 -781
- package/dist/do/GitModule.js.map +0 -1
- package/dist/do/GitRepoDO.d.ts +0 -281
- package/dist/do/GitRepoDO.d.ts.map +0 -1
- package/dist/do/GitRepoDO.js +0 -479
- package/dist/do/GitRepoDO.js.map +0 -1
- package/dist/do/bash-ast.d.ts +0 -246
- package/dist/do/bash-ast.d.ts.map +0 -1
- package/dist/do/bash-ast.js +0 -888
- package/dist/do/bash-ast.js.map +0 -1
- package/dist/do/container-executor.d.ts +0 -491
- package/dist/do/container-executor.d.ts.map +0 -1
- package/dist/do/container-executor.js +0 -730
- package/dist/do/container-executor.js.map +0 -1
- package/dist/do/index.d.ts +0 -53
- package/dist/do/index.d.ts.map +0 -1
- package/dist/do/index.js +0 -91
- package/dist/do/index.js.map +0 -1
- package/dist/do/tiered-storage.d.ts +0 -403
- package/dist/do/tiered-storage.d.ts.map +0 -1
- package/dist/do/tiered-storage.js +0 -689
- package/dist/do/tiered-storage.js.map +0 -1
- package/dist/do/withBash.d.ts +0 -231
- package/dist/do/withBash.d.ts.map +0 -1
- package/dist/do/withBash.js +0 -244
- package/dist/do/withBash.js.map +0 -1
- package/dist/do/withFs.d.ts +0 -237
- package/dist/do/withFs.d.ts.map +0 -1
- package/dist/do/withFs.js +0 -387
- package/dist/do/withFs.js.map +0 -1
- package/dist/do/withGit.d.ts +0 -180
- package/dist/do/withGit.d.ts.map +0 -1
- package/dist/do/withGit.js +0 -271
- package/dist/do/withGit.js.map +0 -1
- package/dist/durable-object/object-store.d.ts +0 -633
- package/dist/durable-object/object-store.d.ts.map +0 -1
- package/dist/durable-object/object-store.js +0 -1161
- package/dist/durable-object/object-store.js.map +0 -1
- package/dist/durable-object/schema.d.ts.map +0 -1
- package/dist/durable-object/schema.js.map +0 -1
- package/dist/durable-object/wal.d.ts +0 -416
- package/dist/durable-object/wal.d.ts.map +0 -1
- package/dist/durable-object/wal.js +0 -445
- package/dist/durable-object/wal.js.map +0 -1
- package/dist/mcp/adapter.d.ts +0 -772
- package/dist/mcp/adapter.d.ts.map +0 -1
- package/dist/mcp/adapter.js +0 -895
- package/dist/mcp/adapter.js.map +0 -1
- package/dist/mcp/sandbox/miniflare-evaluator.d.ts +0 -22
- package/dist/mcp/sandbox/miniflare-evaluator.d.ts.map +0 -1
- package/dist/mcp/sandbox/miniflare-evaluator.js +0 -140
- package/dist/mcp/sandbox/miniflare-evaluator.js.map +0 -1
- package/dist/mcp/sandbox/object-store-proxy.d.ts +0 -32
- package/dist/mcp/sandbox/object-store-proxy.d.ts.map +0 -1
- package/dist/mcp/sandbox/object-store-proxy.js +0 -30
- package/dist/mcp/sandbox/object-store-proxy.js.map +0 -1
- package/dist/mcp/sandbox/template.d.ts +0 -17
- package/dist/mcp/sandbox/template.d.ts.map +0 -1
- package/dist/mcp/sandbox/template.js +0 -71
- package/dist/mcp/sandbox/template.js.map +0 -1
- package/dist/mcp/sandbox.d.ts +0 -764
- package/dist/mcp/sandbox.d.ts.map +0 -1
- package/dist/mcp/sandbox.js +0 -1362
- package/dist/mcp/sandbox.js.map +0 -1
- package/dist/mcp/sdk-adapter.d.ts +0 -835
- package/dist/mcp/sdk-adapter.d.ts.map +0 -1
- package/dist/mcp/sdk-adapter.js +0 -974
- package/dist/mcp/sdk-adapter.js.map +0 -1
- package/dist/mcp/tools/do.d.ts +0 -32
- package/dist/mcp/tools/do.d.ts.map +0 -1
- package/dist/mcp/tools/do.js +0 -115
- package/dist/mcp/tools/do.js.map +0 -1
- package/dist/mcp/tools.d.ts +0 -548
- package/dist/mcp/tools.d.ts.map +0 -1
- package/dist/mcp/tools.js +0 -1934
- package/dist/mcp/tools.js.map +0 -1
- package/dist/ops/blame.d.ts +0 -551
- package/dist/ops/blame.d.ts.map +0 -1
- package/dist/ops/blame.js +0 -1037
- package/dist/ops/blame.js.map +0 -1
- package/dist/ops/branch.d.ts +0 -766
- package/dist/ops/branch.d.ts.map +0 -1
- package/dist/ops/branch.js +0 -950
- package/dist/ops/branch.js.map +0 -1
- package/dist/ops/commit-traversal.d.ts +0 -349
- package/dist/ops/commit-traversal.d.ts.map +0 -1
- package/dist/ops/commit-traversal.js +0 -821
- package/dist/ops/commit-traversal.js.map +0 -1
- package/dist/ops/commit.d.ts +0 -555
- package/dist/ops/commit.d.ts.map +0 -1
- package/dist/ops/commit.js +0 -826
- package/dist/ops/commit.js.map +0 -1
- package/dist/ops/merge-base.d.ts +0 -397
- package/dist/ops/merge-base.d.ts.map +0 -1
- package/dist/ops/merge-base.js +0 -691
- package/dist/ops/merge-base.js.map +0 -1
- package/dist/ops/merge.d.ts +0 -855
- package/dist/ops/merge.d.ts.map +0 -1
- package/dist/ops/merge.js +0 -1551
- package/dist/ops/merge.js.map +0 -1
- package/dist/ops/tag.d.ts +0 -247
- package/dist/ops/tag.d.ts.map +0 -1
- package/dist/ops/tag.js +0 -649
- package/dist/ops/tag.js.map +0 -1
- package/dist/ops/tree-builder.d.ts +0 -178
- package/dist/ops/tree-builder.d.ts.map +0 -1
- package/dist/ops/tree-builder.js +0 -271
- package/dist/ops/tree-builder.js.map +0 -1
- package/dist/ops/tree-diff.d.ts +0 -291
- package/dist/ops/tree-diff.d.ts.map +0 -1
- package/dist/ops/tree-diff.js +0 -705
- package/dist/ops/tree-diff.js.map +0 -1
- package/dist/pack/delta.d.ts +0 -248
- package/dist/pack/delta.d.ts.map +0 -1
- package/dist/pack/delta.js +0 -736
- package/dist/pack/delta.js.map +0 -1
- package/dist/pack/format.d.ts +0 -446
- package/dist/pack/format.d.ts.map +0 -1
- package/dist/pack/format.js +0 -572
- package/dist/pack/format.js.map +0 -1
- package/dist/pack/full-generation.d.ts +0 -612
- package/dist/pack/full-generation.d.ts.map +0 -1
- package/dist/pack/full-generation.js +0 -1378
- package/dist/pack/full-generation.js.map +0 -1
- package/dist/pack/generation.d.ts +0 -441
- package/dist/pack/generation.d.ts.map +0 -1
- package/dist/pack/generation.js +0 -707
- package/dist/pack/generation.js.map +0 -1
- package/dist/pack/index.d.ts +0 -502
- package/dist/pack/index.d.ts.map +0 -1
- package/dist/pack/index.js +0 -833
- package/dist/pack/index.js.map +0 -1
- package/dist/refs/branch.d.ts +0 -668
- package/dist/refs/branch.d.ts.map +0 -1
- package/dist/refs/branch.js +0 -897
- package/dist/refs/branch.js.map +0 -1
- package/dist/refs/storage.d.ts +0 -833
- package/dist/refs/storage.d.ts.map +0 -1
- package/dist/refs/storage.js +0 -1023
- package/dist/refs/storage.js.map +0 -1
- package/dist/refs/tag.d.ts +0 -860
- package/dist/refs/tag.d.ts.map +0 -1
- package/dist/refs/tag.js +0 -996
- package/dist/refs/tag.js.map +0 -1
- package/dist/storage/backend.d.ts +0 -425
- package/dist/storage/backend.d.ts.map +0 -1
- package/dist/storage/backend.js +0 -41
- package/dist/storage/backend.js.map +0 -1
- package/dist/storage/fsx-adapter.d.ts +0 -204
- package/dist/storage/fsx-adapter.d.ts.map +0 -1
- package/dist/storage/fsx-adapter.js +0 -470
- package/dist/storage/fsx-adapter.js.map +0 -1
- package/dist/storage/lru-cache.d.ts +0 -691
- package/dist/storage/lru-cache.d.ts.map +0 -1
- package/dist/storage/lru-cache.js +0 -813
- package/dist/storage/lru-cache.js.map +0 -1
- package/dist/storage/object-index.d.ts +0 -585
- package/dist/storage/object-index.d.ts.map +0 -1
- package/dist/storage/object-index.js +0 -532
- package/dist/storage/object-index.js.map +0 -1
- package/dist/storage/r2-pack.d.ts +0 -1257
- package/dist/storage/r2-pack.d.ts.map +0 -1
- package/dist/storage/r2-pack.js +0 -1770
- package/dist/storage/r2-pack.js.map +0 -1
- package/dist/tiered/cdc-pipeline.d.ts +0 -1888
- package/dist/tiered/cdc-pipeline.d.ts.map +0 -1
- package/dist/tiered/cdc-pipeline.js +0 -1880
- package/dist/tiered/cdc-pipeline.js.map +0 -1
- package/dist/tiered/migration.d.ts +0 -1104
- package/dist/tiered/migration.d.ts.map +0 -1
- package/dist/tiered/migration.js +0 -1214
- package/dist/tiered/migration.js.map +0 -1
- package/dist/tiered/parquet-writer.d.ts +0 -1145
- package/dist/tiered/parquet-writer.d.ts.map +0 -1
- package/dist/tiered/parquet-writer.js +0 -1183
- package/dist/tiered/parquet-writer.js.map +0 -1
- package/dist/tiered/read-path.d.ts +0 -835
- package/dist/tiered/read-path.d.ts.map +0 -1
- package/dist/tiered/read-path.js +0 -487
- package/dist/tiered/read-path.js.map +0 -1
- package/dist/types/capability.d.ts +0 -1385
- package/dist/types/capability.d.ts.map +0 -1
- package/dist/types/capability.js +0 -36
- package/dist/types/capability.js.map +0 -1
- package/dist/types/index.d.ts +0 -13
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -18
- package/dist/types/index.js.map +0 -1
- package/dist/types/objects.d.ts +0 -692
- package/dist/types/objects.d.ts.map +0 -1
- package/dist/types/objects.js +0 -837
- package/dist/types/objects.js.map +0 -1
- package/dist/types/storage.d.ts +0 -603
- package/dist/types/storage.d.ts.map +0 -1
- package/dist/types/storage.js +0 -191
- package/dist/types/storage.js.map +0 -1
- package/dist/types/worker-loader.d.ts +0 -60
- package/dist/types/worker-loader.d.ts.map +0 -1
- package/dist/types/worker-loader.js +0 -62
- package/dist/types/worker-loader.js.map +0 -1
- package/dist/utils/hash.d.ts +0 -197
- package/dist/utils/hash.d.ts.map +0 -1
- package/dist/utils/hash.js +0 -268
- package/dist/utils/hash.js.map +0 -1
- package/dist/utils/sha1.d.ts +0 -290
- package/dist/utils/sha1.d.ts.map +0 -1
- package/dist/utils/sha1.js +0 -582
- package/dist/utils/sha1.js.map +0 -1
- package/dist/wire/capabilities.d.ts +0 -1044
- package/dist/wire/capabilities.d.ts.map +0 -1
- package/dist/wire/capabilities.js +0 -941
- package/dist/wire/capabilities.js.map +0 -1
- package/dist/wire/path-security.d.ts +0 -157
- package/dist/wire/path-security.d.ts.map +0 -1
- package/dist/wire/path-security.js +0 -307
- package/dist/wire/path-security.js.map +0 -1
- package/dist/wire/pkt-line.d.ts +0 -345
- package/dist/wire/pkt-line.d.ts.map +0 -1
- package/dist/wire/pkt-line.js +0 -381
- package/dist/wire/pkt-line.js.map +0 -1
- package/dist/wire/receive-pack.d.ts +0 -1059
- package/dist/wire/receive-pack.d.ts.map +0 -1
- package/dist/wire/receive-pack.js +0 -1414
- package/dist/wire/receive-pack.js.map +0 -1
- package/dist/wire/smart-http.d.ts +0 -799
- package/dist/wire/smart-http.d.ts.map +0 -1
- package/dist/wire/smart-http.js +0 -945
- package/dist/wire/smart-http.js.map +0 -1
- package/dist/wire/upload-pack.d.ts +0 -727
- package/dist/wire/upload-pack.d.ts.map +0 -1
- package/dist/wire/upload-pack.js +0 -1138
- package/dist/wire/upload-pack.js.map +0 -1
package/dist/storage/r2-pack.js
DELETED
|
@@ -1,1770 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @fileoverview R2 Packfile Storage Module
|
|
3
|
-
*
|
|
4
|
-
* This module manages Git packfiles stored in Cloudflare R2 object storage.
|
|
5
|
-
* It provides comprehensive functionality for:
|
|
6
|
-
*
|
|
7
|
-
* - **Uploading and downloading packfiles** with their indices using atomic operations
|
|
8
|
-
* - **Multi-pack index (MIDX)** for efficient object lookup across multiple packs
|
|
9
|
-
* - **Concurrent access control** with distributed locking using R2 conditional writes
|
|
10
|
-
* - **Pack verification** and integrity checks via SHA-1 checksums
|
|
11
|
-
* - **Atomic uploads** using a manifest-based pattern to ensure data consistency
|
|
12
|
-
*
|
|
13
|
-
* The module implements Git's packfile format (version 2 and 3) and provides
|
|
14
|
-
* both class-based (`R2PackStorage`) and standalone function APIs for flexibility.
|
|
15
|
-
*
|
|
16
|
-
* @module storage/r2-pack
|
|
17
|
-
*
|
|
18
|
-
* @example
|
|
19
|
-
* ```typescript
|
|
20
|
-
* // Using the class-based API
|
|
21
|
-
* const storage = new R2PackStorage({
|
|
22
|
-
* bucket: myR2Bucket,
|
|
23
|
-
* prefix: 'repos/my-repo/',
|
|
24
|
-
* cacheSize: 100,
|
|
25
|
-
* cacheTTL: 3600
|
|
26
|
-
* });
|
|
27
|
-
*
|
|
28
|
-
* // Upload a packfile
|
|
29
|
-
* const result = await storage.uploadPackfile(packData, indexData);
|
|
30
|
-
* console.log(`Uploaded pack: ${result.packId}`);
|
|
31
|
-
*
|
|
32
|
-
* // Download with verification
|
|
33
|
-
* const download = await storage.downloadPackfile(result.packId, {
|
|
34
|
-
* verify: true,
|
|
35
|
-
* includeIndex: true
|
|
36
|
-
* });
|
|
37
|
-
* ```
|
|
38
|
-
*
|
|
39
|
-
* @example
|
|
40
|
-
* ```typescript
|
|
41
|
-
* // Using standalone functions
|
|
42
|
-
* const result = await uploadPackfile(bucket, packData, indexData, {
|
|
43
|
-
* prefix: 'repos/my-repo/'
|
|
44
|
-
* });
|
|
45
|
-
*
|
|
46
|
-
* const packfiles = await listPackfiles(bucket, {
|
|
47
|
-
* prefix: 'repos/my-repo/',
|
|
48
|
-
* limit: 10
|
|
49
|
-
* });
|
|
50
|
-
* ```
|
|
51
|
-
*/
|
|
52
|
-
/**
|
|
53
|
-
* Error thrown by R2 pack operations.
|
|
54
|
-
*
|
|
55
|
-
* @description
|
|
56
|
-
* Custom error class for R2 packfile operations with error codes for
|
|
57
|
-
* programmatic error handling.
|
|
58
|
-
*
|
|
59
|
-
* Error codes:
|
|
60
|
-
* - `NOT_FOUND`: Packfile does not exist
|
|
61
|
-
* - `LOCKED`: Packfile is locked by another process
|
|
62
|
-
* - `INVALID_DATA`: Packfile format is invalid
|
|
63
|
-
* - `CHECKSUM_MISMATCH`: Checksum verification failed
|
|
64
|
-
* - `NETWORK_ERROR`: R2 network/connectivity issue
|
|
65
|
-
*
|
|
66
|
-
* @example
|
|
67
|
-
* ```typescript
|
|
68
|
-
* try {
|
|
69
|
-
* await storage.downloadPackfile(packId, { required: true });
|
|
70
|
-
* } catch (error) {
|
|
71
|
-
* if (error instanceof R2PackError) {
|
|
72
|
-
* switch (error.code) {
|
|
73
|
-
* case 'NOT_FOUND':
|
|
74
|
-
* console.log('Pack does not exist');
|
|
75
|
-
* break;
|
|
76
|
-
* case 'CHECKSUM_MISMATCH':
|
|
77
|
-
* console.log('Pack is corrupted');
|
|
78
|
-
* break;
|
|
79
|
-
* }
|
|
80
|
-
* }
|
|
81
|
-
* }
|
|
82
|
-
* ```
|
|
83
|
-
*/
|
|
84
|
-
export class R2PackError extends Error {
|
|
85
|
-
code;
|
|
86
|
-
packId;
|
|
87
|
-
/**
|
|
88
|
-
* Creates a new R2PackError.
|
|
89
|
-
*
|
|
90
|
-
* @param message - Human-readable error message
|
|
91
|
-
* @param code - Error code for programmatic handling
|
|
92
|
-
* @param packId - Optional pack ID related to the error
|
|
93
|
-
*/
|
|
94
|
-
constructor(message, code, packId) {
|
|
95
|
-
super(message);
|
|
96
|
-
this.code = code;
|
|
97
|
-
this.packId = packId;
|
|
98
|
-
this.name = 'R2PackError';
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
// PACK signature bytes: "PACK"
|
|
102
|
-
const PACK_SIGNATURE = new Uint8Array([0x50, 0x41, 0x43, 0x4b]);
|
|
103
|
-
// Multi-pack index signature
|
|
104
|
-
const MIDX_SIGNATURE = new Uint8Array([0x4d, 0x49, 0x44, 0x58]); // "MIDX"
|
|
105
|
-
/**
|
|
106
|
-
* Validates a packfile header and extracts version and object count.
|
|
107
|
-
*
|
|
108
|
-
* @description
|
|
109
|
-
* Checks that the packfile has a valid PACK signature and supported version (2 or 3).
|
|
110
|
-
*
|
|
111
|
-
* @param data - Raw packfile bytes
|
|
112
|
-
* @returns Object containing version number and object count
|
|
113
|
-
*
|
|
114
|
-
* @throws {R2PackError} With code 'INVALID_DATA' if packfile is invalid
|
|
115
|
-
*
|
|
116
|
-
* @example
|
|
117
|
-
* ```typescript
|
|
118
|
-
* const { version, objectCount } = validatePackfile(packData);
|
|
119
|
-
* console.log(`Pack version ${version} with ${objectCount} objects`);
|
|
120
|
-
* ```
|
|
121
|
-
*
|
|
122
|
-
* @internal
|
|
123
|
-
*/
|
|
124
|
-
function validatePackfile(data) {
|
|
125
|
-
if (data.length < 12) {
|
|
126
|
-
throw new R2PackError('Packfile too small', 'INVALID_DATA');
|
|
127
|
-
}
|
|
128
|
-
// Check signature
|
|
129
|
-
for (let i = 0; i < 4; i++) {
|
|
130
|
-
if (data[i] !== PACK_SIGNATURE[i]) {
|
|
131
|
-
throw new R2PackError('Invalid packfile signature', 'INVALID_DATA');
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
// Read version (big endian 4 bytes)
|
|
135
|
-
const version = (data[4] << 24) | (data[5] << 16) | (data[6] << 8) | data[7];
|
|
136
|
-
if (version !== 2 && version !== 3) {
|
|
137
|
-
throw new R2PackError(`Unsupported pack version: ${version}`, 'INVALID_DATA');
|
|
138
|
-
}
|
|
139
|
-
// Read object count (big endian 4 bytes)
|
|
140
|
-
const objectCount = (data[8] << 24) | (data[9] << 16) | (data[10] << 8) | data[11];
|
|
141
|
-
return { version, objectCount };
|
|
142
|
-
}
|
|
143
|
-
/**
|
|
144
|
-
* Computes SHA-1 checksum of data as a hexadecimal string.
|
|
145
|
-
*
|
|
146
|
-
* @description
|
|
147
|
-
* Uses the Web Crypto API to compute SHA-1 hash for Git compatibility.
|
|
148
|
-
*
|
|
149
|
-
* @param data - Data to hash
|
|
150
|
-
* @returns 40-character lowercase hexadecimal SHA-1 hash
|
|
151
|
-
*
|
|
152
|
-
* @example
|
|
153
|
-
* ```typescript
|
|
154
|
-
* const checksum = await computeChecksum(packData);
|
|
155
|
-
* console.log(`SHA-1: ${checksum}`);
|
|
156
|
-
* ```
|
|
157
|
-
*
|
|
158
|
-
* @internal
|
|
159
|
-
*/
|
|
160
|
-
async function computeChecksum(data) {
|
|
161
|
-
const hashBuffer = await crypto.subtle.digest('SHA-1', data);
|
|
162
|
-
const hashArray = new Uint8Array(hashBuffer);
|
|
163
|
-
return Array.from(hashArray)
|
|
164
|
-
.map(b => b.toString(16).padStart(2, '0'))
|
|
165
|
-
.join('');
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* Generates a unique pack ID.
|
|
169
|
-
*
|
|
170
|
-
* @description
|
|
171
|
-
* Creates a cryptographically random pack identifier in the format 'pack-{16 hex chars}'.
|
|
172
|
-
*
|
|
173
|
-
* @returns Unique pack ID string
|
|
174
|
-
*
|
|
175
|
-
* @example
|
|
176
|
-
* ```typescript
|
|
177
|
-
* const packId = generatePackId();
|
|
178
|
-
* // Returns something like: 'pack-a1b2c3d4e5f67890'
|
|
179
|
-
* ```
|
|
180
|
-
*
|
|
181
|
-
* @internal
|
|
182
|
-
*/
|
|
183
|
-
function generatePackId() {
|
|
184
|
-
const randomBytes = new Uint8Array(8);
|
|
185
|
-
crypto.getRandomValues(randomBytes);
|
|
186
|
-
const hex = Array.from(randomBytes)
|
|
187
|
-
.map(b => b.toString(16).padStart(2, '0'))
|
|
188
|
-
.join('');
|
|
189
|
-
return `pack-${hex}`;
|
|
190
|
-
}
|
|
191
|
-
/**
|
|
192
|
-
* Builds the full key path with prefix.
|
|
193
|
-
*
|
|
194
|
-
* @description
|
|
195
|
-
* Normalizes the prefix to ensure it has a trailing slash and prepends it to the path.
|
|
196
|
-
*
|
|
197
|
-
* @param prefix - Storage prefix (may or may not have trailing slash)
|
|
198
|
-
* @param path - Path to append to prefix
|
|
199
|
-
* @returns Full key path
|
|
200
|
-
*
|
|
201
|
-
* @example
|
|
202
|
-
* ```typescript
|
|
203
|
-
* buildKey('repos/my-repo', 'packs/pack-123.pack')
|
|
204
|
-
* // Returns: 'repos/my-repo/packs/pack-123.pack'
|
|
205
|
-
* ```
|
|
206
|
-
*
|
|
207
|
-
* @internal
|
|
208
|
-
*/
|
|
209
|
-
function buildKey(prefix, path) {
|
|
210
|
-
if (!prefix) {
|
|
211
|
-
return path;
|
|
212
|
-
}
|
|
213
|
-
// Normalize prefix to have trailing slash
|
|
214
|
-
const normalizedPrefix = prefix.endsWith('/') ? prefix : prefix + '/';
|
|
215
|
-
return normalizedPrefix + path;
|
|
216
|
-
}
|
|
217
|
-
/**
|
|
218
|
-
* Generates a unique lock ID.
|
|
219
|
-
*
|
|
220
|
-
* @description
|
|
221
|
-
* Creates a cryptographically random lock identifier (32 hex chars).
|
|
222
|
-
*
|
|
223
|
-
* @returns Unique lock ID string
|
|
224
|
-
*
|
|
225
|
-
* @internal
|
|
226
|
-
*/
|
|
227
|
-
function generateLockId() {
|
|
228
|
-
const randomBytes = new Uint8Array(16);
|
|
229
|
-
crypto.getRandomValues(randomBytes);
|
|
230
|
-
return Array.from(randomBytes)
|
|
231
|
-
.map(b => b.toString(16).padStart(2, '0'))
|
|
232
|
-
.join('');
|
|
233
|
-
}
|
|
234
|
-
/**
|
|
235
|
-
* R2 Packfile Storage class.
|
|
236
|
-
*
|
|
237
|
-
* @description
|
|
238
|
-
* Main class for managing Git packfiles in Cloudflare R2 object storage.
|
|
239
|
-
* Provides methods for uploading, downloading, listing, and managing packfiles
|
|
240
|
-
* with support for atomic uploads, distributed locking, and multi-pack indexing.
|
|
241
|
-
*
|
|
242
|
-
* @example
|
|
243
|
-
* ```typescript
|
|
244
|
-
* // Initialize storage
|
|
245
|
-
* const storage = new R2PackStorage({
|
|
246
|
-
* bucket: env.GIT_BUCKET,
|
|
247
|
-
* prefix: 'repos/my-repo/',
|
|
248
|
-
* cacheSize: 100,
|
|
249
|
-
* cacheTTL: 3600
|
|
250
|
-
* });
|
|
251
|
-
*
|
|
252
|
-
* // Upload a packfile atomically
|
|
253
|
-
* const result = await storage.uploadPackfile(packData, indexData);
|
|
254
|
-
*
|
|
255
|
-
* // Download with verification
|
|
256
|
-
* const download = await storage.downloadPackfile(result.packId, {
|
|
257
|
-
* verify: true,
|
|
258
|
-
* includeIndex: true
|
|
259
|
-
* });
|
|
260
|
-
*
|
|
261
|
-
* // List all packfiles
|
|
262
|
-
* const list = await storage.listPackfiles();
|
|
263
|
-
*
|
|
264
|
-
* // Acquire lock for write operations
|
|
265
|
-
* const lock = await storage.acquireLock(packId, { ttl: 30000 });
|
|
266
|
-
* try {
|
|
267
|
-
* // Perform operations
|
|
268
|
-
* } finally {
|
|
269
|
-
* await lock.release();
|
|
270
|
-
* }
|
|
271
|
-
* ```
|
|
272
|
-
*/
|
|
273
|
-
export class R2PackStorage {
|
|
274
|
-
_bucket;
|
|
275
|
-
_prefix;
|
|
276
|
-
_cacheTTL;
|
|
277
|
-
_midxCache = null;
|
|
278
|
-
_indexChecksums = new Map();
|
|
279
|
-
/**
|
|
280
|
-
* Creates a new R2PackStorage instance.
|
|
281
|
-
*
|
|
282
|
-
* @param options - Configuration options for the storage instance
|
|
283
|
-
*
|
|
284
|
-
* @example
|
|
285
|
-
* ```typescript
|
|
286
|
-
* const storage = new R2PackStorage({
|
|
287
|
-
* bucket: env.MY_BUCKET,
|
|
288
|
-
* prefix: 'repos/my-repo/',
|
|
289
|
-
* cacheSize: 100,
|
|
290
|
-
* cacheTTL: 3600
|
|
291
|
-
* });
|
|
292
|
-
* ```
|
|
293
|
-
*/
|
|
294
|
-
constructor(options) {
|
|
295
|
-
this._bucket = options.bucket;
|
|
296
|
-
this._prefix = options.prefix ?? '';
|
|
297
|
-
void (options.cacheSize ?? 100); // Reserved for LRU cache implementation
|
|
298
|
-
this._cacheTTL = options.cacheTTL ?? 3600;
|
|
299
|
-
}
|
|
300
|
-
_buildKey(path) {
|
|
301
|
-
return buildKey(this._prefix, path);
|
|
302
|
-
}
|
|
303
|
-
/**
|
|
304
|
-
* Uploads a packfile and its index to R2 atomically.
|
|
305
|
-
*
|
|
306
|
-
* @description
|
|
307
|
-
* Uses a manifest-based pattern to ensure atomic uploads:
|
|
308
|
-
* 1. Upload pack and index to staging paths
|
|
309
|
-
* 2. Create manifest in 'staging' status
|
|
310
|
-
* 3. Copy from staging to final location
|
|
311
|
-
* 4. Update manifest to 'complete' status
|
|
312
|
-
* 5. Clean up staging files
|
|
313
|
-
*
|
|
314
|
-
* If the process fails at any point, the pack is not considered complete
|
|
315
|
-
* until a valid manifest with status 'complete' exists.
|
|
316
|
-
*
|
|
317
|
-
* @param packData - Raw packfile bytes (must have valid PACK signature)
|
|
318
|
-
* @param indexData - Pack index file bytes
|
|
319
|
-
* @param options - Optional upload configuration
|
|
320
|
-
*
|
|
321
|
-
* @returns Upload result with pack ID, sizes, checksum, and object count
|
|
322
|
-
*
|
|
323
|
-
* @throws {R2PackError} With code 'INVALID_DATA' if packfile is invalid
|
|
324
|
-
* @throws {R2PackError} With code 'NETWORK_ERROR' if bucket is unavailable
|
|
325
|
-
*
|
|
326
|
-
* @example
|
|
327
|
-
* ```typescript
|
|
328
|
-
* const result = await storage.uploadPackfile(packData, indexData);
|
|
329
|
-
* console.log(`Uploaded: ${result.packId}`);
|
|
330
|
-
* console.log(`Objects: ${result.objectCount}`);
|
|
331
|
-
* console.log(`Checksum: ${result.checksum}`);
|
|
332
|
-
* ```
|
|
333
|
-
*/
|
|
334
|
-
async uploadPackfile(packData, indexData, options) {
|
|
335
|
-
if (!this._bucket) {
|
|
336
|
-
throw new R2PackError('Bucket not available', 'NETWORK_ERROR');
|
|
337
|
-
}
|
|
338
|
-
// Validate packfile
|
|
339
|
-
const { objectCount } = validatePackfile(packData);
|
|
340
|
-
// Generate unique pack ID and checksums
|
|
341
|
-
const packId = generatePackId();
|
|
342
|
-
const packChecksum = await computeChecksum(packData);
|
|
343
|
-
const indexChecksum = await computeChecksum(indexData);
|
|
344
|
-
const uploadedAt = new Date();
|
|
345
|
-
// Store metadata for the files
|
|
346
|
-
const metadata = {
|
|
347
|
-
packId,
|
|
348
|
-
packSize: String(packData.length),
|
|
349
|
-
indexSize: String(indexData.length),
|
|
350
|
-
objectCount: String(objectCount),
|
|
351
|
-
checksum: packChecksum,
|
|
352
|
-
createdAt: uploadedAt.toISOString()
|
|
353
|
-
};
|
|
354
|
-
// If skipAtomic is set, use the simple (non-atomic) upload path
|
|
355
|
-
if (options?.skipAtomic) {
|
|
356
|
-
const packKey = this._buildKey(`packs/${packId}.pack`);
|
|
357
|
-
await this._bucket.put(packKey, packData, { customMetadata: metadata });
|
|
358
|
-
const idxKey = this._buildKey(`packs/${packId}.idx`);
|
|
359
|
-
await this._bucket.put(idxKey, indexData, { customMetadata: metadata });
|
|
360
|
-
this._indexChecksums.set(packId, indexChecksum);
|
|
361
|
-
return {
|
|
362
|
-
packId,
|
|
363
|
-
packSize: packData.length,
|
|
364
|
-
indexSize: indexData.length,
|
|
365
|
-
checksum: packChecksum,
|
|
366
|
-
objectCount,
|
|
367
|
-
uploadedAt
|
|
368
|
-
};
|
|
369
|
-
}
|
|
370
|
-
// Step 1: Upload to staging paths
|
|
371
|
-
const stagingPackKey = this._buildKey(`staging/${packId}.pack`);
|
|
372
|
-
const stagingIdxKey = this._buildKey(`staging/${packId}.idx`);
|
|
373
|
-
const manifestKey = this._buildKey(`packs/${packId}.manifest`);
|
|
374
|
-
try {
|
|
375
|
-
// Upload pack to staging
|
|
376
|
-
await this._bucket.put(stagingPackKey, packData, { customMetadata: metadata });
|
|
377
|
-
// Upload index to staging
|
|
378
|
-
await this._bucket.put(stagingIdxKey, indexData, { customMetadata: metadata });
|
|
379
|
-
// Step 2: Create manifest in 'staging' status
|
|
380
|
-
const manifest = {
|
|
381
|
-
version: 1,
|
|
382
|
-
packId,
|
|
383
|
-
packChecksum,
|
|
384
|
-
indexChecksum,
|
|
385
|
-
packSize: packData.length,
|
|
386
|
-
indexSize: indexData.length,
|
|
387
|
-
objectCount,
|
|
388
|
-
completedAt: uploadedAt.toISOString(),
|
|
389
|
-
status: 'staging'
|
|
390
|
-
};
|
|
391
|
-
await this._bucket.put(manifestKey, JSON.stringify(manifest), {
|
|
392
|
-
customMetadata: { packId, status: 'staging' }
|
|
393
|
-
});
|
|
394
|
-
// Step 3: Copy from staging to final location
|
|
395
|
-
const packKey = this._buildKey(`packs/${packId}.pack`);
|
|
396
|
-
const idxKey = this._buildKey(`packs/${packId}.idx`);
|
|
397
|
-
await this._bucket.put(packKey, packData, { customMetadata: metadata });
|
|
398
|
-
await this._bucket.put(idxKey, indexData, { customMetadata: metadata });
|
|
399
|
-
// Step 4: Update manifest to 'complete' status
|
|
400
|
-
manifest.status = 'complete';
|
|
401
|
-
await this._bucket.put(manifestKey, JSON.stringify(manifest), {
|
|
402
|
-
customMetadata: { packId, status: 'complete' }
|
|
403
|
-
});
|
|
404
|
-
// Step 5: Clean up staging files
|
|
405
|
-
await this._bucket.delete([stagingPackKey, stagingIdxKey]);
|
|
406
|
-
// Store index checksum for verification
|
|
407
|
-
this._indexChecksums.set(packId, indexChecksum);
|
|
408
|
-
return {
|
|
409
|
-
packId,
|
|
410
|
-
packSize: packData.length,
|
|
411
|
-
indexSize: indexData.length,
|
|
412
|
-
checksum: packChecksum,
|
|
413
|
-
objectCount,
|
|
414
|
-
uploadedAt
|
|
415
|
-
};
|
|
416
|
-
}
|
|
417
|
-
catch (error) {
|
|
418
|
-
// Clean up any partial uploads on failure
|
|
419
|
-
try {
|
|
420
|
-
await this._bucket.delete([
|
|
421
|
-
stagingPackKey,
|
|
422
|
-
stagingIdxKey,
|
|
423
|
-
this._buildKey(`packs/${packId}.pack`),
|
|
424
|
-
this._buildKey(`packs/${packId}.idx`),
|
|
425
|
-
manifestKey
|
|
426
|
-
]);
|
|
427
|
-
}
|
|
428
|
-
catch {
|
|
429
|
-
// Ignore cleanup errors
|
|
430
|
-
}
|
|
431
|
-
throw error;
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
|
-
/**
|
|
435
|
-
* Gets the manifest for a packfile.
|
|
436
|
-
*
|
|
437
|
-
* @description
|
|
438
|
-
* Retrieves the manifest JSON that tracks the upload status of a packfile.
|
|
439
|
-
* Returns null if no manifest exists (legacy packs or invalid pack ID).
|
|
440
|
-
*
|
|
441
|
-
* @param packId - Pack identifier to get manifest for
|
|
442
|
-
* @returns Pack manifest or null if not found
|
|
443
|
-
*
|
|
444
|
-
* @example
|
|
445
|
-
* ```typescript
|
|
446
|
-
* const manifest = await storage.getPackManifest('pack-abc123');
|
|
447
|
-
* if (manifest?.status === 'complete') {
|
|
448
|
-
* console.log('Pack is ready for use');
|
|
449
|
-
* } else {
|
|
450
|
-
* console.log('Pack upload is incomplete');
|
|
451
|
-
* }
|
|
452
|
-
* ```
|
|
453
|
-
*/
|
|
454
|
-
async getPackManifest(packId) {
|
|
455
|
-
const manifestKey = this._buildKey(`packs/${packId}.manifest`);
|
|
456
|
-
const manifestObj = await this._bucket.get(manifestKey);
|
|
457
|
-
if (!manifestObj) {
|
|
458
|
-
return null;
|
|
459
|
-
}
|
|
460
|
-
try {
|
|
461
|
-
const text = await manifestObj.text();
|
|
462
|
-
return JSON.parse(text);
|
|
463
|
-
}
|
|
464
|
-
catch {
|
|
465
|
-
return null;
|
|
466
|
-
}
|
|
467
|
-
}
|
|
468
|
-
/**
|
|
469
|
-
* Checks if a packfile upload is complete.
|
|
470
|
-
*
|
|
471
|
-
* @description
|
|
472
|
-
* A pack is considered complete if:
|
|
473
|
-
* 1. It has a manifest with status 'complete', OR
|
|
474
|
-
* 2. It was uploaded before the atomic upload feature (legacy packs without manifest)
|
|
475
|
-
* AND both .pack and .idx files exist
|
|
476
|
-
*
|
|
477
|
-
* @param packId - Pack identifier to check
|
|
478
|
-
* @returns true if pack is complete and ready for use
|
|
479
|
-
*
|
|
480
|
-
* @example
|
|
481
|
-
* ```typescript
|
|
482
|
-
* if (await storage.isPackComplete(packId)) {
|
|
483
|
-
* const data = await storage.downloadPackfile(packId);
|
|
484
|
-
* }
|
|
485
|
-
* ```
|
|
486
|
-
*/
|
|
487
|
-
async isPackComplete(packId) {
|
|
488
|
-
// Check for manifest first
|
|
489
|
-
const manifest = await this.getPackManifest(packId);
|
|
490
|
-
if (manifest) {
|
|
491
|
-
// If manifest exists, it must have 'complete' status
|
|
492
|
-
return manifest.status === 'complete';
|
|
493
|
-
}
|
|
494
|
-
// Legacy pack without manifest - check if both files exist
|
|
495
|
-
const packKey = this._buildKey(`packs/${packId}.pack`);
|
|
496
|
-
const idxKey = this._buildKey(`packs/${packId}.idx`);
|
|
497
|
-
const [packExists, idxExists] = await Promise.all([
|
|
498
|
-
this._bucket.head(packKey),
|
|
499
|
-
this._bucket.head(idxKey)
|
|
500
|
-
]);
|
|
501
|
-
return packExists !== null && idxExists !== null;
|
|
502
|
-
}
|
|
503
|
-
/**
|
|
504
|
-
* Downloads a packfile from R2.
|
|
505
|
-
*
|
|
506
|
-
* @description
|
|
507
|
-
* Downloads pack data with optional index file. Verifies pack completeness
|
|
508
|
-
* before downloading and optionally verifies checksum integrity.
|
|
509
|
-
*
|
|
510
|
-
* @param packId - Pack identifier to download
|
|
511
|
-
* @param options - Download options (includeIndex, verify, byteRange, required)
|
|
512
|
-
*
|
|
513
|
-
* @returns Download result with pack data, or null if not found (unless required=true)
|
|
514
|
-
*
|
|
515
|
-
* @throws {R2PackError} With code 'NOT_FOUND' if required=true and pack not found
|
|
516
|
-
* @throws {R2PackError} With code 'CHECKSUM_MISMATCH' if verify=true and verification fails
|
|
517
|
-
*
|
|
518
|
-
* @example
|
|
519
|
-
* ```typescript
|
|
520
|
-
* // Basic download
|
|
521
|
-
* const result = await storage.downloadPackfile(packId);
|
|
522
|
-
*
|
|
523
|
-
* // Download with verification and index
|
|
524
|
-
* const verified = await storage.downloadPackfile(packId, {
|
|
525
|
-
* verify: true,
|
|
526
|
-
* includeIndex: true
|
|
527
|
-
* });
|
|
528
|
-
*
|
|
529
|
-
* // Required download (throws if not found)
|
|
530
|
-
* const required = await storage.downloadPackfile(packId, { required: true });
|
|
531
|
-
* ```
|
|
532
|
-
*/
|
|
533
|
-
async downloadPackfile(packId, options) {
|
|
534
|
-
// Verify pack completeness before downloading
|
|
535
|
-
const isComplete = await this.isPackComplete(packId);
|
|
536
|
-
if (!isComplete) {
|
|
537
|
-
if (options?.required) {
|
|
538
|
-
throw new R2PackError(`Packfile incomplete or not found: ${packId}`, 'NOT_FOUND', packId);
|
|
539
|
-
}
|
|
540
|
-
return null;
|
|
541
|
-
}
|
|
542
|
-
const packKey = this._buildKey(`packs/${packId}.pack`);
|
|
543
|
-
const packObj = await this._bucket.get(packKey);
|
|
544
|
-
if (!packObj) {
|
|
545
|
-
if (options?.required) {
|
|
546
|
-
throw new R2PackError(`Packfile not found: ${packId}`, 'NOT_FOUND', packId);
|
|
547
|
-
}
|
|
548
|
-
return null;
|
|
549
|
-
}
|
|
550
|
-
let packData = new Uint8Array(await packObj.arrayBuffer());
|
|
551
|
-
// Verify checksum if requested (before byte range slicing)
|
|
552
|
-
if (options?.verify && !options?.byteRange) {
|
|
553
|
-
// Get stored checksum from metadata
|
|
554
|
-
const headObj = await this._bucket.head(packKey);
|
|
555
|
-
const storedChecksum = headObj?.customMetadata?.checksum;
|
|
556
|
-
if (storedChecksum) {
|
|
557
|
-
const computedChecksum = await computeChecksum(packData);
|
|
558
|
-
if (computedChecksum !== storedChecksum) {
|
|
559
|
-
throw new R2PackError(`Checksum mismatch for packfile: ${packId}`, 'CHECKSUM_MISMATCH', packId);
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
else {
|
|
563
|
-
// No stored checksum - data may have been corrupted/replaced
|
|
564
|
-
// Verify using the embedded pack checksum (last 20 bytes of packfile)
|
|
565
|
-
if (packData.length >= 20) {
|
|
566
|
-
const dataWithoutChecksum = packData.slice(0, packData.length - 20);
|
|
567
|
-
const computedChecksum = await computeChecksum(dataWithoutChecksum);
|
|
568
|
-
const embeddedChecksum = Array.from(packData.slice(packData.length - 20))
|
|
569
|
-
.map(b => b.toString(16).padStart(2, '0'))
|
|
570
|
-
.join('');
|
|
571
|
-
if (computedChecksum !== embeddedChecksum) {
|
|
572
|
-
throw new R2PackError(`Checksum mismatch for packfile: ${packId}`, 'CHECKSUM_MISMATCH', packId);
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
else {
|
|
576
|
-
throw new R2PackError(`Packfile too small to verify: ${packId}`, 'CHECKSUM_MISMATCH', packId);
|
|
577
|
-
}
|
|
578
|
-
}
|
|
579
|
-
}
|
|
580
|
-
// Handle byte range
|
|
581
|
-
if (options?.byteRange) {
|
|
582
|
-
const { start, end } = options.byteRange;
|
|
583
|
-
packData = packData.slice(start, end + 1);
|
|
584
|
-
}
|
|
585
|
-
const result = {
|
|
586
|
-
packData,
|
|
587
|
-
verified: options?.verify ? true : undefined
|
|
588
|
-
};
|
|
589
|
-
// Include index if requested
|
|
590
|
-
if (options?.includeIndex) {
|
|
591
|
-
const idxKey = this._buildKey(`packs/${packId}.idx`);
|
|
592
|
-
const idxObj = await this._bucket.get(idxKey);
|
|
593
|
-
if (idxObj) {
|
|
594
|
-
result.indexData = new Uint8Array(await idxObj.arrayBuffer());
|
|
595
|
-
}
|
|
596
|
-
}
|
|
597
|
-
return result;
|
|
598
|
-
}
|
|
599
|
-
/**
|
|
600
|
-
* Gets metadata for a packfile.
|
|
601
|
-
*
|
|
602
|
-
* @description
|
|
603
|
-
* Retrieves metadata about a packfile including size, object count,
|
|
604
|
-
* creation time, and checksum without downloading the full pack.
|
|
605
|
-
*
|
|
606
|
-
* @param packId - Pack identifier to get metadata for
|
|
607
|
-
* @returns Packfile metadata or null if not found
|
|
608
|
-
*
|
|
609
|
-
* @example
|
|
610
|
-
* ```typescript
|
|
611
|
-
* const metadata = await storage.getPackfileMetadata(packId);
|
|
612
|
-
* if (metadata) {
|
|
613
|
-
* console.log(`Size: ${metadata.packSize} bytes`);
|
|
614
|
-
* console.log(`Objects: ${metadata.objectCount}`);
|
|
615
|
-
* }
|
|
616
|
-
* ```
|
|
617
|
-
*/
|
|
618
|
-
async getPackfileMetadata(packId) {
|
|
619
|
-
const packKey = this._buildKey(`packs/${packId}.pack`);
|
|
620
|
-
const headObj = await this._bucket.head(packKey);
|
|
621
|
-
if (!headObj) {
|
|
622
|
-
return null;
|
|
623
|
-
}
|
|
624
|
-
const meta = headObj.customMetadata || {};
|
|
625
|
-
return {
|
|
626
|
-
packId,
|
|
627
|
-
packSize: parseInt(meta.packSize || String(headObj.size), 10),
|
|
628
|
-
indexSize: parseInt(meta.indexSize || '0', 10),
|
|
629
|
-
objectCount: parseInt(meta.objectCount || '0', 10),
|
|
630
|
-
createdAt: new Date(meta.createdAt || Date.now()),
|
|
631
|
-
checksum: meta.checksum || ''
|
|
632
|
-
};
|
|
633
|
-
}
|
|
634
|
-
/**
|
|
635
|
-
* Lists all packfiles in storage.
|
|
636
|
-
*
|
|
637
|
-
* @description
|
|
638
|
-
* Returns a paginated list of packfile metadata. Use the cursor for
|
|
639
|
-
* fetching subsequent pages of results.
|
|
640
|
-
*
|
|
641
|
-
* @param options - Pagination options (limit, cursor)
|
|
642
|
-
* @returns List of packfile metadata with optional cursor for pagination
|
|
643
|
-
*
|
|
644
|
-
* @example
|
|
645
|
-
* ```typescript
|
|
646
|
-
* // List first 10 packfiles
|
|
647
|
-
* const first = await storage.listPackfiles({ limit: 10 });
|
|
648
|
-
*
|
|
649
|
-
* // Get next page
|
|
650
|
-
* if (first.cursor) {
|
|
651
|
-
* const next = await storage.listPackfiles({ limit: 10, cursor: first.cursor });
|
|
652
|
-
* }
|
|
653
|
-
* ```
|
|
654
|
-
*/
|
|
655
|
-
async listPackfiles(options) {
|
|
656
|
-
const prefix = this._buildKey('packs/');
|
|
657
|
-
const listResult = await this._bucket.list({ prefix, cursor: options?.cursor });
|
|
658
|
-
// Filter for .pack files only
|
|
659
|
-
let packFiles = listResult.objects.filter(obj => obj.key.endsWith('.pack'));
|
|
660
|
-
// Handle pagination with cursor (cursor is the index to start from)
|
|
661
|
-
let startIndex = 0;
|
|
662
|
-
if (options?.cursor) {
|
|
663
|
-
startIndex = parseInt(options.cursor, 10) || 0;
|
|
664
|
-
}
|
|
665
|
-
// Slice from cursor position
|
|
666
|
-
packFiles = packFiles.slice(startIndex);
|
|
667
|
-
// Apply limit
|
|
668
|
-
const hasLimit = options?.limit !== undefined && options.limit > 0;
|
|
669
|
-
const limitedPackFiles = hasLimit ? packFiles.slice(0, options.limit) : packFiles;
|
|
670
|
-
const items = [];
|
|
671
|
-
for (const obj of limitedPackFiles) {
|
|
672
|
-
// Extract packId from key
|
|
673
|
-
const match = obj.key.match(/([^/]+)\.pack$/);
|
|
674
|
-
if (match) {
|
|
675
|
-
const packId = match[1];
|
|
676
|
-
const metadata = await this.getPackfileMetadata(packId);
|
|
677
|
-
if (metadata) {
|
|
678
|
-
items.push(metadata);
|
|
679
|
-
}
|
|
680
|
-
}
|
|
681
|
-
}
|
|
682
|
-
// If no pagination options and no items, return a plain empty array
|
|
683
|
-
// This ensures toEqual([]) works as expected
|
|
684
|
-
if (items.length === 0 && !options?.limit && !options?.cursor) {
|
|
685
|
-
return [];
|
|
686
|
-
}
|
|
687
|
-
// Create a new array that also has ListPackfilesResult properties
|
|
688
|
-
const resultArray = [...items];
|
|
689
|
-
const result = resultArray;
|
|
690
|
-
result.items = items;
|
|
691
|
-
// Set cursor for next page if there are more items
|
|
692
|
-
if (hasLimit && packFiles.length > options.limit) {
|
|
693
|
-
result.cursor = String(startIndex + options.limit);
|
|
694
|
-
}
|
|
695
|
-
return result;
|
|
696
|
-
}
|
|
697
|
-
/**
|
|
698
|
-
* Deletes a packfile, its index, and manifest.
|
|
699
|
-
*
|
|
700
|
-
* @description
|
|
701
|
-
* Removes all files associated with a packfile and updates the
|
|
702
|
-
* multi-pack index if needed.
|
|
703
|
-
*
|
|
704
|
-
* @param packId - Pack identifier to delete
|
|
705
|
-
* @returns true if pack was deleted, false if it didn't exist
|
|
706
|
-
*
|
|
707
|
-
* @example
|
|
708
|
-
* ```typescript
|
|
709
|
-
* if (await storage.deletePackfile(packId)) {
|
|
710
|
-
* console.log('Pack deleted successfully');
|
|
711
|
-
* } else {
|
|
712
|
-
* console.log('Pack not found');
|
|
713
|
-
* }
|
|
714
|
-
* ```
|
|
715
|
-
*/
|
|
716
|
-
async deletePackfile(packId) {
|
|
717
|
-
const packKey = this._buildKey(`packs/${packId}.pack`);
|
|
718
|
-
const idxKey = this._buildKey(`packs/${packId}.idx`);
|
|
719
|
-
const manifestKey = this._buildKey(`packs/${packId}.manifest`);
|
|
720
|
-
// Check if pack exists
|
|
721
|
-
const exists = await this._bucket.head(packKey);
|
|
722
|
-
if (!exists) {
|
|
723
|
-
return false;
|
|
724
|
-
}
|
|
725
|
-
// Delete pack, index, and manifest atomically
|
|
726
|
-
await this._bucket.delete([packKey, idxKey, manifestKey]);
|
|
727
|
-
// Clear from index checksum cache
|
|
728
|
-
this._indexChecksums.delete(packId);
|
|
729
|
-
// Update multi-pack index if it exists
|
|
730
|
-
try {
|
|
731
|
-
const midx = await this.getMultiPackIndex();
|
|
732
|
-
if (midx.packIds.includes(packId)) {
|
|
733
|
-
// Rebuild without this pack
|
|
734
|
-
await this.rebuildMultiPackIndex();
|
|
735
|
-
}
|
|
736
|
-
}
|
|
737
|
-
catch {
|
|
738
|
-
// Ignore errors when updating multi-pack index
|
|
739
|
-
}
|
|
740
|
-
return true;
|
|
741
|
-
}
|
|
742
|
-
/**
|
|
743
|
-
* Downloads just the index file for a packfile.
|
|
744
|
-
*
|
|
745
|
-
* @description
|
|
746
|
-
* Retrieves only the pack index file, useful for object lookups
|
|
747
|
-
* without downloading the full packfile.
|
|
748
|
-
*
|
|
749
|
-
* @param packId - Pack identifier to download index for
|
|
750
|
-
* @returns Index data or null if not found
|
|
751
|
-
*
|
|
752
|
-
* @example
|
|
753
|
-
* ```typescript
|
|
754
|
-
* const indexData = await storage.downloadIndex(packId);
|
|
755
|
-
* if (indexData) {
|
|
756
|
-
* // Parse and use the index
|
|
757
|
-
* }
|
|
758
|
-
* ```
|
|
759
|
-
*/
|
|
760
|
-
async downloadIndex(packId) {
|
|
761
|
-
const idxKey = this._buildKey(`packs/${packId}.idx`);
|
|
762
|
-
const idxObj = await this._bucket.get(idxKey);
|
|
763
|
-
if (!idxObj) {
|
|
764
|
-
return null;
|
|
765
|
-
}
|
|
766
|
-
return new Uint8Array(await idxObj.arrayBuffer());
|
|
767
|
-
}
|
|
768
|
-
/**
|
|
769
|
-
* Uploads a new index for an existing packfile.
|
|
770
|
-
*
|
|
771
|
-
* @description
|
|
772
|
-
* Replaces the index file for an existing packfile. Useful for
|
|
773
|
-
* regenerating corrupted indices or updating index format.
|
|
774
|
-
*
|
|
775
|
-
* @param packId - Pack identifier to upload index for
|
|
776
|
-
* @param indexData - New index file data
|
|
777
|
-
*
|
|
778
|
-
* @throws {R2PackError} With code 'NOT_FOUND' if packfile doesn't exist
|
|
779
|
-
*
|
|
780
|
-
* @example
|
|
781
|
-
* ```typescript
|
|
782
|
-
* const newIndex = generatePackIndex(packData);
|
|
783
|
-
* await storage.uploadIndex(packId, newIndex);
|
|
784
|
-
* ```
|
|
785
|
-
*/
|
|
786
|
-
async uploadIndex(packId, indexData) {
|
|
787
|
-
// Check if pack exists
|
|
788
|
-
const packKey = this._buildKey(`packs/${packId}.pack`);
|
|
789
|
-
const exists = await this._bucket.head(packKey);
|
|
790
|
-
if (!exists) {
|
|
791
|
-
throw new R2PackError(`Packfile not found: ${packId}`, 'NOT_FOUND', packId);
|
|
792
|
-
}
|
|
793
|
-
// Upload new index
|
|
794
|
-
const idxKey = this._buildKey(`packs/${packId}.idx`);
|
|
795
|
-
await this._bucket.put(idxKey, indexData);
|
|
796
|
-
// Update checksum cache
|
|
797
|
-
const indexChecksum = await computeChecksum(indexData);
|
|
798
|
-
this._indexChecksums.set(packId, indexChecksum);
|
|
799
|
-
}
|
|
800
|
-
/**
|
|
801
|
-
* Verifies that an index matches its packfile.
|
|
802
|
-
*
|
|
803
|
-
* @description
|
|
804
|
-
* Compares the current index checksum against the stored checksum
|
|
805
|
-
* to detect corruption or tampering.
|
|
806
|
-
*
|
|
807
|
-
* @param packId - Pack identifier to verify index for
|
|
808
|
-
* @returns true if index is valid, false if missing or corrupted
|
|
809
|
-
*
|
|
810
|
-
* @example
|
|
811
|
-
* ```typescript
|
|
812
|
-
* if (await storage.verifyIndex(packId)) {
|
|
813
|
-
* console.log('Index is valid');
|
|
814
|
-
* } else {
|
|
815
|
-
* console.log('Index needs to be regenerated');
|
|
816
|
-
* }
|
|
817
|
-
* ```
|
|
818
|
-
*/
|
|
819
|
-
async verifyIndex(packId) {
|
|
820
|
-
// Get current index
|
|
821
|
-
const currentIndex = await this.downloadIndex(packId);
|
|
822
|
-
if (!currentIndex) {
|
|
823
|
-
return false;
|
|
824
|
-
}
|
|
825
|
-
// Compare with stored checksum
|
|
826
|
-
const storedChecksum = this._indexChecksums.get(packId);
|
|
827
|
-
if (storedChecksum) {
|
|
828
|
-
const currentChecksum = await computeChecksum(currentIndex);
|
|
829
|
-
return currentChecksum === storedChecksum;
|
|
830
|
-
}
|
|
831
|
-
// If no stored checksum, consider it valid (basic check)
|
|
832
|
-
return true;
|
|
833
|
-
}
|
|
834
|
-
/**
|
|
835
|
-
* Cleans up orphaned staging files.
|
|
836
|
-
*
|
|
837
|
-
* @description
|
|
838
|
-
* This should be called on startup to clean up any staging files
|
|
839
|
-
* left behind by failed uploads. It will:
|
|
840
|
-
* 1. List all files in the staging directory
|
|
841
|
-
* 2. For each pack ID found, check if it has a complete manifest
|
|
842
|
-
* 3. If not complete, delete the staging files and any partial final files
|
|
843
|
-
*
|
|
844
|
-
* @returns Array of pack IDs that were cleaned up
|
|
845
|
-
*
|
|
846
|
-
* @example
|
|
847
|
-
* ```typescript
|
|
848
|
-
* // Call on worker startup
|
|
849
|
-
* const cleaned = await storage.cleanupOrphanedStagingFiles();
|
|
850
|
-
* if (cleaned.length > 0) {
|
|
851
|
-
* console.log(`Cleaned up ${cleaned.length} orphaned uploads`);
|
|
852
|
-
* }
|
|
853
|
-
* ```
|
|
854
|
-
*/
|
|
855
|
-
async cleanupOrphanedStagingFiles() {
|
|
856
|
-
const stagingPrefix = this._buildKey('staging/');
|
|
857
|
-
const listResult = await this._bucket.list({ prefix: stagingPrefix });
|
|
858
|
-
// Extract unique pack IDs from staging files
|
|
859
|
-
const orphanedPackIds = new Set();
|
|
860
|
-
for (const obj of listResult.objects) {
|
|
861
|
-
// Extract pack ID from key like "staging/pack-xxx.pack" or "staging/pack-xxx.idx"
|
|
862
|
-
const match = obj.key.match(/staging\/([^/]+)\.(pack|idx)$/);
|
|
863
|
-
if (match) {
|
|
864
|
-
orphanedPackIds.add(match[1]);
|
|
865
|
-
}
|
|
866
|
-
}
|
|
867
|
-
const cleanedUp = [];
|
|
868
|
-
for (const packId of orphanedPackIds) {
|
|
869
|
-
// Check if this pack is complete
|
|
870
|
-
const isComplete = await this.isPackComplete(packId);
|
|
871
|
-
if (!isComplete) {
|
|
872
|
-
// Pack is incomplete - clean up all related files
|
|
873
|
-
const filesToDelete = [
|
|
874
|
-
this._buildKey(`staging/${packId}.pack`),
|
|
875
|
-
this._buildKey(`staging/${packId}.idx`),
|
|
876
|
-
this._buildKey(`packs/${packId}.pack`),
|
|
877
|
-
this._buildKey(`packs/${packId}.idx`),
|
|
878
|
-
this._buildKey(`packs/${packId}.manifest`)
|
|
879
|
-
];
|
|
880
|
-
try {
|
|
881
|
-
await this._bucket.delete(filesToDelete);
|
|
882
|
-
cleanedUp.push(packId);
|
|
883
|
-
}
|
|
884
|
-
catch {
|
|
885
|
-
// Ignore errors during cleanup
|
|
886
|
-
}
|
|
887
|
-
}
|
|
888
|
-
else {
|
|
889
|
-
// Pack is complete - just clean up staging files
|
|
890
|
-
const stagingFiles = [
|
|
891
|
-
this._buildKey(`staging/${packId}.pack`),
|
|
892
|
-
this._buildKey(`staging/${packId}.idx`)
|
|
893
|
-
];
|
|
894
|
-
try {
|
|
895
|
-
await this._bucket.delete(stagingFiles);
|
|
896
|
-
cleanedUp.push(packId);
|
|
897
|
-
}
|
|
898
|
-
catch {
|
|
899
|
-
// Ignore errors during cleanup
|
|
900
|
-
}
|
|
901
|
-
}
|
|
902
|
-
}
|
|
903
|
-
return cleanedUp;
|
|
904
|
-
}
|
|
905
|
-
/**
|
|
906
|
-
* Rebuilds the multi-pack index from all packfiles.
|
|
907
|
-
*
|
|
908
|
-
* @description
|
|
909
|
-
* Creates a new MIDX by scanning all packfiles and building a sorted
|
|
910
|
-
* index of all objects. Call this after adding or removing packs.
|
|
911
|
-
*
|
|
912
|
-
* @example
|
|
913
|
-
* ```typescript
|
|
914
|
-
* await storage.rebuildMultiPackIndex();
|
|
915
|
-
* const midx = await storage.getMultiPackIndex();
|
|
916
|
-
* console.log(`Indexed ${midx.entries.length} objects`);
|
|
917
|
-
* ```
|
|
918
|
-
*/
|
|
919
|
-
async rebuildMultiPackIndex() {
|
|
920
|
-
// List all packs
|
|
921
|
-
const packs = await this.listPackfiles();
|
|
922
|
-
const packIds = packs.map(p => p.packId);
|
|
923
|
-
// Create entries for all objects in all packs
|
|
924
|
-
const entries = [];
|
|
925
|
-
for (let packIndex = 0; packIndex < packIds.length; packIndex++) {
|
|
926
|
-
const packId = packIds[packIndex];
|
|
927
|
-
// For now, create a synthetic entry per pack
|
|
928
|
-
// In a real implementation, we would parse the index file
|
|
929
|
-
const metadata = await this.getPackfileMetadata(packId);
|
|
930
|
-
if (metadata) {
|
|
931
|
-
// Create synthetic entries based on object count
|
|
932
|
-
for (let i = 0; i < metadata.objectCount; i++) {
|
|
933
|
-
// Generate synthetic object IDs based on pack checksum and index
|
|
934
|
-
const objectId = metadata.checksum.slice(0, 32) + i.toString(16).padStart(8, '0');
|
|
935
|
-
entries.push({
|
|
936
|
-
objectId,
|
|
937
|
-
packIndex,
|
|
938
|
-
offset: 12 + i * 100 // Synthetic offset
|
|
939
|
-
});
|
|
940
|
-
}
|
|
941
|
-
}
|
|
942
|
-
}
|
|
943
|
-
// Sort entries by objectId for binary search
|
|
944
|
-
entries.sort((a, b) => a.objectId.localeCompare(b.objectId));
|
|
945
|
-
// Create multi-pack index
|
|
946
|
-
const midx = {
|
|
947
|
-
version: 1,
|
|
948
|
-
packIds,
|
|
949
|
-
entries,
|
|
950
|
-
checksum: new Uint8Array(20)
|
|
951
|
-
};
|
|
952
|
-
// Serialize and store
|
|
953
|
-
const serialized = serializeMultiPackIndex(midx);
|
|
954
|
-
const midxKey = this._buildKey('packs/multi-pack-index');
|
|
955
|
-
await this._bucket.put(midxKey, serialized);
|
|
956
|
-
// Update cache
|
|
957
|
-
this._midxCache = {
|
|
958
|
-
midx,
|
|
959
|
-
expiresAt: Date.now() + this._cacheTTL * 1000
|
|
960
|
-
};
|
|
961
|
-
}
|
|
962
|
-
/**
|
|
963
|
-
* Gets the current multi-pack index.
|
|
964
|
-
*
|
|
965
|
-
* @description
|
|
966
|
-
* Returns the MIDX from cache if available and not expired,
|
|
967
|
-
* otherwise fetches from R2. Returns an empty index if none exists.
|
|
968
|
-
*
|
|
969
|
-
* @returns Current multi-pack index
|
|
970
|
-
*
|
|
971
|
-
* @example
|
|
972
|
-
* ```typescript
|
|
973
|
-
* const midx = await storage.getMultiPackIndex();
|
|
974
|
-
* const entry = lookupObjectInMultiPack(midx, objectSha);
|
|
975
|
-
* if (entry) {
|
|
976
|
-
* const packId = midx.packIds[entry.packIndex];
|
|
977
|
-
* console.log(`Object is in pack ${packId}`);
|
|
978
|
-
* }
|
|
979
|
-
* ```
|
|
980
|
-
*/
|
|
981
|
-
async getMultiPackIndex() {
|
|
982
|
-
// Check cache first
|
|
983
|
-
if (this._midxCache && this._midxCache.expiresAt > Date.now()) {
|
|
984
|
-
return this._midxCache.midx;
|
|
985
|
-
}
|
|
986
|
-
const midxKey = this._buildKey('packs/multi-pack-index');
|
|
987
|
-
const midxObj = await this._bucket.get(midxKey);
|
|
988
|
-
if (!midxObj) {
|
|
989
|
-
// Return empty index
|
|
990
|
-
return {
|
|
991
|
-
version: 1,
|
|
992
|
-
packIds: [],
|
|
993
|
-
entries: [],
|
|
994
|
-
checksum: new Uint8Array(20)
|
|
995
|
-
};
|
|
996
|
-
}
|
|
997
|
-
const data = new Uint8Array(await midxObj.arrayBuffer());
|
|
998
|
-
const midx = parseMultiPackIndex(data);
|
|
999
|
-
// Update cache
|
|
1000
|
-
this._midxCache = {
|
|
1001
|
-
midx,
|
|
1002
|
-
expiresAt: Date.now() + this._cacheTTL * 1000
|
|
1003
|
-
};
|
|
1004
|
-
return midx;
|
|
1005
|
-
}
|
|
1006
|
-
/**
|
|
1007
|
-
* Acquires a distributed lock on a resource using R2 conditional writes.
|
|
1008
|
-
*
|
|
1009
|
-
* @description
|
|
1010
|
-
* Uses R2's conditional write feature (ETags) to implement distributed locking.
|
|
1011
|
-
* Locks automatically expire after the TTL to prevent deadlocks.
|
|
1012
|
-
*
|
|
1013
|
-
* @param resource - Resource identifier to lock
|
|
1014
|
-
* @param ttlMs - Time-to-live in milliseconds
|
|
1015
|
-
* @param holder - Optional identifier for the lock holder (for debugging)
|
|
1016
|
-
*
|
|
1017
|
-
* @returns LockHandle if acquired, null if lock is held by another process
|
|
1018
|
-
*
|
|
1019
|
-
* @example
|
|
1020
|
-
* ```typescript
|
|
1021
|
-
* const handle = await storage.acquireDistributedLock('my-resource', 30000, 'worker-1');
|
|
1022
|
-
* if (handle) {
|
|
1023
|
-
* try {
|
|
1024
|
-
* // Do work while holding the lock
|
|
1025
|
-
* } finally {
|
|
1026
|
-
* await storage.releaseDistributedLock(handle);
|
|
1027
|
-
* }
|
|
1028
|
-
* } else {
|
|
1029
|
-
* console.log('Could not acquire lock - resource is busy');
|
|
1030
|
-
* }
|
|
1031
|
-
* ```
|
|
1032
|
-
*/
|
|
1033
|
-
async acquireDistributedLock(resource, ttlMs = 30000, holder) {
|
|
1034
|
-
const lockKey = this._buildKey(`locks/${resource}.lock`);
|
|
1035
|
-
const now = Date.now();
|
|
1036
|
-
const lockId = generateLockId();
|
|
1037
|
-
const expiresAt = now + ttlMs;
|
|
1038
|
-
const lockContent = {
|
|
1039
|
-
lockId,
|
|
1040
|
-
resource,
|
|
1041
|
-
expiresAt,
|
|
1042
|
-
acquiredAt: now,
|
|
1043
|
-
holder
|
|
1044
|
-
};
|
|
1045
|
-
const lockData = new TextEncoder().encode(JSON.stringify(lockContent));
|
|
1046
|
-
// Try to check if there's an existing lock
|
|
1047
|
-
const existingObj = await this._bucket.head(lockKey);
|
|
1048
|
-
if (existingObj) {
|
|
1049
|
-
// Lock file exists, check if it's expired
|
|
1050
|
-
const existingLockObj = await this._bucket.get(lockKey);
|
|
1051
|
-
if (existingLockObj) {
|
|
1052
|
-
try {
|
|
1053
|
-
const existingContent = JSON.parse(new TextDecoder().decode(new Uint8Array(await existingLockObj.arrayBuffer())));
|
|
1054
|
-
if (existingContent.expiresAt > now) {
|
|
1055
|
-
// Lock is still valid, cannot acquire
|
|
1056
|
-
return null;
|
|
1057
|
-
}
|
|
1058
|
-
// Lock is expired, try to overwrite with conditional write
|
|
1059
|
-
// Use the existing etag to ensure atomicity
|
|
1060
|
-
try {
|
|
1061
|
-
await this._bucket.put(lockKey, lockData, {
|
|
1062
|
-
onlyIf: { etagMatches: existingObj.etag }
|
|
1063
|
-
});
|
|
1064
|
-
// Get the new etag after successful write
|
|
1065
|
-
const newObj = await this._bucket.head(lockKey);
|
|
1066
|
-
if (!newObj) {
|
|
1067
|
-
return null;
|
|
1068
|
-
}
|
|
1069
|
-
return {
|
|
1070
|
-
resource,
|
|
1071
|
-
lockId,
|
|
1072
|
-
etag: newObj.etag,
|
|
1073
|
-
expiresAt
|
|
1074
|
-
};
|
|
1075
|
-
}
|
|
1076
|
-
catch {
|
|
1077
|
-
// Conditional write failed - another process got the lock
|
|
1078
|
-
return null;
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
1081
|
-
catch {
|
|
1082
|
-
// Failed to parse lock content, try to clean up and acquire
|
|
1083
|
-
return null;
|
|
1084
|
-
}
|
|
1085
|
-
}
|
|
1086
|
-
}
|
|
1087
|
-
// No existing lock, try to create new one with onlyIf condition
|
|
1088
|
-
try {
|
|
1089
|
-
// Use onlyIf with etagDoesNotMatch to ensure the object doesn't exist
|
|
1090
|
-
// R2 will fail if object already exists when we use this condition
|
|
1091
|
-
await this._bucket.put(lockKey, lockData, {
|
|
1092
|
-
onlyIf: { etagDoesNotMatch: '*' }
|
|
1093
|
-
});
|
|
1094
|
-
// Get the etag of the newly created lock
|
|
1095
|
-
const newObj = await this._bucket.head(lockKey);
|
|
1096
|
-
if (!newObj) {
|
|
1097
|
-
return null;
|
|
1098
|
-
}
|
|
1099
|
-
// Verify we actually own this lock by checking the lockId
|
|
1100
|
-
const verifyObj = await this._bucket.get(lockKey);
|
|
1101
|
-
if (verifyObj) {
|
|
1102
|
-
const content = JSON.parse(new TextDecoder().decode(new Uint8Array(await verifyObj.arrayBuffer())));
|
|
1103
|
-
if (content.lockId !== lockId) {
|
|
1104
|
-
// Another process created the lock
|
|
1105
|
-
return null;
|
|
1106
|
-
}
|
|
1107
|
-
}
|
|
1108
|
-
return {
|
|
1109
|
-
resource,
|
|
1110
|
-
lockId,
|
|
1111
|
-
etag: newObj.etag,
|
|
1112
|
-
expiresAt
|
|
1113
|
-
};
|
|
1114
|
-
}
|
|
1115
|
-
catch {
|
|
1116
|
-
// Failed to create lock - likely another process created it first
|
|
1117
|
-
return null;
|
|
1118
|
-
}
|
|
1119
|
-
}
|
|
1120
|
-
/**
|
|
1121
|
-
* Releases a distributed lock.
|
|
1122
|
-
*
|
|
1123
|
-
* @description
|
|
1124
|
-
* Releases the lock only if the caller still owns it (verified by lockId).
|
|
1125
|
-
* Safe to call even if lock has expired or been taken by another process.
|
|
1126
|
-
*
|
|
1127
|
-
* @param handle - Lock handle returned from acquireDistributedLock
|
|
1128
|
-
*
|
|
1129
|
-
* @example
|
|
1130
|
-
* ```typescript
|
|
1131
|
-
* const handle = await storage.acquireDistributedLock('resource');
|
|
1132
|
-
* if (handle) {
|
|
1133
|
-
* try {
|
|
1134
|
-
* // Do work
|
|
1135
|
-
* } finally {
|
|
1136
|
-
* await storage.releaseDistributedLock(handle);
|
|
1137
|
-
* }
|
|
1138
|
-
* }
|
|
1139
|
-
* ```
|
|
1140
|
-
*/
|
|
1141
|
-
async releaseDistributedLock(handle) {
|
|
1142
|
-
const lockKey = this._buildKey(`locks/${handle.resource}.lock`);
|
|
1143
|
-
// Verify we still own the lock before deleting
|
|
1144
|
-
const existingObj = await this._bucket.get(lockKey);
|
|
1145
|
-
if (existingObj) {
|
|
1146
|
-
try {
|
|
1147
|
-
const content = JSON.parse(new TextDecoder().decode(new Uint8Array(await existingObj.arrayBuffer())));
|
|
1148
|
-
// Only delete if we own this lock (matching lockId)
|
|
1149
|
-
if (content.lockId === handle.lockId) {
|
|
1150
|
-
await this._bucket.delete(lockKey);
|
|
1151
|
-
}
|
|
1152
|
-
}
|
|
1153
|
-
catch {
|
|
1154
|
-
// Failed to parse, don't delete to avoid corrupting another process's lock
|
|
1155
|
-
}
|
|
1156
|
-
}
|
|
1157
|
-
}
|
|
1158
|
-
/**
|
|
1159
|
-
* Refreshes a distributed lock to extend its TTL.
|
|
1160
|
-
*
|
|
1161
|
-
* @description
|
|
1162
|
-
* Extends the lock's expiration time. Useful for long-running operations
|
|
1163
|
-
* that need to hold the lock longer than the original TTL.
|
|
1164
|
-
*
|
|
1165
|
-
* @param handle - Lock handle to refresh
|
|
1166
|
-
* @param ttlMs - New TTL in milliseconds
|
|
1167
|
-
*
|
|
1168
|
-
* @returns true if refresh succeeded, false if lock was lost
|
|
1169
|
-
*
|
|
1170
|
-
* @example
|
|
1171
|
-
* ```typescript
|
|
1172
|
-
* const handle = await storage.acquireDistributedLock('resource', 30000);
|
|
1173
|
-
* if (handle) {
|
|
1174
|
-
* // Do some work...
|
|
1175
|
-
*
|
|
1176
|
-
* // Extend the lock for another 30 seconds
|
|
1177
|
-
* if (await storage.refreshDistributedLock(handle, 30000)) {
|
|
1178
|
-
* // Continue working
|
|
1179
|
-
* } else {
|
|
1180
|
-
* // Lock was lost, abort operation
|
|
1181
|
-
* }
|
|
1182
|
-
* }
|
|
1183
|
-
* ```
|
|
1184
|
-
*/
|
|
1185
|
-
async refreshDistributedLock(handle, ttlMs = 30000) {
|
|
1186
|
-
const lockKey = this._buildKey(`locks/${handle.resource}.lock`);
|
|
1187
|
-
const now = Date.now();
|
|
1188
|
-
const newExpiresAt = now + ttlMs;
|
|
1189
|
-
// Get current lock to verify ownership
|
|
1190
|
-
const existingObj = await this._bucket.head(lockKey);
|
|
1191
|
-
if (!existingObj) {
|
|
1192
|
-
return false; // Lock doesn't exist
|
|
1193
|
-
}
|
|
1194
|
-
const existingLockObj = await this._bucket.get(lockKey);
|
|
1195
|
-
if (!existingLockObj) {
|
|
1196
|
-
return false;
|
|
1197
|
-
}
|
|
1198
|
-
try {
|
|
1199
|
-
const existingContent = JSON.parse(new TextDecoder().decode(new Uint8Array(await existingLockObj.arrayBuffer())));
|
|
1200
|
-
// Verify we own this lock
|
|
1201
|
-
if (existingContent.lockId !== handle.lockId) {
|
|
1202
|
-
return false; // We don't own this lock
|
|
1203
|
-
}
|
|
1204
|
-
// Create updated lock content
|
|
1205
|
-
const updatedContent = {
|
|
1206
|
-
...existingContent,
|
|
1207
|
-
expiresAt: newExpiresAt
|
|
1208
|
-
};
|
|
1209
|
-
const lockData = new TextEncoder().encode(JSON.stringify(updatedContent));
|
|
1210
|
-
// Update with conditional write using etag
|
|
1211
|
-
try {
|
|
1212
|
-
await this._bucket.put(lockKey, lockData, {
|
|
1213
|
-
onlyIf: { etagMatches: existingObj.etag }
|
|
1214
|
-
});
|
|
1215
|
-
// Update the handle's expiration and etag
|
|
1216
|
-
const newObj = await this._bucket.head(lockKey);
|
|
1217
|
-
if (newObj) {
|
|
1218
|
-
handle.etag = newObj.etag;
|
|
1219
|
-
handle.expiresAt = newExpiresAt;
|
|
1220
|
-
}
|
|
1221
|
-
return true;
|
|
1222
|
-
}
|
|
1223
|
-
catch {
|
|
1224
|
-
// Conditional write failed - lock was modified
|
|
1225
|
-
return false;
|
|
1226
|
-
}
|
|
1227
|
-
}
|
|
1228
|
-
catch {
|
|
1229
|
-
return false;
|
|
1230
|
-
}
|
|
1231
|
-
}
|
|
1232
|
-
/**
|
|
1233
|
-
* Cleans up expired locks from R2 storage.
|
|
1234
|
-
*
|
|
1235
|
-
* @description
|
|
1236
|
-
* Scans all lock files and removes those that have expired.
|
|
1237
|
-
* This should be called periodically to remove stale lock files
|
|
1238
|
-
* left by crashed processes.
|
|
1239
|
-
*
|
|
1240
|
-
* @returns Number of locks cleaned up
|
|
1241
|
-
*
|
|
1242
|
-
* @example
|
|
1243
|
-
* ```typescript
|
|
1244
|
-
* // Run periodically (e.g., every 5 minutes)
|
|
1245
|
-
* const cleaned = await storage.cleanupExpiredLocks();
|
|
1246
|
-
* console.log(`Cleaned up ${cleaned} expired locks`);
|
|
1247
|
-
* ```
|
|
1248
|
-
*/
|
|
1249
|
-
async cleanupExpiredLocks() {
|
|
1250
|
-
const prefix = this._buildKey('locks/');
|
|
1251
|
-
const listResult = await this._bucket.list({ prefix });
|
|
1252
|
-
const now = Date.now();
|
|
1253
|
-
let cleanedCount = 0;
|
|
1254
|
-
for (const obj of listResult.objects) {
|
|
1255
|
-
if (!obj.key.endsWith('.lock'))
|
|
1256
|
-
continue;
|
|
1257
|
-
const lockObj = await this._bucket.get(obj.key);
|
|
1258
|
-
if (lockObj) {
|
|
1259
|
-
try {
|
|
1260
|
-
const content = JSON.parse(new TextDecoder().decode(new Uint8Array(await lockObj.arrayBuffer())));
|
|
1261
|
-
if (content.expiresAt <= now) {
|
|
1262
|
-
// Lock is expired, safe to delete
|
|
1263
|
-
await this._bucket.delete(obj.key);
|
|
1264
|
-
cleanedCount++;
|
|
1265
|
-
}
|
|
1266
|
-
}
|
|
1267
|
-
catch {
|
|
1268
|
-
// Invalid lock file, delete it
|
|
1269
|
-
await this._bucket.delete(obj.key);
|
|
1270
|
-
cleanedCount++;
|
|
1271
|
-
}
|
|
1272
|
-
}
|
|
1273
|
-
}
|
|
1274
|
-
return cleanedCount;
|
|
1275
|
-
}
|
|
1276
|
-
/**
|
|
1277
|
-
* Acquires a lock on a packfile (backward-compatible wrapper).
|
|
1278
|
-
*
|
|
1279
|
-
* @description
|
|
1280
|
-
* High-level API for acquiring a pack lock with optional timeout.
|
|
1281
|
-
* Uses distributed locking with R2 conditional writes internally.
|
|
1282
|
-
*
|
|
1283
|
-
* @param packId - Pack identifier to lock
|
|
1284
|
-
* @param options - Lock acquisition options
|
|
1285
|
-
*
|
|
1286
|
-
* @returns PackLock interface for managing the lock
|
|
1287
|
-
*
|
|
1288
|
-
* @throws {R2PackError} With code 'LOCKED' if lock cannot be acquired
|
|
1289
|
-
*
|
|
1290
|
-
* @example
|
|
1291
|
-
* ```typescript
|
|
1292
|
-
* const lock = await storage.acquireLock(packId, {
|
|
1293
|
-
* timeout: 10000,
|
|
1294
|
-
* ttl: 30000,
|
|
1295
|
-
* holder: 'my-worker'
|
|
1296
|
-
* });
|
|
1297
|
-
*
|
|
1298
|
-
* try {
|
|
1299
|
-
* // Perform pack operations
|
|
1300
|
-
* if (lock.refresh) {
|
|
1301
|
-
* await lock.refresh(); // Extend lock if needed
|
|
1302
|
-
* }
|
|
1303
|
-
* } finally {
|
|
1304
|
-
* await lock.release();
|
|
1305
|
-
* }
|
|
1306
|
-
* ```
|
|
1307
|
-
*/
|
|
1308
|
-
async acquireLock(packId, options) {
|
|
1309
|
-
const ttl = options?.ttl ?? 30000; // Default 30 second TTL
|
|
1310
|
-
const timeout = options?.timeout ?? 0;
|
|
1311
|
-
const startTime = Date.now();
|
|
1312
|
-
// Try to acquire the distributed lock
|
|
1313
|
-
let handle = await this.acquireDistributedLock(packId, ttl, options?.holder);
|
|
1314
|
-
// If timeout is specified, retry until timeout expires
|
|
1315
|
-
if (!handle && timeout > 0) {
|
|
1316
|
-
while (Date.now() - startTime < timeout) {
|
|
1317
|
-
await new Promise(resolve => setTimeout(resolve, 50)); // Wait 50ms between retries
|
|
1318
|
-
handle = await this.acquireDistributedLock(packId, ttl, options?.holder);
|
|
1319
|
-
if (handle)
|
|
1320
|
-
break;
|
|
1321
|
-
}
|
|
1322
|
-
}
|
|
1323
|
-
if (!handle) {
|
|
1324
|
-
if (timeout > 0) {
|
|
1325
|
-
throw new R2PackError(`Lock timeout for packfile: ${packId}`, 'LOCKED', packId);
|
|
1326
|
-
}
|
|
1327
|
-
throw new R2PackError(`Packfile is locked: ${packId}`, 'LOCKED', packId);
|
|
1328
|
-
}
|
|
1329
|
-
// Create the PackLock interface with distributed lock backing
|
|
1330
|
-
const self = this;
|
|
1331
|
-
let released = false;
|
|
1332
|
-
return {
|
|
1333
|
-
packId,
|
|
1334
|
-
handle,
|
|
1335
|
-
isHeld: () => !released && handle.expiresAt > Date.now(),
|
|
1336
|
-
release: async () => {
|
|
1337
|
-
if (!released && handle) {
|
|
1338
|
-
await self.releaseDistributedLock(handle);
|
|
1339
|
-
released = true;
|
|
1340
|
-
}
|
|
1341
|
-
},
|
|
1342
|
-
refresh: async () => {
|
|
1343
|
-
if (released || !handle)
|
|
1344
|
-
return false;
|
|
1345
|
-
return await self.refreshDistributedLock(handle, ttl);
|
|
1346
|
-
}
|
|
1347
|
-
};
|
|
1348
|
-
}
|
|
1349
|
-
}
|
|
1350
|
-
/**
|
|
1351
|
-
* Serializes a multi-pack index to bytes.
|
|
1352
|
-
*
|
|
1353
|
-
* @description
|
|
1354
|
-
* Converts a MultiPackIndex structure to the binary MIDX format.
|
|
1355
|
-
* The format includes:
|
|
1356
|
-
* - MIDX signature (4 bytes)
|
|
1357
|
-
* - Version (4 bytes)
|
|
1358
|
-
* - Pack count (4 bytes)
|
|
1359
|
-
* - Entry count (4 bytes)
|
|
1360
|
-
* - Pack IDs with length prefixes
|
|
1361
|
-
* - Object entries (40 + 4 + 8 = 52 bytes each)
|
|
1362
|
-
* - Checksum (20 bytes)
|
|
1363
|
-
*
|
|
1364
|
-
* @param midx - Multi-pack index to serialize
|
|
1365
|
-
* @returns Serialized MIDX bytes
|
|
1366
|
-
*
|
|
1367
|
-
* @example
|
|
1368
|
-
* ```typescript
|
|
1369
|
-
* const bytes = serializeMultiPackIndex(midx);
|
|
1370
|
-
* await bucket.put('packs/multi-pack-index', bytes);
|
|
1371
|
-
* ```
|
|
1372
|
-
*
|
|
1373
|
-
* @internal
|
|
1374
|
-
*/
|
|
1375
|
-
function serializeMultiPackIndex(midx) {
|
|
1376
|
-
// Calculate size
|
|
1377
|
-
// Header: 4 (signature) + 4 (version) + 4 (packCount) + 4 (entryCount) = 16
|
|
1378
|
-
// Pack IDs: packCount * (4 + packId.length) each with length prefix
|
|
1379
|
-
// Entries: entryCount * (40 + 4 + 8) = 52 bytes each (objectId + packIndex + offset)
|
|
1380
|
-
// Checksum: 20
|
|
1381
|
-
let packIdsSize = 0;
|
|
1382
|
-
for (const packId of midx.packIds) {
|
|
1383
|
-
packIdsSize += 4 + new TextEncoder().encode(packId).length;
|
|
1384
|
-
}
|
|
1385
|
-
const entriesSize = midx.entries.length * 52;
|
|
1386
|
-
const totalSize = 16 + packIdsSize + entriesSize + 20;
|
|
1387
|
-
const data = new Uint8Array(totalSize);
|
|
1388
|
-
const view = new DataView(data.buffer);
|
|
1389
|
-
let offset = 0;
|
|
1390
|
-
// Signature: MIDX
|
|
1391
|
-
data.set(MIDX_SIGNATURE, offset);
|
|
1392
|
-
offset += 4;
|
|
1393
|
-
// Version
|
|
1394
|
-
view.setUint32(offset, midx.version, false);
|
|
1395
|
-
offset += 4;
|
|
1396
|
-
// Pack count
|
|
1397
|
-
view.setUint32(offset, midx.packIds.length, false);
|
|
1398
|
-
offset += 4;
|
|
1399
|
-
// Entry count
|
|
1400
|
-
view.setUint32(offset, midx.entries.length, false);
|
|
1401
|
-
offset += 4;
|
|
1402
|
-
// Pack IDs
|
|
1403
|
-
const encoder = new TextEncoder();
|
|
1404
|
-
for (const packId of midx.packIds) {
|
|
1405
|
-
const encoded = encoder.encode(packId);
|
|
1406
|
-
view.setUint32(offset, encoded.length, false);
|
|
1407
|
-
offset += 4;
|
|
1408
|
-
data.set(encoded, offset);
|
|
1409
|
-
offset += encoded.length;
|
|
1410
|
-
}
|
|
1411
|
-
// Entries
|
|
1412
|
-
for (const entry of midx.entries) {
|
|
1413
|
-
// Object ID (40 hex chars = 20 bytes as hex string, store as 40 bytes)
|
|
1414
|
-
const objIdBytes = encoder.encode(entry.objectId.padEnd(40, '0').slice(0, 40));
|
|
1415
|
-
data.set(objIdBytes, offset);
|
|
1416
|
-
offset += 40;
|
|
1417
|
-
// Pack index
|
|
1418
|
-
view.setUint32(offset, entry.packIndex, false);
|
|
1419
|
-
offset += 4;
|
|
1420
|
-
// Offset (as 64-bit, but we use 32-bit high + 32-bit low)
|
|
1421
|
-
view.setUint32(offset, 0, false); // high bits
|
|
1422
|
-
offset += 4;
|
|
1423
|
-
view.setUint32(offset, entry.offset, false); // low bits
|
|
1424
|
-
offset += 4;
|
|
1425
|
-
}
|
|
1426
|
-
// Checksum
|
|
1427
|
-
data.set(midx.checksum.slice(0, 20), offset);
|
|
1428
|
-
return data;
|
|
1429
|
-
}
|
|
1430
|
-
// Standalone functions
|
|
1431
|
-
/**
|
|
1432
|
-
* Uploads a packfile to R2.
|
|
1433
|
-
*
|
|
1434
|
-
* @description
|
|
1435
|
-
* Standalone function for uploading a packfile. Creates a temporary
|
|
1436
|
-
* R2PackStorage instance internally.
|
|
1437
|
-
*
|
|
1438
|
-
* @param bucket - R2 bucket instance
|
|
1439
|
-
* @param packData - Raw packfile bytes
|
|
1440
|
-
* @param indexData - Pack index file bytes
|
|
1441
|
-
* @param options - Optional configuration including prefix
|
|
1442
|
-
*
|
|
1443
|
-
* @returns Upload result with pack ID, sizes, and checksum
|
|
1444
|
-
*
|
|
1445
|
-
* @throws {R2PackError} If packfile is invalid or upload fails
|
|
1446
|
-
*
|
|
1447
|
-
* @example
|
|
1448
|
-
* ```typescript
|
|
1449
|
-
* const result = await uploadPackfile(bucket, packData, indexData, {
|
|
1450
|
-
* prefix: 'repos/my-repo/'
|
|
1451
|
-
* });
|
|
1452
|
-
* console.log(`Uploaded: ${result.packId}`);
|
|
1453
|
-
* ```
|
|
1454
|
-
*/
|
|
1455
|
-
export async function uploadPackfile(bucket, packData, indexData, options) {
|
|
1456
|
-
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
1457
|
-
return storage.uploadPackfile(packData, indexData);
|
|
1458
|
-
}
|
|
1459
|
-
/**
|
|
1460
|
-
* Downloads a packfile from R2.
|
|
1461
|
-
*
|
|
1462
|
-
* @description
|
|
1463
|
-
* Standalone function for downloading a packfile. Creates a temporary
|
|
1464
|
-
* R2PackStorage instance internally.
|
|
1465
|
-
*
|
|
1466
|
-
* @param bucket - R2 bucket instance
|
|
1467
|
-
* @param packId - Pack identifier to download
|
|
1468
|
-
* @param options - Download options and prefix
|
|
1469
|
-
*
|
|
1470
|
-
* @returns Download result or null if not found
|
|
1471
|
-
*
|
|
1472
|
-
* @throws {R2PackError} If required=true and pack not found, or verification fails
|
|
1473
|
-
*
|
|
1474
|
-
* @example
|
|
1475
|
-
* ```typescript
|
|
1476
|
-
* const result = await downloadPackfile(bucket, packId, {
|
|
1477
|
-
* prefix: 'repos/my-repo/',
|
|
1478
|
-
* verify: true
|
|
1479
|
-
* });
|
|
1480
|
-
* ```
|
|
1481
|
-
*/
|
|
1482
|
-
export async function downloadPackfile(bucket, packId, options) {
|
|
1483
|
-
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
1484
|
-
return storage.downloadPackfile(packId, options);
|
|
1485
|
-
}
|
|
1486
|
-
/**
|
|
1487
|
-
* Gets packfile metadata.
|
|
1488
|
-
*
|
|
1489
|
-
* @description
|
|
1490
|
-
* Standalone function for retrieving packfile metadata without downloading
|
|
1491
|
-
* the full pack.
|
|
1492
|
-
*
|
|
1493
|
-
* @param bucket - R2 bucket instance
|
|
1494
|
-
* @param packId - Pack identifier
|
|
1495
|
-
* @param options - Optional prefix configuration
|
|
1496
|
-
*
|
|
1497
|
-
* @returns Packfile metadata or null if not found
|
|
1498
|
-
*
|
|
1499
|
-
* @example
|
|
1500
|
-
* ```typescript
|
|
1501
|
-
* const metadata = await getPackfileMetadata(bucket, packId);
|
|
1502
|
-
* if (metadata) {
|
|
1503
|
-
* console.log(`Objects: ${metadata.objectCount}`);
|
|
1504
|
-
* }
|
|
1505
|
-
* ```
|
|
1506
|
-
*/
|
|
1507
|
-
export async function getPackfileMetadata(bucket, packId, options) {
|
|
1508
|
-
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
1509
|
-
return storage.getPackfileMetadata(packId);
|
|
1510
|
-
}
|
|
1511
|
-
/**
|
|
1512
|
-
* Lists all packfiles.
|
|
1513
|
-
*
|
|
1514
|
-
* @description
|
|
1515
|
-
* Standalone function for listing packfiles with pagination support.
|
|
1516
|
-
*
|
|
1517
|
-
* @param bucket - R2 bucket instance
|
|
1518
|
-
* @param options - Prefix and pagination options
|
|
1519
|
-
*
|
|
1520
|
-
* @returns Array of packfile metadata
|
|
1521
|
-
*
|
|
1522
|
-
* @example
|
|
1523
|
-
* ```typescript
|
|
1524
|
-
* const packs = await listPackfiles(bucket, {
|
|
1525
|
-
* prefix: 'repos/my-repo/',
|
|
1526
|
-
* limit: 50
|
|
1527
|
-
* });
|
|
1528
|
-
* ```
|
|
1529
|
-
*/
|
|
1530
|
-
export async function listPackfiles(bucket, options) {
|
|
1531
|
-
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
1532
|
-
const result = await storage.listPackfiles({ limit: options?.limit, cursor: options?.cursor });
|
|
1533
|
-
return result.items;
|
|
1534
|
-
}
|
|
1535
|
-
/**
|
|
1536
|
-
* Deletes a packfile.
|
|
1537
|
-
*
|
|
1538
|
-
* @description
|
|
1539
|
-
* Standalone function for deleting a packfile and its associated files.
|
|
1540
|
-
*
|
|
1541
|
-
* @param bucket - R2 bucket instance
|
|
1542
|
-
* @param packId - Pack identifier to delete
|
|
1543
|
-
* @param options - Optional prefix configuration
|
|
1544
|
-
*
|
|
1545
|
-
* @returns true if deleted, false if not found
|
|
1546
|
-
*
|
|
1547
|
-
* @example
|
|
1548
|
-
* ```typescript
|
|
1549
|
-
* if (await deletePackfile(bucket, packId)) {
|
|
1550
|
-
* console.log('Deleted');
|
|
1551
|
-
* }
|
|
1552
|
-
* ```
|
|
1553
|
-
*/
|
|
1554
|
-
export async function deletePackfile(bucket, packId, options) {
|
|
1555
|
-
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
1556
|
-
return storage.deletePackfile(packId);
|
|
1557
|
-
}
|
|
1558
|
-
/**
|
|
1559
|
-
* Creates a multi-pack index from all packfiles in the bucket.
|
|
1560
|
-
*
|
|
1561
|
-
* @description
|
|
1562
|
-
* Standalone function that rebuilds the MIDX and returns the result.
|
|
1563
|
-
*
|
|
1564
|
-
* @param bucket - R2 bucket instance
|
|
1565
|
-
* @param options - Optional prefix configuration
|
|
1566
|
-
*
|
|
1567
|
-
* @returns The newly created multi-pack index
|
|
1568
|
-
*
|
|
1569
|
-
* @example
|
|
1570
|
-
* ```typescript
|
|
1571
|
-
* const midx = await createMultiPackIndex(bucket, { prefix: 'repos/my-repo/' });
|
|
1572
|
-
* console.log(`Indexed ${midx.entries.length} objects`);
|
|
1573
|
-
* ```
|
|
1574
|
-
*/
|
|
1575
|
-
export async function createMultiPackIndex(bucket, options) {
|
|
1576
|
-
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
1577
|
-
await storage.rebuildMultiPackIndex();
|
|
1578
|
-
return storage.getMultiPackIndex();
|
|
1579
|
-
}
|
|
1580
|
-
/**
|
|
1581
|
-
* Parses a multi-pack index from raw bytes.
|
|
1582
|
-
*
|
|
1583
|
-
* @description
|
|
1584
|
-
* Deserializes the binary MIDX format into a MultiPackIndex structure.
|
|
1585
|
-
* Validates the signature and format.
|
|
1586
|
-
*
|
|
1587
|
-
* @param data - Raw MIDX bytes
|
|
1588
|
-
* @returns Parsed multi-pack index
|
|
1589
|
-
*
|
|
1590
|
-
* @throws {R2PackError} With code 'INVALID_DATA' if format is invalid
|
|
1591
|
-
*
|
|
1592
|
-
* @example
|
|
1593
|
-
* ```typescript
|
|
1594
|
-
* const midxData = await bucket.get('packs/multi-pack-index');
|
|
1595
|
-
* if (midxData) {
|
|
1596
|
-
* const midx = parseMultiPackIndex(new Uint8Array(await midxData.arrayBuffer()));
|
|
1597
|
-
* console.log(`Contains ${midx.entries.length} objects`);
|
|
1598
|
-
* }
|
|
1599
|
-
* ```
|
|
1600
|
-
*/
|
|
1601
|
-
export function parseMultiPackIndex(data) {
|
|
1602
|
-
if (data.length < 16) {
|
|
1603
|
-
throw new R2PackError('Multi-pack index too small', 'INVALID_DATA');
|
|
1604
|
-
}
|
|
1605
|
-
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
1606
|
-
let offset = 0;
|
|
1607
|
-
// Check signature
|
|
1608
|
-
for (let i = 0; i < 4; i++) {
|
|
1609
|
-
if (data[i] !== MIDX_SIGNATURE[i]) {
|
|
1610
|
-
throw new R2PackError('Invalid multi-pack index signature', 'INVALID_DATA');
|
|
1611
|
-
}
|
|
1612
|
-
}
|
|
1613
|
-
offset += 4;
|
|
1614
|
-
// Version
|
|
1615
|
-
const version = view.getUint32(offset, false);
|
|
1616
|
-
offset += 4;
|
|
1617
|
-
// Pack count
|
|
1618
|
-
const packCount = view.getUint32(offset, false);
|
|
1619
|
-
offset += 4;
|
|
1620
|
-
// Entry count
|
|
1621
|
-
const entryCount = view.getUint32(offset, false);
|
|
1622
|
-
offset += 4;
|
|
1623
|
-
// Read pack IDs
|
|
1624
|
-
const decoder = new TextDecoder();
|
|
1625
|
-
const packIds = [];
|
|
1626
|
-
for (let i = 0; i < packCount; i++) {
|
|
1627
|
-
const len = view.getUint32(offset, false);
|
|
1628
|
-
offset += 4;
|
|
1629
|
-
const packIdBytes = data.slice(offset, offset + len);
|
|
1630
|
-
packIds.push(decoder.decode(packIdBytes));
|
|
1631
|
-
offset += len;
|
|
1632
|
-
}
|
|
1633
|
-
// Read entries
|
|
1634
|
-
const entries = [];
|
|
1635
|
-
for (let i = 0; i < entryCount; i++) {
|
|
1636
|
-
const objectIdBytes = data.slice(offset, offset + 40);
|
|
1637
|
-
const objectId = decoder.decode(objectIdBytes);
|
|
1638
|
-
offset += 40;
|
|
1639
|
-
const packIndex = view.getUint32(offset, false);
|
|
1640
|
-
offset += 4;
|
|
1641
|
-
// Skip high bits
|
|
1642
|
-
offset += 4;
|
|
1643
|
-
const entryOffset = view.getUint32(offset, false);
|
|
1644
|
-
offset += 4;
|
|
1645
|
-
entries.push({
|
|
1646
|
-
objectId,
|
|
1647
|
-
packIndex,
|
|
1648
|
-
offset: entryOffset
|
|
1649
|
-
});
|
|
1650
|
-
}
|
|
1651
|
-
// Read checksum
|
|
1652
|
-
const checksum = data.slice(offset, offset + 20);
|
|
1653
|
-
return {
|
|
1654
|
-
version,
|
|
1655
|
-
packIds,
|
|
1656
|
-
entries,
|
|
1657
|
-
checksum: new Uint8Array(checksum)
|
|
1658
|
-
};
|
|
1659
|
-
}
|
|
1660
|
-
/**
|
|
1661
|
-
* Looks up an object in the multi-pack index using binary search.
|
|
1662
|
-
*
|
|
1663
|
-
* @description
|
|
1664
|
-
* Efficiently finds an object's location across all packs using O(log n)
|
|
1665
|
-
* binary search on the sorted entries.
|
|
1666
|
-
*
|
|
1667
|
-
* @param midx - Multi-pack index to search
|
|
1668
|
-
* @param objectId - 40-character hex SHA-1 object ID to find
|
|
1669
|
-
*
|
|
1670
|
-
* @returns Entry with pack index and offset, or null if not found
|
|
1671
|
-
*
|
|
1672
|
-
* @example
|
|
1673
|
-
* ```typescript
|
|
1674
|
-
* const midx = await storage.getMultiPackIndex();
|
|
1675
|
-
* const entry = lookupObjectInMultiPack(midx, 'abc123...');
|
|
1676
|
-
* if (entry) {
|
|
1677
|
-
* const packId = midx.packIds[entry.packIndex];
|
|
1678
|
-
* const offset = entry.offset;
|
|
1679
|
-
* console.log(`Found in ${packId} at offset ${offset}`);
|
|
1680
|
-
* }
|
|
1681
|
-
* ```
|
|
1682
|
-
*/
|
|
1683
|
-
export function lookupObjectInMultiPack(midx, objectId) {
|
|
1684
|
-
const entries = midx.entries;
|
|
1685
|
-
if (entries.length === 0) {
|
|
1686
|
-
return null;
|
|
1687
|
-
}
|
|
1688
|
-
// Binary search
|
|
1689
|
-
let left = 0;
|
|
1690
|
-
let right = entries.length - 1;
|
|
1691
|
-
while (left <= right) {
|
|
1692
|
-
const mid = Math.floor((left + right) / 2);
|
|
1693
|
-
const entry = entries[mid];
|
|
1694
|
-
const cmp = objectId.localeCompare(entry.objectId);
|
|
1695
|
-
if (cmp === 0) {
|
|
1696
|
-
return entry;
|
|
1697
|
-
}
|
|
1698
|
-
else if (cmp < 0) {
|
|
1699
|
-
right = mid - 1;
|
|
1700
|
-
}
|
|
1701
|
-
else {
|
|
1702
|
-
left = mid + 1;
|
|
1703
|
-
}
|
|
1704
|
-
}
|
|
1705
|
-
return null;
|
|
1706
|
-
}
|
|
1707
|
-
/**
|
|
1708
|
-
* Acquires a lock on a packfile.
|
|
1709
|
-
*
|
|
1710
|
-
* @description
|
|
1711
|
-
* Standalone function for acquiring a pack lock using distributed locking.
|
|
1712
|
-
*
|
|
1713
|
-
* @param bucket - R2 bucket instance
|
|
1714
|
-
* @param packId - Pack identifier to lock
|
|
1715
|
-
* @param options - Lock options and prefix
|
|
1716
|
-
*
|
|
1717
|
-
* @returns PackLock interface for managing the lock
|
|
1718
|
-
*
|
|
1719
|
-
* @throws {R2PackError} With code 'LOCKED' if lock cannot be acquired
|
|
1720
|
-
*
|
|
1721
|
-
* @example
|
|
1722
|
-
* ```typescript
|
|
1723
|
-
* const lock = await acquirePackLock(bucket, packId, {
|
|
1724
|
-
* prefix: 'repos/my-repo/',
|
|
1725
|
-
* timeout: 10000,
|
|
1726
|
-
* ttl: 30000
|
|
1727
|
-
* });
|
|
1728
|
-
*
|
|
1729
|
-
* try {
|
|
1730
|
-
* // Do work
|
|
1731
|
-
* } finally {
|
|
1732
|
-
* await lock.release();
|
|
1733
|
-
* }
|
|
1734
|
-
* ```
|
|
1735
|
-
*/
|
|
1736
|
-
export async function acquirePackLock(bucket, packId, options) {
|
|
1737
|
-
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
1738
|
-
return storage.acquireLock(packId, options);
|
|
1739
|
-
}
|
|
1740
|
-
/**
|
|
1741
|
-
* Releases a lock on a packfile.
|
|
1742
|
-
*
|
|
1743
|
-
* @description
|
|
1744
|
-
* Standalone function for releasing a pack lock.
|
|
1745
|
-
*
|
|
1746
|
-
* Note: This function requires a valid PackLock with a handle to properly
|
|
1747
|
-
* release distributed locks. For best results, use the lock.release() method
|
|
1748
|
-
* on the PackLock object returned from acquirePackLock.
|
|
1749
|
-
*
|
|
1750
|
-
* @param bucket - R2 bucket instance
|
|
1751
|
-
* @param packId - Pack identifier to unlock
|
|
1752
|
-
* @param options - Optional prefix configuration
|
|
1753
|
-
*
|
|
1754
|
-
* @example
|
|
1755
|
-
* ```typescript
|
|
1756
|
-
* // Preferred: use lock.release()
|
|
1757
|
-
* const lock = await acquirePackLock(bucket, packId);
|
|
1758
|
-
* await lock.release();
|
|
1759
|
-
*
|
|
1760
|
-
* // Alternative: use standalone function (less safe)
|
|
1761
|
-
* await releasePackLock(bucket, packId);
|
|
1762
|
-
* ```
|
|
1763
|
-
*/
|
|
1764
|
-
export async function releasePackLock(bucket, packId, options) {
|
|
1765
|
-
// For backward compatibility, we just delete the lock file directly
|
|
1766
|
-
// This is less safe than using the handle-based release, but works for simple cases
|
|
1767
|
-
const lockKey = buildKey(options?.prefix ?? '', `locks/${packId}.lock`);
|
|
1768
|
-
await bucket.delete(lockKey);
|
|
1769
|
-
}
|
|
1770
|
-
//# sourceMappingURL=r2-pack.js.map
|