gitx.do 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/blame.d.ts +259 -0
- package/dist/cli/commands/blame.d.ts.map +1 -0
- package/dist/cli/commands/blame.js +609 -0
- package/dist/cli/commands/blame.js.map +1 -0
- package/dist/cli/commands/branch.d.ts +249 -0
- package/dist/cli/commands/branch.d.ts.map +1 -0
- package/dist/cli/commands/branch.js +693 -0
- package/dist/cli/commands/branch.js.map +1 -0
- package/dist/cli/commands/commit.d.ts +182 -0
- package/dist/cli/commands/commit.d.ts.map +1 -0
- package/dist/cli/commands/commit.js +437 -0
- package/dist/cli/commands/commit.js.map +1 -0
- package/dist/cli/commands/diff.d.ts +464 -0
- package/dist/cli/commands/diff.d.ts.map +1 -0
- package/dist/cli/commands/diff.js +958 -0
- package/dist/cli/commands/diff.js.map +1 -0
- package/dist/cli/commands/log.d.ts +239 -0
- package/dist/cli/commands/log.d.ts.map +1 -0
- package/dist/cli/commands/log.js +535 -0
- package/dist/cli/commands/log.js.map +1 -0
- package/dist/cli/commands/review.d.ts +457 -0
- package/dist/cli/commands/review.d.ts.map +1 -0
- package/dist/cli/commands/review.js +533 -0
- package/dist/cli/commands/review.js.map +1 -0
- package/dist/cli/commands/status.d.ts +269 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +493 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/commands/web.d.ts +199 -0
- package/dist/cli/commands/web.d.ts.map +1 -0
- package/dist/cli/commands/web.js +696 -0
- package/dist/cli/commands/web.js.map +1 -0
- package/dist/cli/fs-adapter.d.ts +656 -0
- package/dist/cli/fs-adapter.d.ts.map +1 -0
- package/dist/cli/fs-adapter.js +1179 -0
- package/dist/cli/fs-adapter.js.map +1 -0
- package/dist/cli/index.d.ts +387 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +523 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/ui/components/DiffView.d.ts +7 -0
- package/dist/cli/ui/components/DiffView.d.ts.map +1 -0
- package/dist/cli/ui/components/DiffView.js +11 -0
- package/dist/cli/ui/components/DiffView.js.map +1 -0
- package/dist/cli/ui/components/ErrorDisplay.d.ts +6 -0
- package/dist/cli/ui/components/ErrorDisplay.d.ts.map +1 -0
- package/dist/cli/ui/components/ErrorDisplay.js +11 -0
- package/dist/cli/ui/components/ErrorDisplay.js.map +1 -0
- package/dist/cli/ui/components/FuzzySearch.d.ts +9 -0
- package/dist/cli/ui/components/FuzzySearch.d.ts.map +1 -0
- package/dist/cli/ui/components/FuzzySearch.js +12 -0
- package/dist/cli/ui/components/FuzzySearch.js.map +1 -0
- package/dist/cli/ui/components/LoadingSpinner.d.ts +6 -0
- package/dist/cli/ui/components/LoadingSpinner.d.ts.map +1 -0
- package/dist/cli/ui/components/LoadingSpinner.js +10 -0
- package/dist/cli/ui/components/LoadingSpinner.js.map +1 -0
- package/dist/cli/ui/components/NavigationList.d.ts +9 -0
- package/dist/cli/ui/components/NavigationList.d.ts.map +1 -0
- package/dist/cli/ui/components/NavigationList.js +11 -0
- package/dist/cli/ui/components/NavigationList.js.map +1 -0
- package/dist/cli/ui/components/ScrollableContent.d.ts +8 -0
- package/dist/cli/ui/components/ScrollableContent.d.ts.map +1 -0
- package/dist/cli/ui/components/ScrollableContent.js +11 -0
- package/dist/cli/ui/components/ScrollableContent.js.map +1 -0
- package/dist/cli/ui/components/index.d.ts +7 -0
- package/dist/cli/ui/components/index.d.ts.map +1 -0
- package/dist/cli/ui/components/index.js +9 -0
- package/dist/cli/ui/components/index.js.map +1 -0
- package/dist/cli/ui/terminal-ui.d.ts +52 -0
- package/dist/cli/ui/terminal-ui.d.ts.map +1 -0
- package/dist/cli/ui/terminal-ui.js +121 -0
- package/dist/cli/ui/terminal-ui.js.map +1 -0
- package/dist/durable-object/object-store.d.ts +401 -23
- package/dist/durable-object/object-store.d.ts.map +1 -1
- package/dist/durable-object/object-store.js +414 -25
- package/dist/durable-object/object-store.js.map +1 -1
- package/dist/durable-object/schema.d.ts +188 -0
- package/dist/durable-object/schema.d.ts.map +1 -1
- package/dist/durable-object/schema.js +160 -0
- package/dist/durable-object/schema.js.map +1 -1
- package/dist/durable-object/wal.d.ts +336 -31
- package/dist/durable-object/wal.d.ts.map +1 -1
- package/dist/durable-object/wal.js +272 -27
- package/dist/durable-object/wal.js.map +1 -1
- package/dist/index.d.ts +379 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +379 -7
- package/dist/index.js.map +1 -1
- package/dist/mcp/adapter.d.ts +579 -38
- package/dist/mcp/adapter.d.ts.map +1 -1
- package/dist/mcp/adapter.js +426 -33
- package/dist/mcp/adapter.js.map +1 -1
- package/dist/mcp/sandbox.d.ts +532 -29
- package/dist/mcp/sandbox.d.ts.map +1 -1
- package/dist/mcp/sandbox.js +389 -22
- package/dist/mcp/sandbox.js.map +1 -1
- package/dist/mcp/sdk-adapter.d.ts +478 -56
- package/dist/mcp/sdk-adapter.d.ts.map +1 -1
- package/dist/mcp/sdk-adapter.js +346 -44
- package/dist/mcp/sdk-adapter.js.map +1 -1
- package/dist/mcp/tools.d.ts +445 -30
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +363 -33
- package/dist/mcp/tools.js.map +1 -1
- package/dist/ops/blame.d.ts +424 -21
- package/dist/ops/blame.d.ts.map +1 -1
- package/dist/ops/blame.js +303 -20
- package/dist/ops/blame.js.map +1 -1
- package/dist/ops/branch.d.ts +583 -32
- package/dist/ops/branch.d.ts.map +1 -1
- package/dist/ops/branch.js +365 -23
- package/dist/ops/branch.js.map +1 -1
- package/dist/ops/commit-traversal.d.ts +164 -24
- package/dist/ops/commit-traversal.d.ts.map +1 -1
- package/dist/ops/commit-traversal.js +68 -2
- package/dist/ops/commit-traversal.js.map +1 -1
- package/dist/ops/commit.d.ts +387 -53
- package/dist/ops/commit.d.ts.map +1 -1
- package/dist/ops/commit.js +249 -29
- package/dist/ops/commit.js.map +1 -1
- package/dist/ops/merge-base.d.ts +195 -21
- package/dist/ops/merge-base.d.ts.map +1 -1
- package/dist/ops/merge-base.js +122 -12
- package/dist/ops/merge-base.js.map +1 -1
- package/dist/ops/merge.d.ts +600 -130
- package/dist/ops/merge.d.ts.map +1 -1
- package/dist/ops/merge.js +408 -60
- package/dist/ops/merge.js.map +1 -1
- package/dist/ops/tag.d.ts +67 -2
- package/dist/ops/tag.d.ts.map +1 -1
- package/dist/ops/tag.js +42 -1
- package/dist/ops/tag.js.map +1 -1
- package/dist/ops/tree-builder.d.ts +102 -6
- package/dist/ops/tree-builder.d.ts.map +1 -1
- package/dist/ops/tree-builder.js +30 -5
- package/dist/ops/tree-builder.js.map +1 -1
- package/dist/ops/tree-diff.d.ts +50 -2
- package/dist/ops/tree-diff.d.ts.map +1 -1
- package/dist/ops/tree-diff.js +50 -2
- package/dist/ops/tree-diff.js.map +1 -1
- package/dist/pack/delta.d.ts +211 -39
- package/dist/pack/delta.d.ts.map +1 -1
- package/dist/pack/delta.js +232 -46
- package/dist/pack/delta.js.map +1 -1
- package/dist/pack/format.d.ts +390 -28
- package/dist/pack/format.d.ts.map +1 -1
- package/dist/pack/format.js +344 -33
- package/dist/pack/format.js.map +1 -1
- package/dist/pack/full-generation.d.ts +313 -28
- package/dist/pack/full-generation.d.ts.map +1 -1
- package/dist/pack/full-generation.js +238 -19
- package/dist/pack/full-generation.js.map +1 -1
- package/dist/pack/generation.d.ts +346 -23
- package/dist/pack/generation.d.ts.map +1 -1
- package/dist/pack/generation.js +269 -21
- package/dist/pack/generation.js.map +1 -1
- package/dist/pack/index.d.ts +407 -86
- package/dist/pack/index.d.ts.map +1 -1
- package/dist/pack/index.js +351 -70
- package/dist/pack/index.js.map +1 -1
- package/dist/refs/branch.d.ts +517 -71
- package/dist/refs/branch.d.ts.map +1 -1
- package/dist/refs/branch.js +410 -26
- package/dist/refs/branch.js.map +1 -1
- package/dist/refs/storage.d.ts +610 -57
- package/dist/refs/storage.d.ts.map +1 -1
- package/dist/refs/storage.js +481 -29
- package/dist/refs/storage.js.map +1 -1
- package/dist/refs/tag.d.ts +677 -67
- package/dist/refs/tag.d.ts.map +1 -1
- package/dist/refs/tag.js +497 -30
- package/dist/refs/tag.js.map +1 -1
- package/dist/storage/lru-cache.d.ts +556 -53
- package/dist/storage/lru-cache.d.ts.map +1 -1
- package/dist/storage/lru-cache.js +439 -36
- package/dist/storage/lru-cache.js.map +1 -1
- package/dist/storage/object-index.d.ts +483 -38
- package/dist/storage/object-index.d.ts.map +1 -1
- package/dist/storage/object-index.js +388 -22
- package/dist/storage/object-index.js.map +1 -1
- package/dist/storage/r2-pack.d.ts +957 -94
- package/dist/storage/r2-pack.d.ts.map +1 -1
- package/dist/storage/r2-pack.js +756 -48
- package/dist/storage/r2-pack.js.map +1 -1
- package/dist/tiered/cdc-pipeline.d.ts +1610 -38
- package/dist/tiered/cdc-pipeline.d.ts.map +1 -1
- package/dist/tiered/cdc-pipeline.js +1131 -22
- package/dist/tiered/cdc-pipeline.js.map +1 -1
- package/dist/tiered/migration.d.ts +903 -41
- package/dist/tiered/migration.d.ts.map +1 -1
- package/dist/tiered/migration.js +646 -24
- package/dist/tiered/migration.js.map +1 -1
- package/dist/tiered/parquet-writer.d.ts +944 -47
- package/dist/tiered/parquet-writer.d.ts.map +1 -1
- package/dist/tiered/parquet-writer.js +667 -39
- package/dist/tiered/parquet-writer.js.map +1 -1
- package/dist/tiered/read-path.d.ts +728 -34
- package/dist/tiered/read-path.d.ts.map +1 -1
- package/dist/tiered/read-path.js +310 -27
- package/dist/tiered/read-path.js.map +1 -1
- package/dist/types/objects.d.ts +457 -0
- package/dist/types/objects.d.ts.map +1 -1
- package/dist/types/objects.js +305 -4
- package/dist/types/objects.js.map +1 -1
- package/dist/types/storage.d.ts +407 -35
- package/dist/types/storage.d.ts.map +1 -1
- package/dist/types/storage.js +27 -3
- package/dist/types/storage.js.map +1 -1
- package/dist/utils/hash.d.ts +133 -12
- package/dist/utils/hash.d.ts.map +1 -1
- package/dist/utils/hash.js +133 -12
- package/dist/utils/hash.js.map +1 -1
- package/dist/utils/sha1.d.ts +102 -9
- package/dist/utils/sha1.d.ts.map +1 -1
- package/dist/utils/sha1.js +114 -11
- package/dist/utils/sha1.js.map +1 -1
- package/dist/wire/capabilities.d.ts +896 -88
- package/dist/wire/capabilities.d.ts.map +1 -1
- package/dist/wire/capabilities.js +566 -62
- package/dist/wire/capabilities.js.map +1 -1
- package/dist/wire/pkt-line.d.ts +293 -15
- package/dist/wire/pkt-line.d.ts.map +1 -1
- package/dist/wire/pkt-line.js +251 -15
- package/dist/wire/pkt-line.js.map +1 -1
- package/dist/wire/receive-pack.d.ts +814 -64
- package/dist/wire/receive-pack.d.ts.map +1 -1
- package/dist/wire/receive-pack.js +542 -41
- package/dist/wire/receive-pack.js.map +1 -1
- package/dist/wire/smart-http.d.ts +575 -97
- package/dist/wire/smart-http.d.ts.map +1 -1
- package/dist/wire/smart-http.js +337 -46
- package/dist/wire/smart-http.js.map +1 -1
- package/dist/wire/upload-pack.d.ts +492 -98
- package/dist/wire/upload-pack.d.ts.map +1 -1
- package/dist/wire/upload-pack.js +347 -59
- package/dist/wire/upload-pack.js.map +1 -1
- package/package.json +10 -2
package/dist/storage/r2-pack.js
CHANGED
|
@@ -1,19 +1,96 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* R2 Packfile Storage
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* -
|
|
8
|
-
* -
|
|
9
|
-
* -
|
|
2
|
+
* @fileoverview R2 Packfile Storage Module
|
|
3
|
+
*
|
|
4
|
+
* This module manages Git packfiles stored in Cloudflare R2 object storage.
|
|
5
|
+
* It provides comprehensive functionality for:
|
|
6
|
+
*
|
|
7
|
+
* - **Uploading and downloading packfiles** with their indices using atomic operations
|
|
8
|
+
* - **Multi-pack index (MIDX)** for efficient object lookup across multiple packs
|
|
9
|
+
* - **Concurrent access control** with distributed locking using R2 conditional writes
|
|
10
|
+
* - **Pack verification** and integrity checks via SHA-1 checksums
|
|
11
|
+
* - **Atomic uploads** using a manifest-based pattern to ensure data consistency
|
|
12
|
+
*
|
|
13
|
+
* The module implements Git's packfile format (version 2 and 3) and provides
|
|
14
|
+
* both class-based (`R2PackStorage`) and standalone function APIs for flexibility.
|
|
15
|
+
*
|
|
16
|
+
* @module storage/r2-pack
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```typescript
|
|
20
|
+
* // Using the class-based API
|
|
21
|
+
* const storage = new R2PackStorage({
|
|
22
|
+
* bucket: myR2Bucket,
|
|
23
|
+
* prefix: 'repos/my-repo/',
|
|
24
|
+
* cacheSize: 100,
|
|
25
|
+
* cacheTTL: 3600
|
|
26
|
+
* });
|
|
27
|
+
*
|
|
28
|
+
* // Upload a packfile
|
|
29
|
+
* const result = await storage.uploadPackfile(packData, indexData);
|
|
30
|
+
* console.log(`Uploaded pack: ${result.packId}`);
|
|
31
|
+
*
|
|
32
|
+
* // Download with verification
|
|
33
|
+
* const download = await storage.downloadPackfile(result.packId, {
|
|
34
|
+
* verify: true,
|
|
35
|
+
* includeIndex: true
|
|
36
|
+
* });
|
|
37
|
+
* ```
|
|
38
|
+
*
|
|
39
|
+
* @example
|
|
40
|
+
* ```typescript
|
|
41
|
+
* // Using standalone functions
|
|
42
|
+
* const result = await uploadPackfile(bucket, packData, indexData, {
|
|
43
|
+
* prefix: 'repos/my-repo/'
|
|
44
|
+
* });
|
|
45
|
+
*
|
|
46
|
+
* const packfiles = await listPackfiles(bucket, {
|
|
47
|
+
* prefix: 'repos/my-repo/',
|
|
48
|
+
* limit: 10
|
|
49
|
+
* });
|
|
50
|
+
* ```
|
|
10
51
|
*/
|
|
11
52
|
/**
|
|
12
|
-
* Error thrown by R2 pack operations
|
|
53
|
+
* Error thrown by R2 pack operations.
|
|
54
|
+
*
|
|
55
|
+
* @description
|
|
56
|
+
* Custom error class for R2 packfile operations with error codes for
|
|
57
|
+
* programmatic error handling.
|
|
58
|
+
*
|
|
59
|
+
* Error codes:
|
|
60
|
+
* - `NOT_FOUND`: Packfile does not exist
|
|
61
|
+
* - `LOCKED`: Packfile is locked by another process
|
|
62
|
+
* - `INVALID_DATA`: Packfile format is invalid
|
|
63
|
+
* - `CHECKSUM_MISMATCH`: Checksum verification failed
|
|
64
|
+
* - `NETWORK_ERROR`: R2 network/connectivity issue
|
|
65
|
+
*
|
|
66
|
+
* @example
|
|
67
|
+
* ```typescript
|
|
68
|
+
* try {
|
|
69
|
+
* await storage.downloadPackfile(packId, { required: true });
|
|
70
|
+
* } catch (error) {
|
|
71
|
+
* if (error instanceof R2PackError) {
|
|
72
|
+
* switch (error.code) {
|
|
73
|
+
* case 'NOT_FOUND':
|
|
74
|
+
* console.log('Pack does not exist');
|
|
75
|
+
* break;
|
|
76
|
+
* case 'CHECKSUM_MISMATCH':
|
|
77
|
+
* console.log('Pack is corrupted');
|
|
78
|
+
* break;
|
|
79
|
+
* }
|
|
80
|
+
* }
|
|
81
|
+
* }
|
|
82
|
+
* ```
|
|
13
83
|
*/
|
|
14
84
|
export class R2PackError extends Error {
|
|
15
85
|
code;
|
|
16
86
|
packId;
|
|
87
|
+
/**
|
|
88
|
+
* Creates a new R2PackError.
|
|
89
|
+
*
|
|
90
|
+
* @param message - Human-readable error message
|
|
91
|
+
* @param code - Error code for programmatic handling
|
|
92
|
+
* @param packId - Optional pack ID related to the error
|
|
93
|
+
*/
|
|
17
94
|
constructor(message, code, packId) {
|
|
18
95
|
super(message);
|
|
19
96
|
this.code = code;
|
|
@@ -26,7 +103,23 @@ const PACK_SIGNATURE = new Uint8Array([0x50, 0x41, 0x43, 0x4b]);
|
|
|
26
103
|
// Multi-pack index signature
|
|
27
104
|
const MIDX_SIGNATURE = new Uint8Array([0x4d, 0x49, 0x44, 0x58]); // "MIDX"
|
|
28
105
|
/**
|
|
29
|
-
*
|
|
106
|
+
* Validates a packfile header and extracts version and object count.
|
|
107
|
+
*
|
|
108
|
+
* @description
|
|
109
|
+
* Checks that the packfile has a valid PACK signature and supported version (2 or 3).
|
|
110
|
+
*
|
|
111
|
+
* @param data - Raw packfile bytes
|
|
112
|
+
* @returns Object containing version number and object count
|
|
113
|
+
*
|
|
114
|
+
* @throws {R2PackError} With code 'INVALID_DATA' if packfile is invalid
|
|
115
|
+
*
|
|
116
|
+
* @example
|
|
117
|
+
* ```typescript
|
|
118
|
+
* const { version, objectCount } = validatePackfile(packData);
|
|
119
|
+
* console.log(`Pack version ${version} with ${objectCount} objects`);
|
|
120
|
+
* ```
|
|
121
|
+
*
|
|
122
|
+
* @internal
|
|
30
123
|
*/
|
|
31
124
|
function validatePackfile(data) {
|
|
32
125
|
if (data.length < 12) {
|
|
@@ -48,7 +141,21 @@ function validatePackfile(data) {
|
|
|
48
141
|
return { version, objectCount };
|
|
49
142
|
}
|
|
50
143
|
/**
|
|
51
|
-
*
|
|
144
|
+
* Computes SHA-1 checksum of data as a hexadecimal string.
|
|
145
|
+
*
|
|
146
|
+
* @description
|
|
147
|
+
* Uses the Web Crypto API to compute SHA-1 hash for Git compatibility.
|
|
148
|
+
*
|
|
149
|
+
* @param data - Data to hash
|
|
150
|
+
* @returns 40-character lowercase hexadecimal SHA-1 hash
|
|
151
|
+
*
|
|
152
|
+
* @example
|
|
153
|
+
* ```typescript
|
|
154
|
+
* const checksum = await computeChecksum(packData);
|
|
155
|
+
* console.log(`SHA-1: ${checksum}`);
|
|
156
|
+
* ```
|
|
157
|
+
*
|
|
158
|
+
* @internal
|
|
52
159
|
*/
|
|
53
160
|
async function computeChecksum(data) {
|
|
54
161
|
const hashBuffer = await crypto.subtle.digest('SHA-1', data);
|
|
@@ -58,7 +165,20 @@ async function computeChecksum(data) {
|
|
|
58
165
|
.join('');
|
|
59
166
|
}
|
|
60
167
|
/**
|
|
61
|
-
*
|
|
168
|
+
* Generates a unique pack ID.
|
|
169
|
+
*
|
|
170
|
+
* @description
|
|
171
|
+
* Creates a cryptographically random pack identifier in the format 'pack-{16 hex chars}'.
|
|
172
|
+
*
|
|
173
|
+
* @returns Unique pack ID string
|
|
174
|
+
*
|
|
175
|
+
* @example
|
|
176
|
+
* ```typescript
|
|
177
|
+
* const packId = generatePackId();
|
|
178
|
+
* // Returns something like: 'pack-a1b2c3d4e5f67890'
|
|
179
|
+
* ```
|
|
180
|
+
*
|
|
181
|
+
* @internal
|
|
62
182
|
*/
|
|
63
183
|
function generatePackId() {
|
|
64
184
|
const randomBytes = new Uint8Array(8);
|
|
@@ -69,7 +189,22 @@ function generatePackId() {
|
|
|
69
189
|
return `pack-${hex}`;
|
|
70
190
|
}
|
|
71
191
|
/**
|
|
72
|
-
*
|
|
192
|
+
* Builds the full key path with prefix.
|
|
193
|
+
*
|
|
194
|
+
* @description
|
|
195
|
+
* Normalizes the prefix to ensure it has a trailing slash and prepends it to the path.
|
|
196
|
+
*
|
|
197
|
+
* @param prefix - Storage prefix (may or may not have trailing slash)
|
|
198
|
+
* @param path - Path to append to prefix
|
|
199
|
+
* @returns Full key path
|
|
200
|
+
*
|
|
201
|
+
* @example
|
|
202
|
+
* ```typescript
|
|
203
|
+
* buildKey('repos/my-repo', 'packs/pack-123.pack')
|
|
204
|
+
* // Returns: 'repos/my-repo/packs/pack-123.pack'
|
|
205
|
+
* ```
|
|
206
|
+
*
|
|
207
|
+
* @internal
|
|
73
208
|
*/
|
|
74
209
|
function buildKey(prefix, path) {
|
|
75
210
|
if (!prefix) {
|
|
@@ -80,7 +215,14 @@ function buildKey(prefix, path) {
|
|
|
80
215
|
return normalizedPrefix + path;
|
|
81
216
|
}
|
|
82
217
|
/**
|
|
83
|
-
*
|
|
218
|
+
* Generates a unique lock ID.
|
|
219
|
+
*
|
|
220
|
+
* @description
|
|
221
|
+
* Creates a cryptographically random lock identifier (32 hex chars).
|
|
222
|
+
*
|
|
223
|
+
* @returns Unique lock ID string
|
|
224
|
+
*
|
|
225
|
+
* @internal
|
|
84
226
|
*/
|
|
85
227
|
function generateLockId() {
|
|
86
228
|
const randomBytes = new Uint8Array(16);
|
|
@@ -90,7 +232,43 @@ function generateLockId() {
|
|
|
90
232
|
.join('');
|
|
91
233
|
}
|
|
92
234
|
/**
|
|
93
|
-
* R2 Packfile Storage class
|
|
235
|
+
* R2 Packfile Storage class.
|
|
236
|
+
*
|
|
237
|
+
* @description
|
|
238
|
+
* Main class for managing Git packfiles in Cloudflare R2 object storage.
|
|
239
|
+
* Provides methods for uploading, downloading, listing, and managing packfiles
|
|
240
|
+
* with support for atomic uploads, distributed locking, and multi-pack indexing.
|
|
241
|
+
*
|
|
242
|
+
* @example
|
|
243
|
+
* ```typescript
|
|
244
|
+
* // Initialize storage
|
|
245
|
+
* const storage = new R2PackStorage({
|
|
246
|
+
* bucket: env.GIT_BUCKET,
|
|
247
|
+
* prefix: 'repos/my-repo/',
|
|
248
|
+
* cacheSize: 100,
|
|
249
|
+
* cacheTTL: 3600
|
|
250
|
+
* });
|
|
251
|
+
*
|
|
252
|
+
* // Upload a packfile atomically
|
|
253
|
+
* const result = await storage.uploadPackfile(packData, indexData);
|
|
254
|
+
*
|
|
255
|
+
* // Download with verification
|
|
256
|
+
* const download = await storage.downloadPackfile(result.packId, {
|
|
257
|
+
* verify: true,
|
|
258
|
+
* includeIndex: true
|
|
259
|
+
* });
|
|
260
|
+
*
|
|
261
|
+
* // List all packfiles
|
|
262
|
+
* const list = await storage.listPackfiles();
|
|
263
|
+
*
|
|
264
|
+
* // Acquire lock for write operations
|
|
265
|
+
* const lock = await storage.acquireLock(packId, { ttl: 30000 });
|
|
266
|
+
* try {
|
|
267
|
+
* // Perform operations
|
|
268
|
+
* } finally {
|
|
269
|
+
* await lock.release();
|
|
270
|
+
* }
|
|
271
|
+
* ```
|
|
94
272
|
*/
|
|
95
273
|
export class R2PackStorage {
|
|
96
274
|
_bucket;
|
|
@@ -98,6 +276,21 @@ export class R2PackStorage {
|
|
|
98
276
|
_cacheTTL;
|
|
99
277
|
_midxCache = null;
|
|
100
278
|
_indexChecksums = new Map();
|
|
279
|
+
/**
|
|
280
|
+
* Creates a new R2PackStorage instance.
|
|
281
|
+
*
|
|
282
|
+
* @param options - Configuration options for the storage instance
|
|
283
|
+
*
|
|
284
|
+
* @example
|
|
285
|
+
* ```typescript
|
|
286
|
+
* const storage = new R2PackStorage({
|
|
287
|
+
* bucket: env.MY_BUCKET,
|
|
288
|
+
* prefix: 'repos/my-repo/',
|
|
289
|
+
* cacheSize: 100,
|
|
290
|
+
* cacheTTL: 3600
|
|
291
|
+
* });
|
|
292
|
+
* ```
|
|
293
|
+
*/
|
|
101
294
|
constructor(options) {
|
|
102
295
|
this._bucket = options.bucket;
|
|
103
296
|
this._prefix = options.prefix ?? '';
|
|
@@ -108,8 +301,9 @@ export class R2PackStorage {
|
|
|
108
301
|
return buildKey(this._prefix, path);
|
|
109
302
|
}
|
|
110
303
|
/**
|
|
111
|
-
*
|
|
304
|
+
* Uploads a packfile and its index to R2 atomically.
|
|
112
305
|
*
|
|
306
|
+
* @description
|
|
113
307
|
* Uses a manifest-based pattern to ensure atomic uploads:
|
|
114
308
|
* 1. Upload pack and index to staging paths
|
|
115
309
|
* 2. Create manifest in 'staging' status
|
|
@@ -119,6 +313,23 @@ export class R2PackStorage {
|
|
|
119
313
|
*
|
|
120
314
|
* If the process fails at any point, the pack is not considered complete
|
|
121
315
|
* until a valid manifest with status 'complete' exists.
|
|
316
|
+
*
|
|
317
|
+
* @param packData - Raw packfile bytes (must have valid PACK signature)
|
|
318
|
+
* @param indexData - Pack index file bytes
|
|
319
|
+
* @param options - Optional upload configuration
|
|
320
|
+
*
|
|
321
|
+
* @returns Upload result with pack ID, sizes, checksum, and object count
|
|
322
|
+
*
|
|
323
|
+
* @throws {R2PackError} With code 'INVALID_DATA' if packfile is invalid
|
|
324
|
+
* @throws {R2PackError} With code 'NETWORK_ERROR' if bucket is unavailable
|
|
325
|
+
*
|
|
326
|
+
* @example
|
|
327
|
+
* ```typescript
|
|
328
|
+
* const result = await storage.uploadPackfile(packData, indexData);
|
|
329
|
+
* console.log(`Uploaded: ${result.packId}`);
|
|
330
|
+
* console.log(`Objects: ${result.objectCount}`);
|
|
331
|
+
* console.log(`Checksum: ${result.checksum}`);
|
|
332
|
+
* ```
|
|
122
333
|
*/
|
|
123
334
|
async uploadPackfile(packData, indexData, options) {
|
|
124
335
|
if (!this._bucket) {
|
|
@@ -221,7 +432,24 @@ export class R2PackStorage {
|
|
|
221
432
|
}
|
|
222
433
|
}
|
|
223
434
|
/**
|
|
224
|
-
*
|
|
435
|
+
* Gets the manifest for a packfile.
|
|
436
|
+
*
|
|
437
|
+
* @description
|
|
438
|
+
* Retrieves the manifest JSON that tracks the upload status of a packfile.
|
|
439
|
+
* Returns null if no manifest exists (legacy packs or invalid pack ID).
|
|
440
|
+
*
|
|
441
|
+
* @param packId - Pack identifier to get manifest for
|
|
442
|
+
* @returns Pack manifest or null if not found
|
|
443
|
+
*
|
|
444
|
+
* @example
|
|
445
|
+
* ```typescript
|
|
446
|
+
* const manifest = await storage.getPackManifest('pack-abc123');
|
|
447
|
+
* if (manifest?.status === 'complete') {
|
|
448
|
+
* console.log('Pack is ready for use');
|
|
449
|
+
* } else {
|
|
450
|
+
* console.log('Pack upload is incomplete');
|
|
451
|
+
* }
|
|
452
|
+
* ```
|
|
225
453
|
*/
|
|
226
454
|
async getPackManifest(packId) {
|
|
227
455
|
const manifestKey = this._buildKey(`packs/${packId}.manifest`);
|
|
@@ -238,12 +466,23 @@ export class R2PackStorage {
|
|
|
238
466
|
}
|
|
239
467
|
}
|
|
240
468
|
/**
|
|
241
|
-
*
|
|
469
|
+
* Checks if a packfile upload is complete.
|
|
242
470
|
*
|
|
471
|
+
* @description
|
|
243
472
|
* A pack is considered complete if:
|
|
244
473
|
* 1. It has a manifest with status 'complete', OR
|
|
245
474
|
* 2. It was uploaded before the atomic upload feature (legacy packs without manifest)
|
|
246
475
|
* AND both .pack and .idx files exist
|
|
476
|
+
*
|
|
477
|
+
* @param packId - Pack identifier to check
|
|
478
|
+
* @returns true if pack is complete and ready for use
|
|
479
|
+
*
|
|
480
|
+
* @example
|
|
481
|
+
* ```typescript
|
|
482
|
+
* if (await storage.isPackComplete(packId)) {
|
|
483
|
+
* const data = await storage.downloadPackfile(packId);
|
|
484
|
+
* }
|
|
485
|
+
* ```
|
|
247
486
|
*/
|
|
248
487
|
async isPackComplete(packId) {
|
|
249
488
|
// Check for manifest first
|
|
@@ -262,7 +501,34 @@ export class R2PackStorage {
|
|
|
262
501
|
return packExists !== null && idxExists !== null;
|
|
263
502
|
}
|
|
264
503
|
/**
|
|
265
|
-
*
|
|
504
|
+
* Downloads a packfile from R2.
|
|
505
|
+
*
|
|
506
|
+
* @description
|
|
507
|
+
* Downloads pack data with optional index file. Verifies pack completeness
|
|
508
|
+
* before downloading and optionally verifies checksum integrity.
|
|
509
|
+
*
|
|
510
|
+
* @param packId - Pack identifier to download
|
|
511
|
+
* @param options - Download options (includeIndex, verify, byteRange, required)
|
|
512
|
+
*
|
|
513
|
+
* @returns Download result with pack data, or null if not found (unless required=true)
|
|
514
|
+
*
|
|
515
|
+
* @throws {R2PackError} With code 'NOT_FOUND' if required=true and pack not found
|
|
516
|
+
* @throws {R2PackError} With code 'CHECKSUM_MISMATCH' if verify=true and verification fails
|
|
517
|
+
*
|
|
518
|
+
* @example
|
|
519
|
+
* ```typescript
|
|
520
|
+
* // Basic download
|
|
521
|
+
* const result = await storage.downloadPackfile(packId);
|
|
522
|
+
*
|
|
523
|
+
* // Download with verification and index
|
|
524
|
+
* const verified = await storage.downloadPackfile(packId, {
|
|
525
|
+
* verify: true,
|
|
526
|
+
* includeIndex: true
|
|
527
|
+
* });
|
|
528
|
+
*
|
|
529
|
+
* // Required download (throws if not found)
|
|
530
|
+
* const required = await storage.downloadPackfile(packId, { required: true });
|
|
531
|
+
* ```
|
|
266
532
|
*/
|
|
267
533
|
async downloadPackfile(packId, options) {
|
|
268
534
|
// Verify pack completeness before downloading
|
|
@@ -331,7 +597,23 @@ export class R2PackStorage {
|
|
|
331
597
|
return result;
|
|
332
598
|
}
|
|
333
599
|
/**
|
|
334
|
-
*
|
|
600
|
+
* Gets metadata for a packfile.
|
|
601
|
+
*
|
|
602
|
+
* @description
|
|
603
|
+
* Retrieves metadata about a packfile including size, object count,
|
|
604
|
+
* creation time, and checksum without downloading the full pack.
|
|
605
|
+
*
|
|
606
|
+
* @param packId - Pack identifier to get metadata for
|
|
607
|
+
* @returns Packfile metadata or null if not found
|
|
608
|
+
*
|
|
609
|
+
* @example
|
|
610
|
+
* ```typescript
|
|
611
|
+
* const metadata = await storage.getPackfileMetadata(packId);
|
|
612
|
+
* if (metadata) {
|
|
613
|
+
* console.log(`Size: ${metadata.packSize} bytes`);
|
|
614
|
+
* console.log(`Objects: ${metadata.objectCount}`);
|
|
615
|
+
* }
|
|
616
|
+
* ```
|
|
335
617
|
*/
|
|
336
618
|
async getPackfileMetadata(packId) {
|
|
337
619
|
const packKey = this._buildKey(`packs/${packId}.pack`);
|
|
@@ -350,7 +632,25 @@ export class R2PackStorage {
|
|
|
350
632
|
};
|
|
351
633
|
}
|
|
352
634
|
/**
|
|
353
|
-
*
|
|
635
|
+
* Lists all packfiles in storage.
|
|
636
|
+
*
|
|
637
|
+
* @description
|
|
638
|
+
* Returns a paginated list of packfile metadata. Use the cursor for
|
|
639
|
+
* fetching subsequent pages of results.
|
|
640
|
+
*
|
|
641
|
+
* @param options - Pagination options (limit, cursor)
|
|
642
|
+
* @returns List of packfile metadata with optional cursor for pagination
|
|
643
|
+
*
|
|
644
|
+
* @example
|
|
645
|
+
* ```typescript
|
|
646
|
+
* // List first 10 packfiles
|
|
647
|
+
* const first = await storage.listPackfiles({ limit: 10 });
|
|
648
|
+
*
|
|
649
|
+
* // Get next page
|
|
650
|
+
* if (first.cursor) {
|
|
651
|
+
* const next = await storage.listPackfiles({ limit: 10, cursor: first.cursor });
|
|
652
|
+
* }
|
|
653
|
+
* ```
|
|
354
654
|
*/
|
|
355
655
|
async listPackfiles(options) {
|
|
356
656
|
const prefix = this._buildKey('packs/');
|
|
@@ -395,7 +695,23 @@ export class R2PackStorage {
|
|
|
395
695
|
return result;
|
|
396
696
|
}
|
|
397
697
|
/**
|
|
398
|
-
*
|
|
698
|
+
* Deletes a packfile, its index, and manifest.
|
|
699
|
+
*
|
|
700
|
+
* @description
|
|
701
|
+
* Removes all files associated with a packfile and updates the
|
|
702
|
+
* multi-pack index if needed.
|
|
703
|
+
*
|
|
704
|
+
* @param packId - Pack identifier to delete
|
|
705
|
+
* @returns true if pack was deleted, false if it didn't exist
|
|
706
|
+
*
|
|
707
|
+
* @example
|
|
708
|
+
* ```typescript
|
|
709
|
+
* if (await storage.deletePackfile(packId)) {
|
|
710
|
+
* console.log('Pack deleted successfully');
|
|
711
|
+
* } else {
|
|
712
|
+
* console.log('Pack not found');
|
|
713
|
+
* }
|
|
714
|
+
* ```
|
|
399
715
|
*/
|
|
400
716
|
async deletePackfile(packId) {
|
|
401
717
|
const packKey = this._buildKey(`packs/${packId}.pack`);
|
|
@@ -424,7 +740,22 @@ export class R2PackStorage {
|
|
|
424
740
|
return true;
|
|
425
741
|
}
|
|
426
742
|
/**
|
|
427
|
-
*
|
|
743
|
+
* Downloads just the index file for a packfile.
|
|
744
|
+
*
|
|
745
|
+
* @description
|
|
746
|
+
* Retrieves only the pack index file, useful for object lookups
|
|
747
|
+
* without downloading the full packfile.
|
|
748
|
+
*
|
|
749
|
+
* @param packId - Pack identifier to download index for
|
|
750
|
+
* @returns Index data or null if not found
|
|
751
|
+
*
|
|
752
|
+
* @example
|
|
753
|
+
* ```typescript
|
|
754
|
+
* const indexData = await storage.downloadIndex(packId);
|
|
755
|
+
* if (indexData) {
|
|
756
|
+
* // Parse and use the index
|
|
757
|
+
* }
|
|
758
|
+
* ```
|
|
428
759
|
*/
|
|
429
760
|
async downloadIndex(packId) {
|
|
430
761
|
const idxKey = this._buildKey(`packs/${packId}.idx`);
|
|
@@ -435,7 +766,22 @@ export class R2PackStorage {
|
|
|
435
766
|
return new Uint8Array(await idxObj.arrayBuffer());
|
|
436
767
|
}
|
|
437
768
|
/**
|
|
438
|
-
*
|
|
769
|
+
* Uploads a new index for an existing packfile.
|
|
770
|
+
*
|
|
771
|
+
* @description
|
|
772
|
+
* Replaces the index file for an existing packfile. Useful for
|
|
773
|
+
* regenerating corrupted indices or updating index format.
|
|
774
|
+
*
|
|
775
|
+
* @param packId - Pack identifier to upload index for
|
|
776
|
+
* @param indexData - New index file data
|
|
777
|
+
*
|
|
778
|
+
* @throws {R2PackError} With code 'NOT_FOUND' if packfile doesn't exist
|
|
779
|
+
*
|
|
780
|
+
* @example
|
|
781
|
+
* ```typescript
|
|
782
|
+
* const newIndex = generatePackIndex(packData);
|
|
783
|
+
* await storage.uploadIndex(packId, newIndex);
|
|
784
|
+
* ```
|
|
439
785
|
*/
|
|
440
786
|
async uploadIndex(packId, indexData) {
|
|
441
787
|
// Check if pack exists
|
|
@@ -452,7 +798,23 @@ export class R2PackStorage {
|
|
|
452
798
|
this._indexChecksums.set(packId, indexChecksum);
|
|
453
799
|
}
|
|
454
800
|
/**
|
|
455
|
-
*
|
|
801
|
+
* Verifies that an index matches its packfile.
|
|
802
|
+
*
|
|
803
|
+
* @description
|
|
804
|
+
* Compares the current index checksum against the stored checksum
|
|
805
|
+
* to detect corruption or tampering.
|
|
806
|
+
*
|
|
807
|
+
* @param packId - Pack identifier to verify index for
|
|
808
|
+
* @returns true if index is valid, false if missing or corrupted
|
|
809
|
+
*
|
|
810
|
+
* @example
|
|
811
|
+
* ```typescript
|
|
812
|
+
* if (await storage.verifyIndex(packId)) {
|
|
813
|
+
* console.log('Index is valid');
|
|
814
|
+
* } else {
|
|
815
|
+
* console.log('Index needs to be regenerated');
|
|
816
|
+
* }
|
|
817
|
+
* ```
|
|
456
818
|
*/
|
|
457
819
|
async verifyIndex(packId) {
|
|
458
820
|
// Get current index
|
|
@@ -470,8 +832,9 @@ export class R2PackStorage {
|
|
|
470
832
|
return true;
|
|
471
833
|
}
|
|
472
834
|
/**
|
|
473
|
-
*
|
|
835
|
+
* Cleans up orphaned staging files.
|
|
474
836
|
*
|
|
837
|
+
* @description
|
|
475
838
|
* This should be called on startup to clean up any staging files
|
|
476
839
|
* left behind by failed uploads. It will:
|
|
477
840
|
* 1. List all files in the staging directory
|
|
@@ -479,6 +842,15 @@ export class R2PackStorage {
|
|
|
479
842
|
* 3. If not complete, delete the staging files and any partial final files
|
|
480
843
|
*
|
|
481
844
|
* @returns Array of pack IDs that were cleaned up
|
|
845
|
+
*
|
|
846
|
+
* @example
|
|
847
|
+
* ```typescript
|
|
848
|
+
* // Call on worker startup
|
|
849
|
+
* const cleaned = await storage.cleanupOrphanedStagingFiles();
|
|
850
|
+
* if (cleaned.length > 0) {
|
|
851
|
+
* console.log(`Cleaned up ${cleaned.length} orphaned uploads`);
|
|
852
|
+
* }
|
|
853
|
+
* ```
|
|
482
854
|
*/
|
|
483
855
|
async cleanupOrphanedStagingFiles() {
|
|
484
856
|
const stagingPrefix = this._buildKey('staging/');
|
|
@@ -531,7 +903,18 @@ export class R2PackStorage {
|
|
|
531
903
|
return cleanedUp;
|
|
532
904
|
}
|
|
533
905
|
/**
|
|
534
|
-
*
|
|
906
|
+
* Rebuilds the multi-pack index from all packfiles.
|
|
907
|
+
*
|
|
908
|
+
* @description
|
|
909
|
+
* Creates a new MIDX by scanning all packfiles and building a sorted
|
|
910
|
+
* index of all objects. Call this after adding or removing packs.
|
|
911
|
+
*
|
|
912
|
+
* @example
|
|
913
|
+
* ```typescript
|
|
914
|
+
* await storage.rebuildMultiPackIndex();
|
|
915
|
+
* const midx = await storage.getMultiPackIndex();
|
|
916
|
+
* console.log(`Indexed ${midx.entries.length} objects`);
|
|
917
|
+
* ```
|
|
535
918
|
*/
|
|
536
919
|
async rebuildMultiPackIndex() {
|
|
537
920
|
// List all packs
|
|
@@ -577,7 +960,23 @@ export class R2PackStorage {
|
|
|
577
960
|
};
|
|
578
961
|
}
|
|
579
962
|
/**
|
|
580
|
-
*
|
|
963
|
+
* Gets the current multi-pack index.
|
|
964
|
+
*
|
|
965
|
+
* @description
|
|
966
|
+
* Returns the MIDX from cache if available and not expired,
|
|
967
|
+
* otherwise fetches from R2. Returns an empty index if none exists.
|
|
968
|
+
*
|
|
969
|
+
* @returns Current multi-pack index
|
|
970
|
+
*
|
|
971
|
+
* @example
|
|
972
|
+
* ```typescript
|
|
973
|
+
* const midx = await storage.getMultiPackIndex();
|
|
974
|
+
* const entry = lookupObjectInMultiPack(midx, objectSha);
|
|
975
|
+
* if (entry) {
|
|
976
|
+
* const packId = midx.packIds[entry.packIndex];
|
|
977
|
+
* console.log(`Object is in pack ${packId}`);
|
|
978
|
+
* }
|
|
979
|
+
* ```
|
|
581
980
|
*/
|
|
582
981
|
async getMultiPackIndex() {
|
|
583
982
|
// Check cache first
|
|
@@ -605,11 +1004,31 @@ export class R2PackStorage {
|
|
|
605
1004
|
return midx;
|
|
606
1005
|
}
|
|
607
1006
|
/**
|
|
608
|
-
*
|
|
1007
|
+
* Acquires a distributed lock on a resource using R2 conditional writes.
|
|
1008
|
+
*
|
|
1009
|
+
* @description
|
|
1010
|
+
* Uses R2's conditional write feature (ETags) to implement distributed locking.
|
|
1011
|
+
* Locks automatically expire after the TTL to prevent deadlocks.
|
|
1012
|
+
*
|
|
609
1013
|
* @param resource - Resource identifier to lock
|
|
610
|
-
* @param ttlMs - Time-to-live in milliseconds
|
|
1014
|
+
* @param ttlMs - Time-to-live in milliseconds
|
|
611
1015
|
* @param holder - Optional identifier for the lock holder (for debugging)
|
|
1016
|
+
*
|
|
612
1017
|
* @returns LockHandle if acquired, null if lock is held by another process
|
|
1018
|
+
*
|
|
1019
|
+
* @example
|
|
1020
|
+
* ```typescript
|
|
1021
|
+
* const handle = await storage.acquireDistributedLock('my-resource', 30000, 'worker-1');
|
|
1022
|
+
* if (handle) {
|
|
1023
|
+
* try {
|
|
1024
|
+
* // Do work while holding the lock
|
|
1025
|
+
* } finally {
|
|
1026
|
+
* await storage.releaseDistributedLock(handle);
|
|
1027
|
+
* }
|
|
1028
|
+
* } else {
|
|
1029
|
+
* console.log('Could not acquire lock - resource is busy');
|
|
1030
|
+
* }
|
|
1031
|
+
* ```
|
|
613
1032
|
*/
|
|
614
1033
|
async acquireDistributedLock(resource, ttlMs = 30000, holder) {
|
|
615
1034
|
const lockKey = this._buildKey(`locks/${resource}.lock`);
|
|
@@ -699,8 +1118,25 @@ export class R2PackStorage {
|
|
|
699
1118
|
}
|
|
700
1119
|
}
|
|
701
1120
|
/**
|
|
702
|
-
*
|
|
1121
|
+
* Releases a distributed lock.
|
|
1122
|
+
*
|
|
1123
|
+
* @description
|
|
1124
|
+
* Releases the lock only if the caller still owns it (verified by lockId).
|
|
1125
|
+
* Safe to call even if lock has expired or been taken by another process.
|
|
1126
|
+
*
|
|
703
1127
|
* @param handle - Lock handle returned from acquireDistributedLock
|
|
1128
|
+
*
|
|
1129
|
+
* @example
|
|
1130
|
+
* ```typescript
|
|
1131
|
+
* const handle = await storage.acquireDistributedLock('resource');
|
|
1132
|
+
* if (handle) {
|
|
1133
|
+
* try {
|
|
1134
|
+
* // Do work
|
|
1135
|
+
* } finally {
|
|
1136
|
+
* await storage.releaseDistributedLock(handle);
|
|
1137
|
+
* }
|
|
1138
|
+
* }
|
|
1139
|
+
* ```
|
|
704
1140
|
*/
|
|
705
1141
|
async releaseDistributedLock(handle) {
|
|
706
1142
|
const lockKey = this._buildKey(`locks/${handle.resource}.lock`);
|
|
@@ -720,10 +1156,31 @@ export class R2PackStorage {
|
|
|
720
1156
|
}
|
|
721
1157
|
}
|
|
722
1158
|
/**
|
|
723
|
-
*
|
|
1159
|
+
* Refreshes a distributed lock to extend its TTL.
|
|
1160
|
+
*
|
|
1161
|
+
* @description
|
|
1162
|
+
* Extends the lock's expiration time. Useful for long-running operations
|
|
1163
|
+
* that need to hold the lock longer than the original TTL.
|
|
1164
|
+
*
|
|
724
1165
|
* @param handle - Lock handle to refresh
|
|
725
|
-
* @param ttlMs - New TTL in milliseconds
|
|
1166
|
+
* @param ttlMs - New TTL in milliseconds
|
|
1167
|
+
*
|
|
726
1168
|
* @returns true if refresh succeeded, false if lock was lost
|
|
1169
|
+
*
|
|
1170
|
+
* @example
|
|
1171
|
+
* ```typescript
|
|
1172
|
+
* const handle = await storage.acquireDistributedLock('resource', 30000);
|
|
1173
|
+
* if (handle) {
|
|
1174
|
+
* // Do some work...
|
|
1175
|
+
*
|
|
1176
|
+
* // Extend the lock for another 30 seconds
|
|
1177
|
+
* if (await storage.refreshDistributedLock(handle, 30000)) {
|
|
1178
|
+
* // Continue working
|
|
1179
|
+
* } else {
|
|
1180
|
+
* // Lock was lost, abort operation
|
|
1181
|
+
* }
|
|
1182
|
+
* }
|
|
1183
|
+
* ```
|
|
727
1184
|
*/
|
|
728
1185
|
async refreshDistributedLock(handle, ttlMs = 30000) {
|
|
729
1186
|
const lockKey = this._buildKey(`locks/${handle.resource}.lock`);
|
|
@@ -773,9 +1230,21 @@ export class R2PackStorage {
|
|
|
773
1230
|
}
|
|
774
1231
|
}
|
|
775
1232
|
/**
|
|
776
|
-
*
|
|
1233
|
+
* Cleans up expired locks from R2 storage.
|
|
1234
|
+
*
|
|
1235
|
+
* @description
|
|
1236
|
+
* Scans all lock files and removes those that have expired.
|
|
777
1237
|
* This should be called periodically to remove stale lock files
|
|
1238
|
+
* left by crashed processes.
|
|
1239
|
+
*
|
|
778
1240
|
* @returns Number of locks cleaned up
|
|
1241
|
+
*
|
|
1242
|
+
* @example
|
|
1243
|
+
* ```typescript
|
|
1244
|
+
* // Run periodically (e.g., every 5 minutes)
|
|
1245
|
+
* const cleaned = await storage.cleanupExpiredLocks();
|
|
1246
|
+
* console.log(`Cleaned up ${cleaned} expired locks`);
|
|
1247
|
+
* ```
|
|
779
1248
|
*/
|
|
780
1249
|
async cleanupExpiredLocks() {
|
|
781
1250
|
const prefix = this._buildKey('locks/');
|
|
@@ -805,8 +1274,36 @@ export class R2PackStorage {
|
|
|
805
1274
|
return cleanedCount;
|
|
806
1275
|
}
|
|
807
1276
|
/**
|
|
808
|
-
*
|
|
809
|
-
*
|
|
1277
|
+
* Acquires a lock on a packfile (backward-compatible wrapper).
|
|
1278
|
+
*
|
|
1279
|
+
* @description
|
|
1280
|
+
* High-level API for acquiring a pack lock with optional timeout.
|
|
1281
|
+
* Uses distributed locking with R2 conditional writes internally.
|
|
1282
|
+
*
|
|
1283
|
+
* @param packId - Pack identifier to lock
|
|
1284
|
+
* @param options - Lock acquisition options
|
|
1285
|
+
*
|
|
1286
|
+
* @returns PackLock interface for managing the lock
|
|
1287
|
+
*
|
|
1288
|
+
* @throws {R2PackError} With code 'LOCKED' if lock cannot be acquired
|
|
1289
|
+
*
|
|
1290
|
+
* @example
|
|
1291
|
+
* ```typescript
|
|
1292
|
+
* const lock = await storage.acquireLock(packId, {
|
|
1293
|
+
* timeout: 10000,
|
|
1294
|
+
* ttl: 30000,
|
|
1295
|
+
* holder: 'my-worker'
|
|
1296
|
+
* });
|
|
1297
|
+
*
|
|
1298
|
+
* try {
|
|
1299
|
+
* // Perform pack operations
|
|
1300
|
+
* if (lock.refresh) {
|
|
1301
|
+
* await lock.refresh(); // Extend lock if needed
|
|
1302
|
+
* }
|
|
1303
|
+
* } finally {
|
|
1304
|
+
* await lock.release();
|
|
1305
|
+
* }
|
|
1306
|
+
* ```
|
|
810
1307
|
*/
|
|
811
1308
|
async acquireLock(packId, options) {
|
|
812
1309
|
const ttl = options?.ttl ?? 30000; // Default 30 second TTL
|
|
@@ -851,7 +1348,29 @@ export class R2PackStorage {
|
|
|
851
1348
|
}
|
|
852
1349
|
}
|
|
853
1350
|
/**
|
|
854
|
-
*
|
|
1351
|
+
* Serializes a multi-pack index to bytes.
|
|
1352
|
+
*
|
|
1353
|
+
* @description
|
|
1354
|
+
* Converts a MultiPackIndex structure to the binary MIDX format.
|
|
1355
|
+
* The format includes:
|
|
1356
|
+
* - MIDX signature (4 bytes)
|
|
1357
|
+
* - Version (4 bytes)
|
|
1358
|
+
* - Pack count (4 bytes)
|
|
1359
|
+
* - Entry count (4 bytes)
|
|
1360
|
+
* - Pack IDs with length prefixes
|
|
1361
|
+
* - Object entries (40 + 4 + 8 = 52 bytes each)
|
|
1362
|
+
* - Checksum (20 bytes)
|
|
1363
|
+
*
|
|
1364
|
+
* @param midx - Multi-pack index to serialize
|
|
1365
|
+
* @returns Serialized MIDX bytes
|
|
1366
|
+
*
|
|
1367
|
+
* @example
|
|
1368
|
+
* ```typescript
|
|
1369
|
+
* const bytes = serializeMultiPackIndex(midx);
|
|
1370
|
+
* await bucket.put('packs/multi-pack-index', bytes);
|
|
1371
|
+
* ```
|
|
1372
|
+
*
|
|
1373
|
+
* @internal
|
|
855
1374
|
*/
|
|
856
1375
|
function serializeMultiPackIndex(midx) {
|
|
857
1376
|
// Calculate size
|
|
@@ -910,28 +1429,103 @@ function serializeMultiPackIndex(midx) {
|
|
|
910
1429
|
}
|
|
911
1430
|
// Standalone functions
|
|
912
1431
|
/**
|
|
913
|
-
*
|
|
1432
|
+
* Uploads a packfile to R2.
|
|
1433
|
+
*
|
|
1434
|
+
* @description
|
|
1435
|
+
* Standalone function for uploading a packfile. Creates a temporary
|
|
1436
|
+
* R2PackStorage instance internally.
|
|
1437
|
+
*
|
|
1438
|
+
* @param bucket - R2 bucket instance
|
|
1439
|
+
* @param packData - Raw packfile bytes
|
|
1440
|
+
* @param indexData - Pack index file bytes
|
|
1441
|
+
* @param options - Optional configuration including prefix
|
|
1442
|
+
*
|
|
1443
|
+
* @returns Upload result with pack ID, sizes, and checksum
|
|
1444
|
+
*
|
|
1445
|
+
* @throws {R2PackError} If packfile is invalid or upload fails
|
|
1446
|
+
*
|
|
1447
|
+
* @example
|
|
1448
|
+
* ```typescript
|
|
1449
|
+
* const result = await uploadPackfile(bucket, packData, indexData, {
|
|
1450
|
+
* prefix: 'repos/my-repo/'
|
|
1451
|
+
* });
|
|
1452
|
+
* console.log(`Uploaded: ${result.packId}`);
|
|
1453
|
+
* ```
|
|
914
1454
|
*/
|
|
915
1455
|
export async function uploadPackfile(bucket, packData, indexData, options) {
|
|
916
1456
|
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
917
1457
|
return storage.uploadPackfile(packData, indexData);
|
|
918
1458
|
}
|
|
919
1459
|
/**
|
|
920
|
-
*
|
|
1460
|
+
* Downloads a packfile from R2.
|
|
1461
|
+
*
|
|
1462
|
+
* @description
|
|
1463
|
+
* Standalone function for downloading a packfile. Creates a temporary
|
|
1464
|
+
* R2PackStorage instance internally.
|
|
1465
|
+
*
|
|
1466
|
+
* @param bucket - R2 bucket instance
|
|
1467
|
+
* @param packId - Pack identifier to download
|
|
1468
|
+
* @param options - Download options and prefix
|
|
1469
|
+
*
|
|
1470
|
+
* @returns Download result or null if not found
|
|
1471
|
+
*
|
|
1472
|
+
* @throws {R2PackError} If required=true and pack not found, or verification fails
|
|
1473
|
+
*
|
|
1474
|
+
* @example
|
|
1475
|
+
* ```typescript
|
|
1476
|
+
* const result = await downloadPackfile(bucket, packId, {
|
|
1477
|
+
* prefix: 'repos/my-repo/',
|
|
1478
|
+
* verify: true
|
|
1479
|
+
* });
|
|
1480
|
+
* ```
|
|
921
1481
|
*/
|
|
922
1482
|
export async function downloadPackfile(bucket, packId, options) {
|
|
923
1483
|
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
924
1484
|
return storage.downloadPackfile(packId, options);
|
|
925
1485
|
}
|
|
926
1486
|
/**
|
|
927
|
-
*
|
|
1487
|
+
* Gets packfile metadata.
|
|
1488
|
+
*
|
|
1489
|
+
* @description
|
|
1490
|
+
* Standalone function for retrieving packfile metadata without downloading
|
|
1491
|
+
* the full pack.
|
|
1492
|
+
*
|
|
1493
|
+
* @param bucket - R2 bucket instance
|
|
1494
|
+
* @param packId - Pack identifier
|
|
1495
|
+
* @param options - Optional prefix configuration
|
|
1496
|
+
*
|
|
1497
|
+
* @returns Packfile metadata or null if not found
|
|
1498
|
+
*
|
|
1499
|
+
* @example
|
|
1500
|
+
* ```typescript
|
|
1501
|
+
* const metadata = await getPackfileMetadata(bucket, packId);
|
|
1502
|
+
* if (metadata) {
|
|
1503
|
+
* console.log(`Objects: ${metadata.objectCount}`);
|
|
1504
|
+
* }
|
|
1505
|
+
* ```
|
|
928
1506
|
*/
|
|
929
1507
|
export async function getPackfileMetadata(bucket, packId, options) {
|
|
930
1508
|
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
931
1509
|
return storage.getPackfileMetadata(packId);
|
|
932
1510
|
}
|
|
933
1511
|
/**
|
|
934
|
-
*
|
|
1512
|
+
* Lists all packfiles.
|
|
1513
|
+
*
|
|
1514
|
+
* @description
|
|
1515
|
+
* Standalone function for listing packfiles with pagination support.
|
|
1516
|
+
*
|
|
1517
|
+
* @param bucket - R2 bucket instance
|
|
1518
|
+
* @param options - Prefix and pagination options
|
|
1519
|
+
*
|
|
1520
|
+
* @returns Array of packfile metadata
|
|
1521
|
+
*
|
|
1522
|
+
* @example
|
|
1523
|
+
* ```typescript
|
|
1524
|
+
* const packs = await listPackfiles(bucket, {
|
|
1525
|
+
* prefix: 'repos/my-repo/',
|
|
1526
|
+
* limit: 50
|
|
1527
|
+
* });
|
|
1528
|
+
* ```
|
|
935
1529
|
*/
|
|
936
1530
|
export async function listPackfiles(bucket, options) {
|
|
937
1531
|
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
@@ -939,14 +1533,44 @@ export async function listPackfiles(bucket, options) {
|
|
|
939
1533
|
return result.items;
|
|
940
1534
|
}
|
|
941
1535
|
/**
|
|
942
|
-
*
|
|
1536
|
+
* Deletes a packfile.
|
|
1537
|
+
*
|
|
1538
|
+
* @description
|
|
1539
|
+
* Standalone function for deleting a packfile and its associated files.
|
|
1540
|
+
*
|
|
1541
|
+
* @param bucket - R2 bucket instance
|
|
1542
|
+
* @param packId - Pack identifier to delete
|
|
1543
|
+
* @param options - Optional prefix configuration
|
|
1544
|
+
*
|
|
1545
|
+
* @returns true if deleted, false if not found
|
|
1546
|
+
*
|
|
1547
|
+
* @example
|
|
1548
|
+
* ```typescript
|
|
1549
|
+
* if (await deletePackfile(bucket, packId)) {
|
|
1550
|
+
* console.log('Deleted');
|
|
1551
|
+
* }
|
|
1552
|
+
* ```
|
|
943
1553
|
*/
|
|
944
1554
|
export async function deletePackfile(bucket, packId, options) {
|
|
945
1555
|
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
946
1556
|
return storage.deletePackfile(packId);
|
|
947
1557
|
}
|
|
948
1558
|
/**
|
|
949
|
-
*
|
|
1559
|
+
* Creates a multi-pack index from all packfiles in the bucket.
|
|
1560
|
+
*
|
|
1561
|
+
* @description
|
|
1562
|
+
* Standalone function that rebuilds the MIDX and returns the result.
|
|
1563
|
+
*
|
|
1564
|
+
* @param bucket - R2 bucket instance
|
|
1565
|
+
* @param options - Optional prefix configuration
|
|
1566
|
+
*
|
|
1567
|
+
* @returns The newly created multi-pack index
|
|
1568
|
+
*
|
|
1569
|
+
* @example
|
|
1570
|
+
* ```typescript
|
|
1571
|
+
* const midx = await createMultiPackIndex(bucket, { prefix: 'repos/my-repo/' });
|
|
1572
|
+
* console.log(`Indexed ${midx.entries.length} objects`);
|
|
1573
|
+
* ```
|
|
950
1574
|
*/
|
|
951
1575
|
export async function createMultiPackIndex(bucket, options) {
|
|
952
1576
|
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
@@ -954,7 +1578,25 @@ export async function createMultiPackIndex(bucket, options) {
|
|
|
954
1578
|
return storage.getMultiPackIndex();
|
|
955
1579
|
}
|
|
956
1580
|
/**
|
|
957
|
-
*
|
|
1581
|
+
* Parses a multi-pack index from raw bytes.
|
|
1582
|
+
*
|
|
1583
|
+
* @description
|
|
1584
|
+
* Deserializes the binary MIDX format into a MultiPackIndex structure.
|
|
1585
|
+
* Validates the signature and format.
|
|
1586
|
+
*
|
|
1587
|
+
* @param data - Raw MIDX bytes
|
|
1588
|
+
* @returns Parsed multi-pack index
|
|
1589
|
+
*
|
|
1590
|
+
* @throws {R2PackError} With code 'INVALID_DATA' if format is invalid
|
|
1591
|
+
*
|
|
1592
|
+
* @example
|
|
1593
|
+
* ```typescript
|
|
1594
|
+
* const midxData = await bucket.get('packs/multi-pack-index');
|
|
1595
|
+
* if (midxData) {
|
|
1596
|
+
* const midx = parseMultiPackIndex(new Uint8Array(await midxData.arrayBuffer()));
|
|
1597
|
+
* console.log(`Contains ${midx.entries.length} objects`);
|
|
1598
|
+
* }
|
|
1599
|
+
* ```
|
|
958
1600
|
*/
|
|
959
1601
|
export function parseMultiPackIndex(data) {
|
|
960
1602
|
if (data.length < 16) {
|
|
@@ -1016,7 +1658,27 @@ export function parseMultiPackIndex(data) {
|
|
|
1016
1658
|
};
|
|
1017
1659
|
}
|
|
1018
1660
|
/**
|
|
1019
|
-
*
|
|
1661
|
+
* Looks up an object in the multi-pack index using binary search.
|
|
1662
|
+
*
|
|
1663
|
+
* @description
|
|
1664
|
+
* Efficiently finds an object's location across all packs using O(log n)
|
|
1665
|
+
* binary search on the sorted entries.
|
|
1666
|
+
*
|
|
1667
|
+
* @param midx - Multi-pack index to search
|
|
1668
|
+
* @param objectId - 40-character hex SHA-1 object ID to find
|
|
1669
|
+
*
|
|
1670
|
+
* @returns Entry with pack index and offset, or null if not found
|
|
1671
|
+
*
|
|
1672
|
+
* @example
|
|
1673
|
+
* ```typescript
|
|
1674
|
+
* const midx = await storage.getMultiPackIndex();
|
|
1675
|
+
* const entry = lookupObjectInMultiPack(midx, 'abc123...');
|
|
1676
|
+
* if (entry) {
|
|
1677
|
+
* const packId = midx.packIds[entry.packIndex];
|
|
1678
|
+
* const offset = entry.offset;
|
|
1679
|
+
* console.log(`Found in ${packId} at offset ${offset}`);
|
|
1680
|
+
* }
|
|
1681
|
+
* ```
|
|
1020
1682
|
*/
|
|
1021
1683
|
export function lookupObjectInMultiPack(midx, objectId) {
|
|
1022
1684
|
const entries = midx.entries;
|
|
@@ -1043,15 +1705,61 @@ export function lookupObjectInMultiPack(midx, objectId) {
|
|
|
1043
1705
|
return null;
|
|
1044
1706
|
}
|
|
1045
1707
|
/**
|
|
1046
|
-
*
|
|
1708
|
+
* Acquires a lock on a packfile.
|
|
1709
|
+
*
|
|
1710
|
+
* @description
|
|
1711
|
+
* Standalone function for acquiring a pack lock using distributed locking.
|
|
1712
|
+
*
|
|
1713
|
+
* @param bucket - R2 bucket instance
|
|
1714
|
+
* @param packId - Pack identifier to lock
|
|
1715
|
+
* @param options - Lock options and prefix
|
|
1716
|
+
*
|
|
1717
|
+
* @returns PackLock interface for managing the lock
|
|
1718
|
+
*
|
|
1719
|
+
* @throws {R2PackError} With code 'LOCKED' if lock cannot be acquired
|
|
1720
|
+
*
|
|
1721
|
+
* @example
|
|
1722
|
+
* ```typescript
|
|
1723
|
+
* const lock = await acquirePackLock(bucket, packId, {
|
|
1724
|
+
* prefix: 'repos/my-repo/',
|
|
1725
|
+
* timeout: 10000,
|
|
1726
|
+
* ttl: 30000
|
|
1727
|
+
* });
|
|
1728
|
+
*
|
|
1729
|
+
* try {
|
|
1730
|
+
* // Do work
|
|
1731
|
+
* } finally {
|
|
1732
|
+
* await lock.release();
|
|
1733
|
+
* }
|
|
1734
|
+
* ```
|
|
1047
1735
|
*/
|
|
1048
1736
|
export async function acquirePackLock(bucket, packId, options) {
|
|
1049
1737
|
const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
|
|
1050
1738
|
return storage.acquireLock(packId, options);
|
|
1051
1739
|
}
|
|
1052
1740
|
/**
|
|
1053
|
-
*
|
|
1054
|
-
*
|
|
1741
|
+
* Releases a lock on a packfile.
|
|
1742
|
+
*
|
|
1743
|
+
* @description
|
|
1744
|
+
* Standalone function for releasing a pack lock.
|
|
1745
|
+
*
|
|
1746
|
+
* Note: This function requires a valid PackLock with a handle to properly
|
|
1747
|
+
* release distributed locks. For best results, use the lock.release() method
|
|
1748
|
+
* on the PackLock object returned from acquirePackLock.
|
|
1749
|
+
*
|
|
1750
|
+
* @param bucket - R2 bucket instance
|
|
1751
|
+
* @param packId - Pack identifier to unlock
|
|
1752
|
+
* @param options - Optional prefix configuration
|
|
1753
|
+
*
|
|
1754
|
+
* @example
|
|
1755
|
+
* ```typescript
|
|
1756
|
+
* // Preferred: use lock.release()
|
|
1757
|
+
* const lock = await acquirePackLock(bucket, packId);
|
|
1758
|
+
* await lock.release();
|
|
1759
|
+
*
|
|
1760
|
+
* // Alternative: use standalone function (less safe)
|
|
1761
|
+
* await releasePackLock(bucket, packId);
|
|
1762
|
+
* ```
|
|
1055
1763
|
*/
|
|
1056
1764
|
export async function releasePackLock(bucket, packId, options) {
|
|
1057
1765
|
// For backward compatibility, we just delete the lock file directly
|