gitx.do 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (356) hide show
  1. package/README.md +40 -353
  2. package/dist/do/logger.d.ts +50 -0
  3. package/dist/do/logger.d.ts.map +1 -0
  4. package/dist/do/logger.js +122 -0
  5. package/dist/do/logger.js.map +1 -0
  6. package/dist/{durable-object → do}/schema.d.ts +3 -3
  7. package/dist/do/schema.d.ts.map +1 -0
  8. package/dist/{durable-object → do}/schema.js +4 -3
  9. package/dist/do/schema.js.map +1 -0
  10. package/dist/do/types.d.ts +267 -0
  11. package/dist/do/types.d.ts.map +1 -0
  12. package/dist/do/types.js +62 -0
  13. package/dist/do/types.js.map +1 -0
  14. package/dist/index.d.ts +14 -469
  15. package/dist/index.d.ts.map +1 -1
  16. package/dist/index.js +31 -483
  17. package/dist/index.js.map +1 -1
  18. package/package.json +13 -21
  19. package/dist/cli/commands/add.d.ts +0 -176
  20. package/dist/cli/commands/add.d.ts.map +0 -1
  21. package/dist/cli/commands/add.js +0 -979
  22. package/dist/cli/commands/add.js.map +0 -1
  23. package/dist/cli/commands/blame.d.ts +0 -259
  24. package/dist/cli/commands/blame.d.ts.map +0 -1
  25. package/dist/cli/commands/blame.js +0 -609
  26. package/dist/cli/commands/blame.js.map +0 -1
  27. package/dist/cli/commands/branch.d.ts +0 -249
  28. package/dist/cli/commands/branch.d.ts.map +0 -1
  29. package/dist/cli/commands/branch.js +0 -693
  30. package/dist/cli/commands/branch.js.map +0 -1
  31. package/dist/cli/commands/checkout.d.ts +0 -73
  32. package/dist/cli/commands/checkout.d.ts.map +0 -1
  33. package/dist/cli/commands/checkout.js +0 -725
  34. package/dist/cli/commands/checkout.js.map +0 -1
  35. package/dist/cli/commands/commit.d.ts +0 -182
  36. package/dist/cli/commands/commit.d.ts.map +0 -1
  37. package/dist/cli/commands/commit.js +0 -457
  38. package/dist/cli/commands/commit.js.map +0 -1
  39. package/dist/cli/commands/diff.d.ts +0 -464
  40. package/dist/cli/commands/diff.d.ts.map +0 -1
  41. package/dist/cli/commands/diff.js +0 -959
  42. package/dist/cli/commands/diff.js.map +0 -1
  43. package/dist/cli/commands/log.d.ts +0 -239
  44. package/dist/cli/commands/log.d.ts.map +0 -1
  45. package/dist/cli/commands/log.js +0 -535
  46. package/dist/cli/commands/log.js.map +0 -1
  47. package/dist/cli/commands/merge.d.ts +0 -106
  48. package/dist/cli/commands/merge.d.ts.map +0 -1
  49. package/dist/cli/commands/merge.js +0 -852
  50. package/dist/cli/commands/merge.js.map +0 -1
  51. package/dist/cli/commands/review.d.ts +0 -457
  52. package/dist/cli/commands/review.d.ts.map +0 -1
  53. package/dist/cli/commands/review.js +0 -558
  54. package/dist/cli/commands/review.js.map +0 -1
  55. package/dist/cli/commands/stash.d.ts +0 -157
  56. package/dist/cli/commands/stash.d.ts.map +0 -1
  57. package/dist/cli/commands/stash.js +0 -655
  58. package/dist/cli/commands/stash.js.map +0 -1
  59. package/dist/cli/commands/status.d.ts +0 -269
  60. package/dist/cli/commands/status.d.ts.map +0 -1
  61. package/dist/cli/commands/status.js +0 -492
  62. package/dist/cli/commands/status.js.map +0 -1
  63. package/dist/cli/commands/web.d.ts +0 -199
  64. package/dist/cli/commands/web.d.ts.map +0 -1
  65. package/dist/cli/commands/web.js +0 -697
  66. package/dist/cli/commands/web.js.map +0 -1
  67. package/dist/cli/fs-adapter.d.ts +0 -656
  68. package/dist/cli/fs-adapter.d.ts.map +0 -1
  69. package/dist/cli/fs-adapter.js +0 -1177
  70. package/dist/cli/fs-adapter.js.map +0 -1
  71. package/dist/cli/fsx-cli-adapter.d.ts +0 -359
  72. package/dist/cli/fsx-cli-adapter.d.ts.map +0 -1
  73. package/dist/cli/fsx-cli-adapter.js +0 -619
  74. package/dist/cli/fsx-cli-adapter.js.map +0 -1
  75. package/dist/cli/index.d.ts +0 -387
  76. package/dist/cli/index.d.ts.map +0 -1
  77. package/dist/cli/index.js +0 -579
  78. package/dist/cli/index.js.map +0 -1
  79. package/dist/cli/ui/components/DiffView.d.ts +0 -12
  80. package/dist/cli/ui/components/DiffView.d.ts.map +0 -1
  81. package/dist/cli/ui/components/DiffView.js +0 -11
  82. package/dist/cli/ui/components/DiffView.js.map +0 -1
  83. package/dist/cli/ui/components/ErrorDisplay.d.ts +0 -10
  84. package/dist/cli/ui/components/ErrorDisplay.d.ts.map +0 -1
  85. package/dist/cli/ui/components/ErrorDisplay.js +0 -11
  86. package/dist/cli/ui/components/ErrorDisplay.js.map +0 -1
  87. package/dist/cli/ui/components/FuzzySearch.d.ts +0 -15
  88. package/dist/cli/ui/components/FuzzySearch.d.ts.map +0 -1
  89. package/dist/cli/ui/components/FuzzySearch.js +0 -12
  90. package/dist/cli/ui/components/FuzzySearch.js.map +0 -1
  91. package/dist/cli/ui/components/LoadingSpinner.d.ts +0 -10
  92. package/dist/cli/ui/components/LoadingSpinner.d.ts.map +0 -1
  93. package/dist/cli/ui/components/LoadingSpinner.js +0 -10
  94. package/dist/cli/ui/components/LoadingSpinner.js.map +0 -1
  95. package/dist/cli/ui/components/NavigationList.d.ts +0 -14
  96. package/dist/cli/ui/components/NavigationList.d.ts.map +0 -1
  97. package/dist/cli/ui/components/NavigationList.js +0 -11
  98. package/dist/cli/ui/components/NavigationList.js.map +0 -1
  99. package/dist/cli/ui/components/ScrollableContent.d.ts +0 -13
  100. package/dist/cli/ui/components/ScrollableContent.d.ts.map +0 -1
  101. package/dist/cli/ui/components/ScrollableContent.js +0 -11
  102. package/dist/cli/ui/components/ScrollableContent.js.map +0 -1
  103. package/dist/cli/ui/components/index.d.ts +0 -7
  104. package/dist/cli/ui/components/index.d.ts.map +0 -1
  105. package/dist/cli/ui/components/index.js +0 -9
  106. package/dist/cli/ui/components/index.js.map +0 -1
  107. package/dist/cli/ui/terminal-ui.d.ts +0 -85
  108. package/dist/cli/ui/terminal-ui.d.ts.map +0 -1
  109. package/dist/cli/ui/terminal-ui.js +0 -121
  110. package/dist/cli/ui/terminal-ui.js.map +0 -1
  111. package/dist/do/BashModule.d.ts +0 -871
  112. package/dist/do/BashModule.d.ts.map +0 -1
  113. package/dist/do/BashModule.js +0 -1143
  114. package/dist/do/BashModule.js.map +0 -1
  115. package/dist/do/FsModule.d.ts +0 -612
  116. package/dist/do/FsModule.d.ts.map +0 -1
  117. package/dist/do/FsModule.js +0 -1120
  118. package/dist/do/FsModule.js.map +0 -1
  119. package/dist/do/GitModule.d.ts +0 -635
  120. package/dist/do/GitModule.d.ts.map +0 -1
  121. package/dist/do/GitModule.js +0 -784
  122. package/dist/do/GitModule.js.map +0 -1
  123. package/dist/do/GitRepoDO.d.ts +0 -281
  124. package/dist/do/GitRepoDO.d.ts.map +0 -1
  125. package/dist/do/GitRepoDO.js +0 -479
  126. package/dist/do/GitRepoDO.js.map +0 -1
  127. package/dist/do/bash-ast.d.ts +0 -246
  128. package/dist/do/bash-ast.d.ts.map +0 -1
  129. package/dist/do/bash-ast.js +0 -888
  130. package/dist/do/bash-ast.js.map +0 -1
  131. package/dist/do/container-executor.d.ts +0 -491
  132. package/dist/do/container-executor.d.ts.map +0 -1
  133. package/dist/do/container-executor.js +0 -731
  134. package/dist/do/container-executor.js.map +0 -1
  135. package/dist/do/index.d.ts +0 -53
  136. package/dist/do/index.d.ts.map +0 -1
  137. package/dist/do/index.js +0 -91
  138. package/dist/do/index.js.map +0 -1
  139. package/dist/do/tiered-storage.d.ts +0 -403
  140. package/dist/do/tiered-storage.d.ts.map +0 -1
  141. package/dist/do/tiered-storage.js +0 -689
  142. package/dist/do/tiered-storage.js.map +0 -1
  143. package/dist/do/withBash.d.ts +0 -231
  144. package/dist/do/withBash.d.ts.map +0 -1
  145. package/dist/do/withBash.js +0 -244
  146. package/dist/do/withBash.js.map +0 -1
  147. package/dist/do/withFs.d.ts +0 -237
  148. package/dist/do/withFs.d.ts.map +0 -1
  149. package/dist/do/withFs.js +0 -387
  150. package/dist/do/withFs.js.map +0 -1
  151. package/dist/do/withGit.d.ts +0 -180
  152. package/dist/do/withGit.d.ts.map +0 -1
  153. package/dist/do/withGit.js +0 -271
  154. package/dist/do/withGit.js.map +0 -1
  155. package/dist/durable-object/object-store.d.ts +0 -633
  156. package/dist/durable-object/object-store.d.ts.map +0 -1
  157. package/dist/durable-object/object-store.js +0 -1164
  158. package/dist/durable-object/object-store.js.map +0 -1
  159. package/dist/durable-object/schema.d.ts.map +0 -1
  160. package/dist/durable-object/schema.js.map +0 -1
  161. package/dist/durable-object/wal.d.ts +0 -416
  162. package/dist/durable-object/wal.d.ts.map +0 -1
  163. package/dist/durable-object/wal.js +0 -445
  164. package/dist/durable-object/wal.js.map +0 -1
  165. package/dist/mcp/adapter.d.ts +0 -772
  166. package/dist/mcp/adapter.d.ts.map +0 -1
  167. package/dist/mcp/adapter.js +0 -895
  168. package/dist/mcp/adapter.js.map +0 -1
  169. package/dist/mcp/sandbox/miniflare-evaluator.d.ts +0 -22
  170. package/dist/mcp/sandbox/miniflare-evaluator.d.ts.map +0 -1
  171. package/dist/mcp/sandbox/miniflare-evaluator.js +0 -140
  172. package/dist/mcp/sandbox/miniflare-evaluator.js.map +0 -1
  173. package/dist/mcp/sandbox/object-store-proxy.d.ts +0 -32
  174. package/dist/mcp/sandbox/object-store-proxy.d.ts.map +0 -1
  175. package/dist/mcp/sandbox/object-store-proxy.js +0 -30
  176. package/dist/mcp/sandbox/object-store-proxy.js.map +0 -1
  177. package/dist/mcp/sandbox/template.d.ts +0 -17
  178. package/dist/mcp/sandbox/template.d.ts.map +0 -1
  179. package/dist/mcp/sandbox/template.js +0 -71
  180. package/dist/mcp/sandbox/template.js.map +0 -1
  181. package/dist/mcp/sandbox.d.ts +0 -764
  182. package/dist/mcp/sandbox.d.ts.map +0 -1
  183. package/dist/mcp/sandbox.js +0 -1362
  184. package/dist/mcp/sandbox.js.map +0 -1
  185. package/dist/mcp/sdk-adapter.d.ts +0 -835
  186. package/dist/mcp/sdk-adapter.d.ts.map +0 -1
  187. package/dist/mcp/sdk-adapter.js +0 -974
  188. package/dist/mcp/sdk-adapter.js.map +0 -1
  189. package/dist/mcp/tools/do.d.ts +0 -32
  190. package/dist/mcp/tools/do.d.ts.map +0 -1
  191. package/dist/mcp/tools/do.js +0 -117
  192. package/dist/mcp/tools/do.js.map +0 -1
  193. package/dist/mcp/tools.d.ts +0 -548
  194. package/dist/mcp/tools.d.ts.map +0 -1
  195. package/dist/mcp/tools.js +0 -3170
  196. package/dist/mcp/tools.js.map +0 -1
  197. package/dist/ops/blame.d.ts +0 -551
  198. package/dist/ops/blame.d.ts.map +0 -1
  199. package/dist/ops/blame.js +0 -1037
  200. package/dist/ops/blame.js.map +0 -1
  201. package/dist/ops/branch.d.ts +0 -766
  202. package/dist/ops/branch.d.ts.map +0 -1
  203. package/dist/ops/branch.js +0 -950
  204. package/dist/ops/branch.js.map +0 -1
  205. package/dist/ops/commit-traversal.d.ts +0 -349
  206. package/dist/ops/commit-traversal.d.ts.map +0 -1
  207. package/dist/ops/commit-traversal.js +0 -821
  208. package/dist/ops/commit-traversal.js.map +0 -1
  209. package/dist/ops/commit.d.ts +0 -555
  210. package/dist/ops/commit.d.ts.map +0 -1
  211. package/dist/ops/commit.js +0 -826
  212. package/dist/ops/commit.js.map +0 -1
  213. package/dist/ops/merge-base.d.ts +0 -397
  214. package/dist/ops/merge-base.d.ts.map +0 -1
  215. package/dist/ops/merge-base.js +0 -691
  216. package/dist/ops/merge-base.js.map +0 -1
  217. package/dist/ops/merge.d.ts +0 -855
  218. package/dist/ops/merge.d.ts.map +0 -1
  219. package/dist/ops/merge.js +0 -1551
  220. package/dist/ops/merge.js.map +0 -1
  221. package/dist/ops/tag.d.ts +0 -247
  222. package/dist/ops/tag.d.ts.map +0 -1
  223. package/dist/ops/tag.js +0 -649
  224. package/dist/ops/tag.js.map +0 -1
  225. package/dist/ops/tree-builder.d.ts +0 -178
  226. package/dist/ops/tree-builder.d.ts.map +0 -1
  227. package/dist/ops/tree-builder.js +0 -271
  228. package/dist/ops/tree-builder.js.map +0 -1
  229. package/dist/ops/tree-diff.d.ts +0 -291
  230. package/dist/ops/tree-diff.d.ts.map +0 -1
  231. package/dist/ops/tree-diff.js +0 -705
  232. package/dist/ops/tree-diff.js.map +0 -1
  233. package/dist/pack/delta.d.ts +0 -248
  234. package/dist/pack/delta.d.ts.map +0 -1
  235. package/dist/pack/delta.js +0 -740
  236. package/dist/pack/delta.js.map +0 -1
  237. package/dist/pack/format.d.ts +0 -446
  238. package/dist/pack/format.d.ts.map +0 -1
  239. package/dist/pack/format.js +0 -572
  240. package/dist/pack/format.js.map +0 -1
  241. package/dist/pack/full-generation.d.ts +0 -612
  242. package/dist/pack/full-generation.d.ts.map +0 -1
  243. package/dist/pack/full-generation.js +0 -1378
  244. package/dist/pack/full-generation.js.map +0 -1
  245. package/dist/pack/generation.d.ts +0 -441
  246. package/dist/pack/generation.d.ts.map +0 -1
  247. package/dist/pack/generation.js +0 -707
  248. package/dist/pack/generation.js.map +0 -1
  249. package/dist/pack/index.d.ts +0 -502
  250. package/dist/pack/index.d.ts.map +0 -1
  251. package/dist/pack/index.js +0 -833
  252. package/dist/pack/index.js.map +0 -1
  253. package/dist/refs/branch.d.ts +0 -683
  254. package/dist/refs/branch.d.ts.map +0 -1
  255. package/dist/refs/branch.js +0 -881
  256. package/dist/refs/branch.js.map +0 -1
  257. package/dist/refs/storage.d.ts +0 -833
  258. package/dist/refs/storage.d.ts.map +0 -1
  259. package/dist/refs/storage.js +0 -1023
  260. package/dist/refs/storage.js.map +0 -1
  261. package/dist/refs/tag.d.ts +0 -860
  262. package/dist/refs/tag.d.ts.map +0 -1
  263. package/dist/refs/tag.js +0 -996
  264. package/dist/refs/tag.js.map +0 -1
  265. package/dist/storage/backend.d.ts +0 -425
  266. package/dist/storage/backend.d.ts.map +0 -1
  267. package/dist/storage/backend.js +0 -41
  268. package/dist/storage/backend.js.map +0 -1
  269. package/dist/storage/fsx-adapter.d.ts +0 -204
  270. package/dist/storage/fsx-adapter.d.ts.map +0 -1
  271. package/dist/storage/fsx-adapter.js +0 -518
  272. package/dist/storage/fsx-adapter.js.map +0 -1
  273. package/dist/storage/lru-cache.d.ts +0 -691
  274. package/dist/storage/lru-cache.d.ts.map +0 -1
  275. package/dist/storage/lru-cache.js +0 -813
  276. package/dist/storage/lru-cache.js.map +0 -1
  277. package/dist/storage/object-index.d.ts +0 -585
  278. package/dist/storage/object-index.d.ts.map +0 -1
  279. package/dist/storage/object-index.js +0 -532
  280. package/dist/storage/object-index.js.map +0 -1
  281. package/dist/storage/r2-pack.d.ts +0 -1257
  282. package/dist/storage/r2-pack.d.ts.map +0 -1
  283. package/dist/storage/r2-pack.js +0 -1773
  284. package/dist/storage/r2-pack.js.map +0 -1
  285. package/dist/tiered/cdc-pipeline.d.ts +0 -1888
  286. package/dist/tiered/cdc-pipeline.d.ts.map +0 -1
  287. package/dist/tiered/cdc-pipeline.js +0 -1880
  288. package/dist/tiered/cdc-pipeline.js.map +0 -1
  289. package/dist/tiered/migration.d.ts +0 -1104
  290. package/dist/tiered/migration.d.ts.map +0 -1
  291. package/dist/tiered/migration.js +0 -1217
  292. package/dist/tiered/migration.js.map +0 -1
  293. package/dist/tiered/parquet-writer.d.ts +0 -1145
  294. package/dist/tiered/parquet-writer.d.ts.map +0 -1
  295. package/dist/tiered/parquet-writer.js +0 -1183
  296. package/dist/tiered/parquet-writer.js.map +0 -1
  297. package/dist/tiered/read-path.d.ts +0 -835
  298. package/dist/tiered/read-path.d.ts.map +0 -1
  299. package/dist/tiered/read-path.js +0 -487
  300. package/dist/tiered/read-path.js.map +0 -1
  301. package/dist/types/capability.d.ts +0 -1385
  302. package/dist/types/capability.d.ts.map +0 -1
  303. package/dist/types/capability.js +0 -36
  304. package/dist/types/capability.js.map +0 -1
  305. package/dist/types/index.d.ts +0 -13
  306. package/dist/types/index.d.ts.map +0 -1
  307. package/dist/types/index.js +0 -18
  308. package/dist/types/index.js.map +0 -1
  309. package/dist/types/interfaces.d.ts +0 -673
  310. package/dist/types/interfaces.d.ts.map +0 -1
  311. package/dist/types/interfaces.js +0 -26
  312. package/dist/types/interfaces.js.map +0 -1
  313. package/dist/types/objects.d.ts +0 -692
  314. package/dist/types/objects.d.ts.map +0 -1
  315. package/dist/types/objects.js +0 -837
  316. package/dist/types/objects.js.map +0 -1
  317. package/dist/types/storage.d.ts +0 -603
  318. package/dist/types/storage.d.ts.map +0 -1
  319. package/dist/types/storage.js +0 -191
  320. package/dist/types/storage.js.map +0 -1
  321. package/dist/types/worker-loader.d.ts +0 -60
  322. package/dist/types/worker-loader.d.ts.map +0 -1
  323. package/dist/types/worker-loader.js +0 -62
  324. package/dist/types/worker-loader.js.map +0 -1
  325. package/dist/utils/hash.d.ts +0 -198
  326. package/dist/utils/hash.d.ts.map +0 -1
  327. package/dist/utils/hash.js +0 -272
  328. package/dist/utils/hash.js.map +0 -1
  329. package/dist/utils/sha1.d.ts +0 -325
  330. package/dist/utils/sha1.d.ts.map +0 -1
  331. package/dist/utils/sha1.js +0 -635
  332. package/dist/utils/sha1.js.map +0 -1
  333. package/dist/wire/capabilities.d.ts +0 -1044
  334. package/dist/wire/capabilities.d.ts.map +0 -1
  335. package/dist/wire/capabilities.js +0 -941
  336. package/dist/wire/capabilities.js.map +0 -1
  337. package/dist/wire/path-security.d.ts +0 -157
  338. package/dist/wire/path-security.d.ts.map +0 -1
  339. package/dist/wire/path-security.js +0 -307
  340. package/dist/wire/path-security.js.map +0 -1
  341. package/dist/wire/pkt-line.d.ts +0 -345
  342. package/dist/wire/pkt-line.d.ts.map +0 -1
  343. package/dist/wire/pkt-line.js +0 -381
  344. package/dist/wire/pkt-line.js.map +0 -1
  345. package/dist/wire/receive-pack.d.ts +0 -1059
  346. package/dist/wire/receive-pack.d.ts.map +0 -1
  347. package/dist/wire/receive-pack.js +0 -1414
  348. package/dist/wire/receive-pack.js.map +0 -1
  349. package/dist/wire/smart-http.d.ts +0 -799
  350. package/dist/wire/smart-http.d.ts.map +0 -1
  351. package/dist/wire/smart-http.js +0 -945
  352. package/dist/wire/smart-http.js.map +0 -1
  353. package/dist/wire/upload-pack.d.ts +0 -727
  354. package/dist/wire/upload-pack.d.ts.map +0 -1
  355. package/dist/wire/upload-pack.js +0 -1141
  356. package/dist/wire/upload-pack.js.map +0 -1
@@ -1,1773 +0,0 @@
1
- /**
2
- * @fileoverview R2 Packfile Storage Module
3
- *
4
- * This module manages Git packfiles stored in Cloudflare R2 object storage.
5
- * It provides comprehensive functionality for:
6
- *
7
- * - **Uploading and downloading packfiles** with their indices using atomic operations
8
- * - **Multi-pack index (MIDX)** for efficient object lookup across multiple packs
9
- * - **Concurrent access control** with distributed locking using R2 conditional writes
10
- * - **Pack verification** and integrity checks via SHA-1 checksums
11
- * - **Atomic uploads** using a manifest-based pattern to ensure data consistency
12
- *
13
- * The module implements Git's packfile format (version 2 and 3) and provides
14
- * both class-based (`R2PackStorage`) and standalone function APIs for flexibility.
15
- *
16
- * @module storage/r2-pack
17
- *
18
- * @example
19
- * ```typescript
20
- * // Using the class-based API
21
- * const storage = new R2PackStorage({
22
- * bucket: myR2Bucket,
23
- * prefix: 'repos/my-repo/',
24
- * cacheSize: 100,
25
- * cacheTTL: 3600
26
- * });
27
- *
28
- * // Upload a packfile
29
- * const result = await storage.uploadPackfile(packData, indexData);
30
- * console.log(`Uploaded pack: ${result.packId}`);
31
- *
32
- * // Download with verification
33
- * const download = await storage.downloadPackfile(result.packId, {
34
- * verify: true,
35
- * includeIndex: true
36
- * });
37
- * ```
38
- *
39
- * @example
40
- * ```typescript
41
- * // Using standalone functions
42
- * const result = await uploadPackfile(bucket, packData, indexData, {
43
- * prefix: 'repos/my-repo/'
44
- * });
45
- *
46
- * const packfiles = await listPackfiles(bucket, {
47
- * prefix: 'repos/my-repo/',
48
- * limit: 10
49
- * });
50
- * ```
51
- */
52
- /**
53
- * Error thrown by R2 pack operations.
54
- *
55
- * @description
56
- * Custom error class for R2 packfile operations with error codes for
57
- * programmatic error handling.
58
- *
59
- * Error codes:
60
- * - `NOT_FOUND`: Packfile does not exist
61
- * - `LOCKED`: Packfile is locked by another process
62
- * - `INVALID_DATA`: Packfile format is invalid
63
- * - `CHECKSUM_MISMATCH`: Checksum verification failed
64
- * - `NETWORK_ERROR`: R2 network/connectivity issue
65
- *
66
- * @example
67
- * ```typescript
68
- * try {
69
- * await storage.downloadPackfile(packId, { required: true });
70
- * } catch (error) {
71
- * if (error instanceof R2PackError) {
72
- * switch (error.code) {
73
- * case 'NOT_FOUND':
74
- * console.log('Pack does not exist');
75
- * break;
76
- * case 'CHECKSUM_MISMATCH':
77
- * console.log('Pack is corrupted');
78
- * break;
79
- * }
80
- * }
81
- * }
82
- * ```
83
- */
84
- export class R2PackError extends Error {
85
- code;
86
- packId;
87
- /**
88
- * Creates a new R2PackError.
89
- *
90
- * @param message - Human-readable error message
91
- * @param code - Error code for programmatic handling
92
- * @param packId - Optional pack ID related to the error
93
- */
94
- constructor(message, code, packId) {
95
- super(message);
96
- this.code = code;
97
- this.packId = packId;
98
- this.name = 'R2PackError';
99
- }
100
- }
101
- // PACK signature bytes: "PACK"
102
- const PACK_SIGNATURE = new Uint8Array([0x50, 0x41, 0x43, 0x4b]);
103
- // Multi-pack index signature
104
- const MIDX_SIGNATURE = new Uint8Array([0x4d, 0x49, 0x44, 0x58]); // "MIDX"
105
- /**
106
- * Validates a packfile header and extracts version and object count.
107
- *
108
- * @description
109
- * Checks that the packfile has a valid PACK signature and supported version (2 or 3).
110
- *
111
- * @param data - Raw packfile bytes
112
- * @returns Object containing version number and object count
113
- *
114
- * @throws {R2PackError} With code 'INVALID_DATA' if packfile is invalid
115
- *
116
- * @example
117
- * ```typescript
118
- * const { version, objectCount } = validatePackfile(packData);
119
- * console.log(`Pack version ${version} with ${objectCount} objects`);
120
- * ```
121
- *
122
- * @internal
123
- */
124
- function validatePackfile(data) {
125
- if (data.length < 12) {
126
- throw new R2PackError('Packfile too small', 'INVALID_DATA');
127
- }
128
- // Check signature
129
- for (let i = 0; i < 4; i++) {
130
- if (data[i] !== PACK_SIGNATURE[i]) {
131
- throw new R2PackError('Invalid packfile signature', 'INVALID_DATA');
132
- }
133
- }
134
- // Read version (big endian 4 bytes)
135
- const version = (data[4] << 24) | (data[5] << 16) | (data[6] << 8) | data[7];
136
- if (version !== 2 && version !== 3) {
137
- throw new R2PackError(`Unsupported pack version: ${version}`, 'INVALID_DATA');
138
- }
139
- // Read object count (big endian 4 bytes)
140
- const objectCount = (data[8] << 24) | (data[9] << 16) | (data[10] << 8) | data[11];
141
- return { version, objectCount };
142
- }
143
- /**
144
- * Computes SHA-1 checksum of data as a hexadecimal string.
145
- *
146
- * @description
147
- * Uses the Web Crypto API to compute SHA-1 hash for Git compatibility.
148
- *
149
- * @param data - Data to hash
150
- * @returns 40-character lowercase hexadecimal SHA-1 hash
151
- *
152
- * @example
153
- * ```typescript
154
- * const checksum = await computeChecksum(packData);
155
- * console.log(`SHA-1: ${checksum}`);
156
- * ```
157
- *
158
- * @internal
159
- */
160
- async function computeChecksum(data) {
161
- // Create a copy as ArrayBuffer to satisfy BufferSource type
162
- const buffer = new ArrayBuffer(data.length);
163
- new Uint8Array(buffer).set(data);
164
- const hashBuffer = await crypto.subtle.digest('SHA-1', buffer);
165
- const hashArray = new Uint8Array(hashBuffer);
166
- return Array.from(hashArray)
167
- .map(b => b.toString(16).padStart(2, '0'))
168
- .join('');
169
- }
170
- /**
171
- * Generates a unique pack ID.
172
- *
173
- * @description
174
- * Creates a cryptographically random pack identifier in the format 'pack-{16 hex chars}'.
175
- *
176
- * @returns Unique pack ID string
177
- *
178
- * @example
179
- * ```typescript
180
- * const packId = generatePackId();
181
- * // Returns something like: 'pack-a1b2c3d4e5f67890'
182
- * ```
183
- *
184
- * @internal
185
- */
186
- function generatePackId() {
187
- const randomBytes = new Uint8Array(8);
188
- crypto.getRandomValues(randomBytes);
189
- const hex = Array.from(randomBytes)
190
- .map(b => b.toString(16).padStart(2, '0'))
191
- .join('');
192
- return `pack-${hex}`;
193
- }
194
- /**
195
- * Builds the full key path with prefix.
196
- *
197
- * @description
198
- * Normalizes the prefix to ensure it has a trailing slash and prepends it to the path.
199
- *
200
- * @param prefix - Storage prefix (may or may not have trailing slash)
201
- * @param path - Path to append to prefix
202
- * @returns Full key path
203
- *
204
- * @example
205
- * ```typescript
206
- * buildKey('repos/my-repo', 'packs/pack-123.pack')
207
- * // Returns: 'repos/my-repo/packs/pack-123.pack'
208
- * ```
209
- *
210
- * @internal
211
- */
212
- function buildKey(prefix, path) {
213
- if (!prefix) {
214
- return path;
215
- }
216
- // Normalize prefix to have trailing slash
217
- const normalizedPrefix = prefix.endsWith('/') ? prefix : prefix + '/';
218
- return normalizedPrefix + path;
219
- }
220
- /**
221
- * Generates a unique lock ID.
222
- *
223
- * @description
224
- * Creates a cryptographically random lock identifier (32 hex chars).
225
- *
226
- * @returns Unique lock ID string
227
- *
228
- * @internal
229
- */
230
- function generateLockId() {
231
- const randomBytes = new Uint8Array(16);
232
- crypto.getRandomValues(randomBytes);
233
- return Array.from(randomBytes)
234
- .map(b => b.toString(16).padStart(2, '0'))
235
- .join('');
236
- }
237
- /**
238
- * R2 Packfile Storage class.
239
- *
240
- * @description
241
- * Main class for managing Git packfiles in Cloudflare R2 object storage.
242
- * Provides methods for uploading, downloading, listing, and managing packfiles
243
- * with support for atomic uploads, distributed locking, and multi-pack indexing.
244
- *
245
- * @example
246
- * ```typescript
247
- * // Initialize storage
248
- * const storage = new R2PackStorage({
249
- * bucket: env.GIT_BUCKET,
250
- * prefix: 'repos/my-repo/',
251
- * cacheSize: 100,
252
- * cacheTTL: 3600
253
- * });
254
- *
255
- * // Upload a packfile atomically
256
- * const result = await storage.uploadPackfile(packData, indexData);
257
- *
258
- * // Download with verification
259
- * const download = await storage.downloadPackfile(result.packId, {
260
- * verify: true,
261
- * includeIndex: true
262
- * });
263
- *
264
- * // List all packfiles
265
- * const list = await storage.listPackfiles();
266
- *
267
- * // Acquire lock for write operations
268
- * const lock = await storage.acquireLock(packId, { ttl: 30000 });
269
- * try {
270
- * // Perform operations
271
- * } finally {
272
- * await lock.release();
273
- * }
274
- * ```
275
- */
276
- export class R2PackStorage {
277
- _bucket;
278
- _prefix;
279
- _cacheTTL;
280
- _midxCache = null;
281
- _indexChecksums = new Map();
282
- /**
283
- * Creates a new R2PackStorage instance.
284
- *
285
- * @param options - Configuration options for the storage instance
286
- *
287
- * @example
288
- * ```typescript
289
- * const storage = new R2PackStorage({
290
- * bucket: env.MY_BUCKET,
291
- * prefix: 'repos/my-repo/',
292
- * cacheSize: 100,
293
- * cacheTTL: 3600
294
- * });
295
- * ```
296
- */
297
- constructor(options) {
298
- this._bucket = options.bucket;
299
- this._prefix = options.prefix ?? '';
300
- void (options.cacheSize ?? 100); // Reserved for LRU cache implementation
301
- this._cacheTTL = options.cacheTTL ?? 3600;
302
- }
303
- _buildKey(path) {
304
- return buildKey(this._prefix, path);
305
- }
306
- /**
307
- * Uploads a packfile and its index to R2 atomically.
308
- *
309
- * @description
310
- * Uses a manifest-based pattern to ensure atomic uploads:
311
- * 1. Upload pack and index to staging paths
312
- * 2. Create manifest in 'staging' status
313
- * 3. Copy from staging to final location
314
- * 4. Update manifest to 'complete' status
315
- * 5. Clean up staging files
316
- *
317
- * If the process fails at any point, the pack is not considered complete
318
- * until a valid manifest with status 'complete' exists.
319
- *
320
- * @param packData - Raw packfile bytes (must have valid PACK signature)
321
- * @param indexData - Pack index file bytes
322
- * @param options - Optional upload configuration
323
- *
324
- * @returns Upload result with pack ID, sizes, checksum, and object count
325
- *
326
- * @throws {R2PackError} With code 'INVALID_DATA' if packfile is invalid
327
- * @throws {R2PackError} With code 'NETWORK_ERROR' if bucket is unavailable
328
- *
329
- * @example
330
- * ```typescript
331
- * const result = await storage.uploadPackfile(packData, indexData);
332
- * console.log(`Uploaded: ${result.packId}`);
333
- * console.log(`Objects: ${result.objectCount}`);
334
- * console.log(`Checksum: ${result.checksum}`);
335
- * ```
336
- */
337
- async uploadPackfile(packData, indexData, options) {
338
- if (!this._bucket) {
339
- throw new R2PackError('Bucket not available', 'NETWORK_ERROR');
340
- }
341
- // Validate packfile
342
- const { objectCount } = validatePackfile(packData);
343
- // Generate unique pack ID and checksums
344
- const packId = generatePackId();
345
- const packChecksum = await computeChecksum(packData);
346
- const indexChecksum = await computeChecksum(indexData);
347
- const uploadedAt = new Date();
348
- // Store metadata for the files
349
- const metadata = {
350
- packId,
351
- packSize: String(packData.length),
352
- indexSize: String(indexData.length),
353
- objectCount: String(objectCount),
354
- checksum: packChecksum,
355
- createdAt: uploadedAt.toISOString()
356
- };
357
- // If skipAtomic is set, use the simple (non-atomic) upload path
358
- if (options?.skipAtomic) {
359
- const packKey = this._buildKey(`packs/${packId}.pack`);
360
- await this._bucket.put(packKey, packData, { customMetadata: metadata });
361
- const idxKey = this._buildKey(`packs/${packId}.idx`);
362
- await this._bucket.put(idxKey, indexData, { customMetadata: metadata });
363
- this._indexChecksums.set(packId, indexChecksum);
364
- return {
365
- packId,
366
- packSize: packData.length,
367
- indexSize: indexData.length,
368
- checksum: packChecksum,
369
- objectCount,
370
- uploadedAt
371
- };
372
- }
373
- // Step 1: Upload to staging paths
374
- const stagingPackKey = this._buildKey(`staging/${packId}.pack`);
375
- const stagingIdxKey = this._buildKey(`staging/${packId}.idx`);
376
- const manifestKey = this._buildKey(`packs/${packId}.manifest`);
377
- try {
378
- // Upload pack to staging
379
- await this._bucket.put(stagingPackKey, packData, { customMetadata: metadata });
380
- // Upload index to staging
381
- await this._bucket.put(stagingIdxKey, indexData, { customMetadata: metadata });
382
- // Step 2: Create manifest in 'staging' status
383
- const manifest = {
384
- version: 1,
385
- packId,
386
- packChecksum,
387
- indexChecksum,
388
- packSize: packData.length,
389
- indexSize: indexData.length,
390
- objectCount,
391
- completedAt: uploadedAt.toISOString(),
392
- status: 'staging'
393
- };
394
- await this._bucket.put(manifestKey, JSON.stringify(manifest), {
395
- customMetadata: { packId, status: 'staging' }
396
- });
397
- // Step 3: Copy from staging to final location
398
- const packKey = this._buildKey(`packs/${packId}.pack`);
399
- const idxKey = this._buildKey(`packs/${packId}.idx`);
400
- await this._bucket.put(packKey, packData, { customMetadata: metadata });
401
- await this._bucket.put(idxKey, indexData, { customMetadata: metadata });
402
- // Step 4: Update manifest to 'complete' status
403
- manifest.status = 'complete';
404
- await this._bucket.put(manifestKey, JSON.stringify(manifest), {
405
- customMetadata: { packId, status: 'complete' }
406
- });
407
- // Step 5: Clean up staging files
408
- await this._bucket.delete([stagingPackKey, stagingIdxKey]);
409
- // Store index checksum for verification
410
- this._indexChecksums.set(packId, indexChecksum);
411
- return {
412
- packId,
413
- packSize: packData.length,
414
- indexSize: indexData.length,
415
- checksum: packChecksum,
416
- objectCount,
417
- uploadedAt
418
- };
419
- }
420
- catch (error) {
421
- // Clean up any partial uploads on failure
422
- try {
423
- await this._bucket.delete([
424
- stagingPackKey,
425
- stagingIdxKey,
426
- this._buildKey(`packs/${packId}.pack`),
427
- this._buildKey(`packs/${packId}.idx`),
428
- manifestKey
429
- ]);
430
- }
431
- catch {
432
- // Ignore cleanup errors
433
- }
434
- throw error;
435
- }
436
- }
437
- /**
438
- * Gets the manifest for a packfile.
439
- *
440
- * @description
441
- * Retrieves the manifest JSON that tracks the upload status of a packfile.
442
- * Returns null if no manifest exists (legacy packs or invalid pack ID).
443
- *
444
- * @param packId - Pack identifier to get manifest for
445
- * @returns Pack manifest or null if not found
446
- *
447
- * @example
448
- * ```typescript
449
- * const manifest = await storage.getPackManifest('pack-abc123');
450
- * if (manifest?.status === 'complete') {
451
- * console.log('Pack is ready for use');
452
- * } else {
453
- * console.log('Pack upload is incomplete');
454
- * }
455
- * ```
456
- */
457
- async getPackManifest(packId) {
458
- const manifestKey = this._buildKey(`packs/${packId}.manifest`);
459
- const manifestObj = await this._bucket.get(manifestKey);
460
- if (!manifestObj) {
461
- return null;
462
- }
463
- try {
464
- const text = await manifestObj.text();
465
- return JSON.parse(text);
466
- }
467
- catch {
468
- return null;
469
- }
470
- }
471
- /**
472
- * Checks if a packfile upload is complete.
473
- *
474
- * @description
475
- * A pack is considered complete if:
476
- * 1. It has a manifest with status 'complete', OR
477
- * 2. It was uploaded before the atomic upload feature (legacy packs without manifest)
478
- * AND both .pack and .idx files exist
479
- *
480
- * @param packId - Pack identifier to check
481
- * @returns true if pack is complete and ready for use
482
- *
483
- * @example
484
- * ```typescript
485
- * if (await storage.isPackComplete(packId)) {
486
- * const data = await storage.downloadPackfile(packId);
487
- * }
488
- * ```
489
- */
490
- async isPackComplete(packId) {
491
- // Check for manifest first
492
- const manifest = await this.getPackManifest(packId);
493
- if (manifest) {
494
- // If manifest exists, it must have 'complete' status
495
- return manifest.status === 'complete';
496
- }
497
- // Legacy pack without manifest - check if both files exist
498
- const packKey = this._buildKey(`packs/${packId}.pack`);
499
- const idxKey = this._buildKey(`packs/${packId}.idx`);
500
- const [packExists, idxExists] = await Promise.all([
501
- this._bucket.head(packKey),
502
- this._bucket.head(idxKey)
503
- ]);
504
- return packExists !== null && idxExists !== null;
505
- }
506
- /**
507
- * Downloads a packfile from R2.
508
- *
509
- * @description
510
- * Downloads pack data with optional index file. Verifies pack completeness
511
- * before downloading and optionally verifies checksum integrity.
512
- *
513
- * @param packId - Pack identifier to download
514
- * @param options - Download options (includeIndex, verify, byteRange, required)
515
- *
516
- * @returns Download result with pack data, or null if not found (unless required=true)
517
- *
518
- * @throws {R2PackError} With code 'NOT_FOUND' if required=true and pack not found
519
- * @throws {R2PackError} With code 'CHECKSUM_MISMATCH' if verify=true and verification fails
520
- *
521
- * @example
522
- * ```typescript
523
- * // Basic download
524
- * const result = await storage.downloadPackfile(packId);
525
- *
526
- * // Download with verification and index
527
- * const verified = await storage.downloadPackfile(packId, {
528
- * verify: true,
529
- * includeIndex: true
530
- * });
531
- *
532
- * // Required download (throws if not found)
533
- * const required = await storage.downloadPackfile(packId, { required: true });
534
- * ```
535
- */
536
- async downloadPackfile(packId, options) {
537
- // Verify pack completeness before downloading
538
- const isComplete = await this.isPackComplete(packId);
539
- if (!isComplete) {
540
- if (options?.required) {
541
- throw new R2PackError(`Packfile incomplete or not found: ${packId}`, 'NOT_FOUND', packId);
542
- }
543
- return null;
544
- }
545
- const packKey = this._buildKey(`packs/${packId}.pack`);
546
- const packObj = await this._bucket.get(packKey);
547
- if (!packObj) {
548
- if (options?.required) {
549
- throw new R2PackError(`Packfile not found: ${packId}`, 'NOT_FOUND', packId);
550
- }
551
- return null;
552
- }
553
- let packData = new Uint8Array(await packObj.arrayBuffer());
554
- // Verify checksum if requested (before byte range slicing)
555
- if (options?.verify && !options?.byteRange) {
556
- // Get stored checksum from metadata
557
- const headObj = await this._bucket.head(packKey);
558
- const storedChecksum = headObj?.customMetadata?.checksum;
559
- if (storedChecksum) {
560
- const computedChecksum = await computeChecksum(packData);
561
- if (computedChecksum !== storedChecksum) {
562
- throw new R2PackError(`Checksum mismatch for packfile: ${packId}`, 'CHECKSUM_MISMATCH', packId);
563
- }
564
- }
565
- else {
566
- // No stored checksum - data may have been corrupted/replaced
567
- // Verify using the embedded pack checksum (last 20 bytes of packfile)
568
- if (packData.length >= 20) {
569
- const dataWithoutChecksum = packData.slice(0, packData.length - 20);
570
- const computedChecksum = await computeChecksum(dataWithoutChecksum);
571
- const embeddedChecksum = Array.from(packData.slice(packData.length - 20))
572
- .map(b => b.toString(16).padStart(2, '0'))
573
- .join('');
574
- if (computedChecksum !== embeddedChecksum) {
575
- throw new R2PackError(`Checksum mismatch for packfile: ${packId}`, 'CHECKSUM_MISMATCH', packId);
576
- }
577
- }
578
- else {
579
- throw new R2PackError(`Packfile too small to verify: ${packId}`, 'CHECKSUM_MISMATCH', packId);
580
- }
581
- }
582
- }
583
- // Handle byte range
584
- if (options?.byteRange) {
585
- const { start, end } = options.byteRange;
586
- packData = packData.slice(start, end + 1);
587
- }
588
- const result = {
589
- packData,
590
- verified: options?.verify ? true : undefined
591
- };
592
- // Include index if requested
593
- if (options?.includeIndex) {
594
- const idxKey = this._buildKey(`packs/${packId}.idx`);
595
- const idxObj = await this._bucket.get(idxKey);
596
- if (idxObj) {
597
- result.indexData = new Uint8Array(await idxObj.arrayBuffer());
598
- }
599
- }
600
- return result;
601
- }
602
- /**
603
- * Gets metadata for a packfile.
604
- *
605
- * @description
606
- * Retrieves metadata about a packfile including size, object count,
607
- * creation time, and checksum without downloading the full pack.
608
- *
609
- * @param packId - Pack identifier to get metadata for
610
- * @returns Packfile metadata or null if not found
611
- *
612
- * @example
613
- * ```typescript
614
- * const metadata = await storage.getPackfileMetadata(packId);
615
- * if (metadata) {
616
- * console.log(`Size: ${metadata.packSize} bytes`);
617
- * console.log(`Objects: ${metadata.objectCount}`);
618
- * }
619
- * ```
620
- */
621
- async getPackfileMetadata(packId) {
622
- const packKey = this._buildKey(`packs/${packId}.pack`);
623
- const headObj = await this._bucket.head(packKey);
624
- if (!headObj) {
625
- return null;
626
- }
627
- const meta = headObj.customMetadata || {};
628
- return {
629
- packId,
630
- packSize: parseInt(meta.packSize || String(headObj.size), 10),
631
- indexSize: parseInt(meta.indexSize || '0', 10),
632
- objectCount: parseInt(meta.objectCount || '0', 10),
633
- createdAt: new Date(meta.createdAt || Date.now()),
634
- checksum: meta.checksum || ''
635
- };
636
- }
637
- /**
638
- * Lists all packfiles in storage.
639
- *
640
- * @description
641
- * Returns a paginated list of packfile metadata. Use the cursor for
642
- * fetching subsequent pages of results.
643
- *
644
- * @param options - Pagination options (limit, cursor)
645
- * @returns List of packfile metadata with optional cursor for pagination
646
- *
647
- * @example
648
- * ```typescript
649
- * // List first 10 packfiles
650
- * const first = await storage.listPackfiles({ limit: 10 });
651
- *
652
- * // Get next page
653
- * if (first.cursor) {
654
- * const next = await storage.listPackfiles({ limit: 10, cursor: first.cursor });
655
- * }
656
- * ```
657
- */
658
- async listPackfiles(options) {
659
- const prefix = this._buildKey('packs/');
660
- const listResult = await this._bucket.list({ prefix, cursor: options?.cursor });
661
- // Filter for .pack files only
662
- let packFiles = listResult.objects.filter(obj => obj.key.endsWith('.pack'));
663
- // Handle pagination with cursor (cursor is the index to start from)
664
- let startIndex = 0;
665
- if (options?.cursor) {
666
- startIndex = parseInt(options.cursor, 10) || 0;
667
- }
668
- // Slice from cursor position
669
- packFiles = packFiles.slice(startIndex);
670
- // Apply limit
671
- const hasLimit = options?.limit !== undefined && options.limit > 0;
672
- const limitedPackFiles = hasLimit ? packFiles.slice(0, options.limit) : packFiles;
673
- const items = [];
674
- for (const obj of limitedPackFiles) {
675
- // Extract packId from key
676
- const match = obj.key.match(/([^/]+)\.pack$/);
677
- if (match) {
678
- const packId = match[1];
679
- const metadata = await this.getPackfileMetadata(packId);
680
- if (metadata) {
681
- items.push(metadata);
682
- }
683
- }
684
- }
685
- // If no pagination options and no items, return a plain empty array
686
- // This ensures toEqual([]) works as expected
687
- if (items.length === 0 && !options?.limit && !options?.cursor) {
688
- return [];
689
- }
690
- // Create a new array that also has ListPackfilesResult properties
691
- const resultArray = [...items];
692
- const result = resultArray;
693
- result.items = items;
694
- // Set cursor for next page if there are more items
695
- if (hasLimit && packFiles.length > options.limit) {
696
- result.cursor = String(startIndex + options.limit);
697
- }
698
- return result;
699
- }
700
- /**
701
- * Deletes a packfile, its index, and manifest.
702
- *
703
- * @description
704
- * Removes all files associated with a packfile and updates the
705
- * multi-pack index if needed.
706
- *
707
- * @param packId - Pack identifier to delete
708
- * @returns true if pack was deleted, false if it didn't exist
709
- *
710
- * @example
711
- * ```typescript
712
- * if (await storage.deletePackfile(packId)) {
713
- * console.log('Pack deleted successfully');
714
- * } else {
715
- * console.log('Pack not found');
716
- * }
717
- * ```
718
- */
719
- async deletePackfile(packId) {
720
- const packKey = this._buildKey(`packs/${packId}.pack`);
721
- const idxKey = this._buildKey(`packs/${packId}.idx`);
722
- const manifestKey = this._buildKey(`packs/${packId}.manifest`);
723
- // Check if pack exists
724
- const exists = await this._bucket.head(packKey);
725
- if (!exists) {
726
- return false;
727
- }
728
- // Delete pack, index, and manifest atomically
729
- await this._bucket.delete([packKey, idxKey, manifestKey]);
730
- // Clear from index checksum cache
731
- this._indexChecksums.delete(packId);
732
- // Update multi-pack index if it exists
733
- try {
734
- const midx = await this.getMultiPackIndex();
735
- if (midx.packIds.includes(packId)) {
736
- // Rebuild without this pack
737
- await this.rebuildMultiPackIndex();
738
- }
739
- }
740
- catch {
741
- // Ignore errors when updating multi-pack index
742
- }
743
- return true;
744
- }
745
- /**
746
- * Downloads just the index file for a packfile.
747
- *
748
- * @description
749
- * Retrieves only the pack index file, useful for object lookups
750
- * without downloading the full packfile.
751
- *
752
- * @param packId - Pack identifier to download index for
753
- * @returns Index data or null if not found
754
- *
755
- * @example
756
- * ```typescript
757
- * const indexData = await storage.downloadIndex(packId);
758
- * if (indexData) {
759
- * // Parse and use the index
760
- * }
761
- * ```
762
- */
763
- async downloadIndex(packId) {
764
- const idxKey = this._buildKey(`packs/${packId}.idx`);
765
- const idxObj = await this._bucket.get(idxKey);
766
- if (!idxObj) {
767
- return null;
768
- }
769
- return new Uint8Array(await idxObj.arrayBuffer());
770
- }
771
- /**
772
- * Uploads a new index for an existing packfile.
773
- *
774
- * @description
775
- * Replaces the index file for an existing packfile. Useful for
776
- * regenerating corrupted indices or updating index format.
777
- *
778
- * @param packId - Pack identifier to upload index for
779
- * @param indexData - New index file data
780
- *
781
- * @throws {R2PackError} With code 'NOT_FOUND' if packfile doesn't exist
782
- *
783
- * @example
784
- * ```typescript
785
- * const newIndex = generatePackIndex(packData);
786
- * await storage.uploadIndex(packId, newIndex);
787
- * ```
788
- */
789
- async uploadIndex(packId, indexData) {
790
- // Check if pack exists
791
- const packKey = this._buildKey(`packs/${packId}.pack`);
792
- const exists = await this._bucket.head(packKey);
793
- if (!exists) {
794
- throw new R2PackError(`Packfile not found: ${packId}`, 'NOT_FOUND', packId);
795
- }
796
- // Upload new index
797
- const idxKey = this._buildKey(`packs/${packId}.idx`);
798
- await this._bucket.put(idxKey, indexData);
799
- // Update checksum cache
800
- const indexChecksum = await computeChecksum(indexData);
801
- this._indexChecksums.set(packId, indexChecksum);
802
- }
803
- /**
804
- * Verifies that an index matches its packfile.
805
- *
806
- * @description
807
- * Compares the current index checksum against the stored checksum
808
- * to detect corruption or tampering.
809
- *
810
- * @param packId - Pack identifier to verify index for
811
- * @returns true if index is valid, false if missing or corrupted
812
- *
813
- * @example
814
- * ```typescript
815
- * if (await storage.verifyIndex(packId)) {
816
- * console.log('Index is valid');
817
- * } else {
818
- * console.log('Index needs to be regenerated');
819
- * }
820
- * ```
821
- */
822
- async verifyIndex(packId) {
823
- // Get current index
824
- const currentIndex = await this.downloadIndex(packId);
825
- if (!currentIndex) {
826
- return false;
827
- }
828
- // Compare with stored checksum
829
- const storedChecksum = this._indexChecksums.get(packId);
830
- if (storedChecksum) {
831
- const currentChecksum = await computeChecksum(currentIndex);
832
- return currentChecksum === storedChecksum;
833
- }
834
- // If no stored checksum, consider it valid (basic check)
835
- return true;
836
- }
837
- /**
838
- * Cleans up orphaned staging files.
839
- *
840
- * @description
841
- * This should be called on startup to clean up any staging files
842
- * left behind by failed uploads. It will:
843
- * 1. List all files in the staging directory
844
- * 2. For each pack ID found, check if it has a complete manifest
845
- * 3. If not complete, delete the staging files and any partial final files
846
- *
847
- * @returns Array of pack IDs that were cleaned up
848
- *
849
- * @example
850
- * ```typescript
851
- * // Call on worker startup
852
- * const cleaned = await storage.cleanupOrphanedStagingFiles();
853
- * if (cleaned.length > 0) {
854
- * console.log(`Cleaned up ${cleaned.length} orphaned uploads`);
855
- * }
856
- * ```
857
- */
858
- async cleanupOrphanedStagingFiles() {
859
- const stagingPrefix = this._buildKey('staging/');
860
- const listResult = await this._bucket.list({ prefix: stagingPrefix });
861
- // Extract unique pack IDs from staging files
862
- const orphanedPackIds = new Set();
863
- for (const obj of listResult.objects) {
864
- // Extract pack ID from key like "staging/pack-xxx.pack" or "staging/pack-xxx.idx"
865
- const match = obj.key.match(/staging\/([^/]+)\.(pack|idx)$/);
866
- if (match) {
867
- orphanedPackIds.add(match[1]);
868
- }
869
- }
870
- const cleanedUp = [];
871
- for (const packId of orphanedPackIds) {
872
- // Check if this pack is complete
873
- const isComplete = await this.isPackComplete(packId);
874
- if (!isComplete) {
875
- // Pack is incomplete - clean up all related files
876
- const filesToDelete = [
877
- this._buildKey(`staging/${packId}.pack`),
878
- this._buildKey(`staging/${packId}.idx`),
879
- this._buildKey(`packs/${packId}.pack`),
880
- this._buildKey(`packs/${packId}.idx`),
881
- this._buildKey(`packs/${packId}.manifest`)
882
- ];
883
- try {
884
- await this._bucket.delete(filesToDelete);
885
- cleanedUp.push(packId);
886
- }
887
- catch {
888
- // Ignore errors during cleanup
889
- }
890
- }
891
- else {
892
- // Pack is complete - just clean up staging files
893
- const stagingFiles = [
894
- this._buildKey(`staging/${packId}.pack`),
895
- this._buildKey(`staging/${packId}.idx`)
896
- ];
897
- try {
898
- await this._bucket.delete(stagingFiles);
899
- cleanedUp.push(packId);
900
- }
901
- catch {
902
- // Ignore errors during cleanup
903
- }
904
- }
905
- }
906
- return cleanedUp;
907
- }
908
- /**
909
- * Rebuilds the multi-pack index from all packfiles.
910
- *
911
- * @description
912
- * Creates a new MIDX by scanning all packfiles and building a sorted
913
- * index of all objects. Call this after adding or removing packs.
914
- *
915
- * @example
916
- * ```typescript
917
- * await storage.rebuildMultiPackIndex();
918
- * const midx = await storage.getMultiPackIndex();
919
- * console.log(`Indexed ${midx.entries.length} objects`);
920
- * ```
921
- */
922
- async rebuildMultiPackIndex() {
923
- // List all packs
924
- const packs = await this.listPackfiles();
925
- const packIds = packs.map(p => p.packId);
926
- // Create entries for all objects in all packs
927
- const entries = [];
928
- for (let packIndex = 0; packIndex < packIds.length; packIndex++) {
929
- const packId = packIds[packIndex];
930
- // For now, create a synthetic entry per pack
931
- // In a real implementation, we would parse the index file
932
- const metadata = await this.getPackfileMetadata(packId);
933
- if (metadata) {
934
- // Create synthetic entries based on object count
935
- for (let i = 0; i < metadata.objectCount; i++) {
936
- // Generate synthetic object IDs based on pack checksum and index
937
- const objectId = metadata.checksum.slice(0, 32) + i.toString(16).padStart(8, '0');
938
- entries.push({
939
- objectId,
940
- packIndex,
941
- offset: 12 + i * 100 // Synthetic offset
942
- });
943
- }
944
- }
945
- }
946
- // Sort entries by objectId for binary search
947
- entries.sort((a, b) => a.objectId.localeCompare(b.objectId));
948
- // Create multi-pack index
949
- const midx = {
950
- version: 1,
951
- packIds,
952
- entries,
953
- checksum: new Uint8Array(20)
954
- };
955
- // Serialize and store
956
- const serialized = serializeMultiPackIndex(midx);
957
- const midxKey = this._buildKey('packs/multi-pack-index');
958
- await this._bucket.put(midxKey, serialized);
959
- // Update cache
960
- this._midxCache = {
961
- midx,
962
- expiresAt: Date.now() + this._cacheTTL * 1000
963
- };
964
- }
965
- /**
966
- * Gets the current multi-pack index.
967
- *
968
- * @description
969
- * Returns the MIDX from cache if available and not expired,
970
- * otherwise fetches from R2. Returns an empty index if none exists.
971
- *
972
- * @returns Current multi-pack index
973
- *
974
- * @example
975
- * ```typescript
976
- * const midx = await storage.getMultiPackIndex();
977
- * const entry = lookupObjectInMultiPack(midx, objectSha);
978
- * if (entry) {
979
- * const packId = midx.packIds[entry.packIndex];
980
- * console.log(`Object is in pack ${packId}`);
981
- * }
982
- * ```
983
- */
984
- async getMultiPackIndex() {
985
- // Check cache first
986
- if (this._midxCache && this._midxCache.expiresAt > Date.now()) {
987
- return this._midxCache.midx;
988
- }
989
- const midxKey = this._buildKey('packs/multi-pack-index');
990
- const midxObj = await this._bucket.get(midxKey);
991
- if (!midxObj) {
992
- // Return empty index
993
- return {
994
- version: 1,
995
- packIds: [],
996
- entries: [],
997
- checksum: new Uint8Array(20)
998
- };
999
- }
1000
- const data = new Uint8Array(await midxObj.arrayBuffer());
1001
- const midx = parseMultiPackIndex(data);
1002
- // Update cache
1003
- this._midxCache = {
1004
- midx,
1005
- expiresAt: Date.now() + this._cacheTTL * 1000
1006
- };
1007
- return midx;
1008
- }
1009
- /**
1010
- * Acquires a distributed lock on a resource using R2 conditional writes.
1011
- *
1012
- * @description
1013
- * Uses R2's conditional write feature (ETags) to implement distributed locking.
1014
- * Locks automatically expire after the TTL to prevent deadlocks.
1015
- *
1016
- * @param resource - Resource identifier to lock
1017
- * @param ttlMs - Time-to-live in milliseconds
1018
- * @param holder - Optional identifier for the lock holder (for debugging)
1019
- *
1020
- * @returns LockHandle if acquired, null if lock is held by another process
1021
- *
1022
- * @example
1023
- * ```typescript
1024
- * const handle = await storage.acquireDistributedLock('my-resource', 30000, 'worker-1');
1025
- * if (handle) {
1026
- * try {
1027
- * // Do work while holding the lock
1028
- * } finally {
1029
- * await storage.releaseDistributedLock(handle);
1030
- * }
1031
- * } else {
1032
- * console.log('Could not acquire lock - resource is busy');
1033
- * }
1034
- * ```
1035
- */
1036
- async acquireDistributedLock(resource, ttlMs = 30000, holder) {
1037
- const lockKey = this._buildKey(`locks/${resource}.lock`);
1038
- const now = Date.now();
1039
- const lockId = generateLockId();
1040
- const expiresAt = now + ttlMs;
1041
- const lockContent = {
1042
- lockId,
1043
- resource,
1044
- expiresAt,
1045
- acquiredAt: now,
1046
- holder
1047
- };
1048
- const lockData = new TextEncoder().encode(JSON.stringify(lockContent));
1049
- // Try to check if there's an existing lock
1050
- const existingObj = await this._bucket.head(lockKey);
1051
- if (existingObj) {
1052
- // Lock file exists, check if it's expired
1053
- const existingLockObj = await this._bucket.get(lockKey);
1054
- if (existingLockObj) {
1055
- try {
1056
- const existingContent = JSON.parse(new TextDecoder().decode(new Uint8Array(await existingLockObj.arrayBuffer())));
1057
- if (existingContent.expiresAt > now) {
1058
- // Lock is still valid, cannot acquire
1059
- return null;
1060
- }
1061
- // Lock is expired, try to overwrite with conditional write
1062
- // Use the existing etag to ensure atomicity
1063
- try {
1064
- await this._bucket.put(lockKey, lockData, {
1065
- onlyIf: { etagMatches: existingObj.etag }
1066
- });
1067
- // Get the new etag after successful write
1068
- const newObj = await this._bucket.head(lockKey);
1069
- if (!newObj) {
1070
- return null;
1071
- }
1072
- return {
1073
- resource,
1074
- lockId,
1075
- etag: newObj.etag,
1076
- expiresAt
1077
- };
1078
- }
1079
- catch {
1080
- // Conditional write failed - another process got the lock
1081
- return null;
1082
- }
1083
- }
1084
- catch {
1085
- // Failed to parse lock content, try to clean up and acquire
1086
- return null;
1087
- }
1088
- }
1089
- }
1090
- // No existing lock, try to create new one with onlyIf condition
1091
- try {
1092
- // Use onlyIf with etagDoesNotMatch to ensure the object doesn't exist
1093
- // R2 will fail if object already exists when we use this condition
1094
- await this._bucket.put(lockKey, lockData, {
1095
- onlyIf: { etagDoesNotMatch: '*' }
1096
- });
1097
- // Get the etag of the newly created lock
1098
- const newObj = await this._bucket.head(lockKey);
1099
- if (!newObj) {
1100
- return null;
1101
- }
1102
- // Verify we actually own this lock by checking the lockId
1103
- const verifyObj = await this._bucket.get(lockKey);
1104
- if (verifyObj) {
1105
- const content = JSON.parse(new TextDecoder().decode(new Uint8Array(await verifyObj.arrayBuffer())));
1106
- if (content.lockId !== lockId) {
1107
- // Another process created the lock
1108
- return null;
1109
- }
1110
- }
1111
- return {
1112
- resource,
1113
- lockId,
1114
- etag: newObj.etag,
1115
- expiresAt
1116
- };
1117
- }
1118
- catch {
1119
- // Failed to create lock - likely another process created it first
1120
- return null;
1121
- }
1122
- }
1123
- /**
1124
- * Releases a distributed lock.
1125
- *
1126
- * @description
1127
- * Releases the lock only if the caller still owns it (verified by lockId).
1128
- * Safe to call even if lock has expired or been taken by another process.
1129
- *
1130
- * @param handle - Lock handle returned from acquireDistributedLock
1131
- *
1132
- * @example
1133
- * ```typescript
1134
- * const handle = await storage.acquireDistributedLock('resource');
1135
- * if (handle) {
1136
- * try {
1137
- * // Do work
1138
- * } finally {
1139
- * await storage.releaseDistributedLock(handle);
1140
- * }
1141
- * }
1142
- * ```
1143
- */
1144
- async releaseDistributedLock(handle) {
1145
- const lockKey = this._buildKey(`locks/${handle.resource}.lock`);
1146
- // Verify we still own the lock before deleting
1147
- const existingObj = await this._bucket.get(lockKey);
1148
- if (existingObj) {
1149
- try {
1150
- const content = JSON.parse(new TextDecoder().decode(new Uint8Array(await existingObj.arrayBuffer())));
1151
- // Only delete if we own this lock (matching lockId)
1152
- if (content.lockId === handle.lockId) {
1153
- await this._bucket.delete(lockKey);
1154
- }
1155
- }
1156
- catch {
1157
- // Failed to parse, don't delete to avoid corrupting another process's lock
1158
- }
1159
- }
1160
- }
1161
- /**
1162
- * Refreshes a distributed lock to extend its TTL.
1163
- *
1164
- * @description
1165
- * Extends the lock's expiration time. Useful for long-running operations
1166
- * that need to hold the lock longer than the original TTL.
1167
- *
1168
- * @param handle - Lock handle to refresh
1169
- * @param ttlMs - New TTL in milliseconds
1170
- *
1171
- * @returns true if refresh succeeded, false if lock was lost
1172
- *
1173
- * @example
1174
- * ```typescript
1175
- * const handle = await storage.acquireDistributedLock('resource', 30000);
1176
- * if (handle) {
1177
- * // Do some work...
1178
- *
1179
- * // Extend the lock for another 30 seconds
1180
- * if (await storage.refreshDistributedLock(handle, 30000)) {
1181
- * // Continue working
1182
- * } else {
1183
- * // Lock was lost, abort operation
1184
- * }
1185
- * }
1186
- * ```
1187
- */
1188
- async refreshDistributedLock(handle, ttlMs = 30000) {
1189
- const lockKey = this._buildKey(`locks/${handle.resource}.lock`);
1190
- const now = Date.now();
1191
- const newExpiresAt = now + ttlMs;
1192
- // Get current lock to verify ownership
1193
- const existingObj = await this._bucket.head(lockKey);
1194
- if (!existingObj) {
1195
- return false; // Lock doesn't exist
1196
- }
1197
- const existingLockObj = await this._bucket.get(lockKey);
1198
- if (!existingLockObj) {
1199
- return false;
1200
- }
1201
- try {
1202
- const existingContent = JSON.parse(new TextDecoder().decode(new Uint8Array(await existingLockObj.arrayBuffer())));
1203
- // Verify we own this lock
1204
- if (existingContent.lockId !== handle.lockId) {
1205
- return false; // We don't own this lock
1206
- }
1207
- // Create updated lock content
1208
- const updatedContent = {
1209
- ...existingContent,
1210
- expiresAt: newExpiresAt
1211
- };
1212
- const lockData = new TextEncoder().encode(JSON.stringify(updatedContent));
1213
- // Update with conditional write using etag
1214
- try {
1215
- await this._bucket.put(lockKey, lockData, {
1216
- onlyIf: { etagMatches: existingObj.etag }
1217
- });
1218
- // Update the handle's expiration and etag
1219
- const newObj = await this._bucket.head(lockKey);
1220
- if (newObj) {
1221
- handle.etag = newObj.etag;
1222
- handle.expiresAt = newExpiresAt;
1223
- }
1224
- return true;
1225
- }
1226
- catch {
1227
- // Conditional write failed - lock was modified
1228
- return false;
1229
- }
1230
- }
1231
- catch {
1232
- return false;
1233
- }
1234
- }
1235
- /**
1236
- * Cleans up expired locks from R2 storage.
1237
- *
1238
- * @description
1239
- * Scans all lock files and removes those that have expired.
1240
- * This should be called periodically to remove stale lock files
1241
- * left by crashed processes.
1242
- *
1243
- * @returns Number of locks cleaned up
1244
- *
1245
- * @example
1246
- * ```typescript
1247
- * // Run periodically (e.g., every 5 minutes)
1248
- * const cleaned = await storage.cleanupExpiredLocks();
1249
- * console.log(`Cleaned up ${cleaned} expired locks`);
1250
- * ```
1251
- */
1252
- async cleanupExpiredLocks() {
1253
- const prefix = this._buildKey('locks/');
1254
- const listResult = await this._bucket.list({ prefix });
1255
- const now = Date.now();
1256
- let cleanedCount = 0;
1257
- for (const obj of listResult.objects) {
1258
- if (!obj.key.endsWith('.lock'))
1259
- continue;
1260
- const lockObj = await this._bucket.get(obj.key);
1261
- if (lockObj) {
1262
- try {
1263
- const content = JSON.parse(new TextDecoder().decode(new Uint8Array(await lockObj.arrayBuffer())));
1264
- if (content.expiresAt <= now) {
1265
- // Lock is expired, safe to delete
1266
- await this._bucket.delete(obj.key);
1267
- cleanedCount++;
1268
- }
1269
- }
1270
- catch {
1271
- // Invalid lock file, delete it
1272
- await this._bucket.delete(obj.key);
1273
- cleanedCount++;
1274
- }
1275
- }
1276
- }
1277
- return cleanedCount;
1278
- }
1279
- /**
1280
- * Acquires a lock on a packfile (backward-compatible wrapper).
1281
- *
1282
- * @description
1283
- * High-level API for acquiring a pack lock with optional timeout.
1284
- * Uses distributed locking with R2 conditional writes internally.
1285
- *
1286
- * @param packId - Pack identifier to lock
1287
- * @param options - Lock acquisition options
1288
- *
1289
- * @returns PackLock interface for managing the lock
1290
- *
1291
- * @throws {R2PackError} With code 'LOCKED' if lock cannot be acquired
1292
- *
1293
- * @example
1294
- * ```typescript
1295
- * const lock = await storage.acquireLock(packId, {
1296
- * timeout: 10000,
1297
- * ttl: 30000,
1298
- * holder: 'my-worker'
1299
- * });
1300
- *
1301
- * try {
1302
- * // Perform pack operations
1303
- * if (lock.refresh) {
1304
- * await lock.refresh(); // Extend lock if needed
1305
- * }
1306
- * } finally {
1307
- * await lock.release();
1308
- * }
1309
- * ```
1310
- */
1311
- async acquireLock(packId, options) {
1312
- const ttl = options?.ttl ?? 30000; // Default 30 second TTL
1313
- const timeout = options?.timeout ?? 0;
1314
- const startTime = Date.now();
1315
- // Try to acquire the distributed lock
1316
- let handle = await this.acquireDistributedLock(packId, ttl, options?.holder);
1317
- // If timeout is specified, retry until timeout expires
1318
- if (!handle && timeout > 0) {
1319
- while (Date.now() - startTime < timeout) {
1320
- await new Promise(resolve => setTimeout(resolve, 50)); // Wait 50ms between retries
1321
- handle = await this.acquireDistributedLock(packId, ttl, options?.holder);
1322
- if (handle)
1323
- break;
1324
- }
1325
- }
1326
- if (!handle) {
1327
- if (timeout > 0) {
1328
- throw new R2PackError(`Lock timeout for packfile: ${packId}`, 'LOCKED', packId);
1329
- }
1330
- throw new R2PackError(`Packfile is locked: ${packId}`, 'LOCKED', packId);
1331
- }
1332
- // Create the PackLock interface with distributed lock backing
1333
- const self = this;
1334
- let released = false;
1335
- return {
1336
- packId,
1337
- handle,
1338
- isHeld: () => !released && handle.expiresAt > Date.now(),
1339
- release: async () => {
1340
- if (!released && handle) {
1341
- await self.releaseDistributedLock(handle);
1342
- released = true;
1343
- }
1344
- },
1345
- refresh: async () => {
1346
- if (released || !handle)
1347
- return false;
1348
- return await self.refreshDistributedLock(handle, ttl);
1349
- }
1350
- };
1351
- }
1352
- }
1353
- /**
1354
- * Serializes a multi-pack index to bytes.
1355
- *
1356
- * @description
1357
- * Converts a MultiPackIndex structure to the binary MIDX format.
1358
- * The format includes:
1359
- * - MIDX signature (4 bytes)
1360
- * - Version (4 bytes)
1361
- * - Pack count (4 bytes)
1362
- * - Entry count (4 bytes)
1363
- * - Pack IDs with length prefixes
1364
- * - Object entries (40 + 4 + 8 = 52 bytes each)
1365
- * - Checksum (20 bytes)
1366
- *
1367
- * @param midx - Multi-pack index to serialize
1368
- * @returns Serialized MIDX bytes
1369
- *
1370
- * @example
1371
- * ```typescript
1372
- * const bytes = serializeMultiPackIndex(midx);
1373
- * await bucket.put('packs/multi-pack-index', bytes);
1374
- * ```
1375
- *
1376
- * @internal
1377
- */
1378
- function serializeMultiPackIndex(midx) {
1379
- // Calculate size
1380
- // Header: 4 (signature) + 4 (version) + 4 (packCount) + 4 (entryCount) = 16
1381
- // Pack IDs: packCount * (4 + packId.length) each with length prefix
1382
- // Entries: entryCount * (40 + 4 + 8) = 52 bytes each (objectId + packIndex + offset)
1383
- // Checksum: 20
1384
- let packIdsSize = 0;
1385
- for (const packId of midx.packIds) {
1386
- packIdsSize += 4 + new TextEncoder().encode(packId).length;
1387
- }
1388
- const entriesSize = midx.entries.length * 52;
1389
- const totalSize = 16 + packIdsSize + entriesSize + 20;
1390
- const data = new Uint8Array(totalSize);
1391
- const view = new DataView(data.buffer);
1392
- let offset = 0;
1393
- // Signature: MIDX
1394
- data.set(MIDX_SIGNATURE, offset);
1395
- offset += 4;
1396
- // Version
1397
- view.setUint32(offset, midx.version, false);
1398
- offset += 4;
1399
- // Pack count
1400
- view.setUint32(offset, midx.packIds.length, false);
1401
- offset += 4;
1402
- // Entry count
1403
- view.setUint32(offset, midx.entries.length, false);
1404
- offset += 4;
1405
- // Pack IDs
1406
- const encoder = new TextEncoder();
1407
- for (const packId of midx.packIds) {
1408
- const encoded = encoder.encode(packId);
1409
- view.setUint32(offset, encoded.length, false);
1410
- offset += 4;
1411
- data.set(encoded, offset);
1412
- offset += encoded.length;
1413
- }
1414
- // Entries
1415
- for (const entry of midx.entries) {
1416
- // Object ID (40 hex chars = 20 bytes as hex string, store as 40 bytes)
1417
- const objIdBytes = encoder.encode(entry.objectId.padEnd(40, '0').slice(0, 40));
1418
- data.set(objIdBytes, offset);
1419
- offset += 40;
1420
- // Pack index
1421
- view.setUint32(offset, entry.packIndex, false);
1422
- offset += 4;
1423
- // Offset (as 64-bit, but we use 32-bit high + 32-bit low)
1424
- view.setUint32(offset, 0, false); // high bits
1425
- offset += 4;
1426
- view.setUint32(offset, entry.offset, false); // low bits
1427
- offset += 4;
1428
- }
1429
- // Checksum
1430
- data.set(midx.checksum.slice(0, 20), offset);
1431
- return data;
1432
- }
1433
- // Standalone functions
1434
- /**
1435
- * Uploads a packfile to R2.
1436
- *
1437
- * @description
1438
- * Standalone function for uploading a packfile. Creates a temporary
1439
- * R2PackStorage instance internally.
1440
- *
1441
- * @param bucket - R2 bucket instance
1442
- * @param packData - Raw packfile bytes
1443
- * @param indexData - Pack index file bytes
1444
- * @param options - Optional configuration including prefix
1445
- *
1446
- * @returns Upload result with pack ID, sizes, and checksum
1447
- *
1448
- * @throws {R2PackError} If packfile is invalid or upload fails
1449
- *
1450
- * @example
1451
- * ```typescript
1452
- * const result = await uploadPackfile(bucket, packData, indexData, {
1453
- * prefix: 'repos/my-repo/'
1454
- * });
1455
- * console.log(`Uploaded: ${result.packId}`);
1456
- * ```
1457
- */
1458
- export async function uploadPackfile(bucket, packData, indexData, options) {
1459
- const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
1460
- return storage.uploadPackfile(packData, indexData);
1461
- }
1462
- /**
1463
- * Downloads a packfile from R2.
1464
- *
1465
- * @description
1466
- * Standalone function for downloading a packfile. Creates a temporary
1467
- * R2PackStorage instance internally.
1468
- *
1469
- * @param bucket - R2 bucket instance
1470
- * @param packId - Pack identifier to download
1471
- * @param options - Download options and prefix
1472
- *
1473
- * @returns Download result or null if not found
1474
- *
1475
- * @throws {R2PackError} If required=true and pack not found, or verification fails
1476
- *
1477
- * @example
1478
- * ```typescript
1479
- * const result = await downloadPackfile(bucket, packId, {
1480
- * prefix: 'repos/my-repo/',
1481
- * verify: true
1482
- * });
1483
- * ```
1484
- */
1485
- export async function downloadPackfile(bucket, packId, options) {
1486
- const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
1487
- return storage.downloadPackfile(packId, options);
1488
- }
1489
- /**
1490
- * Gets packfile metadata.
1491
- *
1492
- * @description
1493
- * Standalone function for retrieving packfile metadata without downloading
1494
- * the full pack.
1495
- *
1496
- * @param bucket - R2 bucket instance
1497
- * @param packId - Pack identifier
1498
- * @param options - Optional prefix configuration
1499
- *
1500
- * @returns Packfile metadata or null if not found
1501
- *
1502
- * @example
1503
- * ```typescript
1504
- * const metadata = await getPackfileMetadata(bucket, packId);
1505
- * if (metadata) {
1506
- * console.log(`Objects: ${metadata.objectCount}`);
1507
- * }
1508
- * ```
1509
- */
1510
- export async function getPackfileMetadata(bucket, packId, options) {
1511
- const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
1512
- return storage.getPackfileMetadata(packId);
1513
- }
1514
- /**
1515
- * Lists all packfiles.
1516
- *
1517
- * @description
1518
- * Standalone function for listing packfiles with pagination support.
1519
- *
1520
- * @param bucket - R2 bucket instance
1521
- * @param options - Prefix and pagination options
1522
- *
1523
- * @returns Array of packfile metadata
1524
- *
1525
- * @example
1526
- * ```typescript
1527
- * const packs = await listPackfiles(bucket, {
1528
- * prefix: 'repos/my-repo/',
1529
- * limit: 50
1530
- * });
1531
- * ```
1532
- */
1533
- export async function listPackfiles(bucket, options) {
1534
- const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
1535
- const result = await storage.listPackfiles({ limit: options?.limit, cursor: options?.cursor });
1536
- return result.items;
1537
- }
1538
- /**
1539
- * Deletes a packfile.
1540
- *
1541
- * @description
1542
- * Standalone function for deleting a packfile and its associated files.
1543
- *
1544
- * @param bucket - R2 bucket instance
1545
- * @param packId - Pack identifier to delete
1546
- * @param options - Optional prefix configuration
1547
- *
1548
- * @returns true if deleted, false if not found
1549
- *
1550
- * @example
1551
- * ```typescript
1552
- * if (await deletePackfile(bucket, packId)) {
1553
- * console.log('Deleted');
1554
- * }
1555
- * ```
1556
- */
1557
- export async function deletePackfile(bucket, packId, options) {
1558
- const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
1559
- return storage.deletePackfile(packId);
1560
- }
1561
- /**
1562
- * Creates a multi-pack index from all packfiles in the bucket.
1563
- *
1564
- * @description
1565
- * Standalone function that rebuilds the MIDX and returns the result.
1566
- *
1567
- * @param bucket - R2 bucket instance
1568
- * @param options - Optional prefix configuration
1569
- *
1570
- * @returns The newly created multi-pack index
1571
- *
1572
- * @example
1573
- * ```typescript
1574
- * const midx = await createMultiPackIndex(bucket, { prefix: 'repos/my-repo/' });
1575
- * console.log(`Indexed ${midx.entries.length} objects`);
1576
- * ```
1577
- */
1578
- export async function createMultiPackIndex(bucket, options) {
1579
- const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
1580
- await storage.rebuildMultiPackIndex();
1581
- return storage.getMultiPackIndex();
1582
- }
1583
- /**
1584
- * Parses a multi-pack index from raw bytes.
1585
- *
1586
- * @description
1587
- * Deserializes the binary MIDX format into a MultiPackIndex structure.
1588
- * Validates the signature and format.
1589
- *
1590
- * @param data - Raw MIDX bytes
1591
- * @returns Parsed multi-pack index
1592
- *
1593
- * @throws {R2PackError} With code 'INVALID_DATA' if format is invalid
1594
- *
1595
- * @example
1596
- * ```typescript
1597
- * const midxData = await bucket.get('packs/multi-pack-index');
1598
- * if (midxData) {
1599
- * const midx = parseMultiPackIndex(new Uint8Array(await midxData.arrayBuffer()));
1600
- * console.log(`Contains ${midx.entries.length} objects`);
1601
- * }
1602
- * ```
1603
- */
1604
- export function parseMultiPackIndex(data) {
1605
- if (data.length < 16) {
1606
- throw new R2PackError('Multi-pack index too small', 'INVALID_DATA');
1607
- }
1608
- const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
1609
- let offset = 0;
1610
- // Check signature
1611
- for (let i = 0; i < 4; i++) {
1612
- if (data[i] !== MIDX_SIGNATURE[i]) {
1613
- throw new R2PackError('Invalid multi-pack index signature', 'INVALID_DATA');
1614
- }
1615
- }
1616
- offset += 4;
1617
- // Version
1618
- const version = view.getUint32(offset, false);
1619
- offset += 4;
1620
- // Pack count
1621
- const packCount = view.getUint32(offset, false);
1622
- offset += 4;
1623
- // Entry count
1624
- const entryCount = view.getUint32(offset, false);
1625
- offset += 4;
1626
- // Read pack IDs
1627
- const decoder = new TextDecoder();
1628
- const packIds = [];
1629
- for (let i = 0; i < packCount; i++) {
1630
- const len = view.getUint32(offset, false);
1631
- offset += 4;
1632
- const packIdBytes = data.slice(offset, offset + len);
1633
- packIds.push(decoder.decode(packIdBytes));
1634
- offset += len;
1635
- }
1636
- // Read entries
1637
- const entries = [];
1638
- for (let i = 0; i < entryCount; i++) {
1639
- const objectIdBytes = data.slice(offset, offset + 40);
1640
- const objectId = decoder.decode(objectIdBytes);
1641
- offset += 40;
1642
- const packIndex = view.getUint32(offset, false);
1643
- offset += 4;
1644
- // Skip high bits
1645
- offset += 4;
1646
- const entryOffset = view.getUint32(offset, false);
1647
- offset += 4;
1648
- entries.push({
1649
- objectId,
1650
- packIndex,
1651
- offset: entryOffset
1652
- });
1653
- }
1654
- // Read checksum
1655
- const checksum = data.slice(offset, offset + 20);
1656
- return {
1657
- version,
1658
- packIds,
1659
- entries,
1660
- checksum: new Uint8Array(checksum)
1661
- };
1662
- }
1663
- /**
1664
- * Looks up an object in the multi-pack index using binary search.
1665
- *
1666
- * @description
1667
- * Efficiently finds an object's location across all packs using O(log n)
1668
- * binary search on the sorted entries.
1669
- *
1670
- * @param midx - Multi-pack index to search
1671
- * @param objectId - 40-character hex SHA-1 object ID to find
1672
- *
1673
- * @returns Entry with pack index and offset, or null if not found
1674
- *
1675
- * @example
1676
- * ```typescript
1677
- * const midx = await storage.getMultiPackIndex();
1678
- * const entry = lookupObjectInMultiPack(midx, 'abc123...');
1679
- * if (entry) {
1680
- * const packId = midx.packIds[entry.packIndex];
1681
- * const offset = entry.offset;
1682
- * console.log(`Found in ${packId} at offset ${offset}`);
1683
- * }
1684
- * ```
1685
- */
1686
- export function lookupObjectInMultiPack(midx, objectId) {
1687
- const entries = midx.entries;
1688
- if (entries.length === 0) {
1689
- return null;
1690
- }
1691
- // Binary search
1692
- let left = 0;
1693
- let right = entries.length - 1;
1694
- while (left <= right) {
1695
- const mid = Math.floor((left + right) / 2);
1696
- const entry = entries[mid];
1697
- const cmp = objectId.localeCompare(entry.objectId);
1698
- if (cmp === 0) {
1699
- return entry;
1700
- }
1701
- else if (cmp < 0) {
1702
- right = mid - 1;
1703
- }
1704
- else {
1705
- left = mid + 1;
1706
- }
1707
- }
1708
- return null;
1709
- }
1710
- /**
1711
- * Acquires a lock on a packfile.
1712
- *
1713
- * @description
1714
- * Standalone function for acquiring a pack lock using distributed locking.
1715
- *
1716
- * @param bucket - R2 bucket instance
1717
- * @param packId - Pack identifier to lock
1718
- * @param options - Lock options and prefix
1719
- *
1720
- * @returns PackLock interface for managing the lock
1721
- *
1722
- * @throws {R2PackError} With code 'LOCKED' if lock cannot be acquired
1723
- *
1724
- * @example
1725
- * ```typescript
1726
- * const lock = await acquirePackLock(bucket, packId, {
1727
- * prefix: 'repos/my-repo/',
1728
- * timeout: 10000,
1729
- * ttl: 30000
1730
- * });
1731
- *
1732
- * try {
1733
- * // Do work
1734
- * } finally {
1735
- * await lock.release();
1736
- * }
1737
- * ```
1738
- */
1739
- export async function acquirePackLock(bucket, packId, options) {
1740
- const storage = new R2PackStorage({ bucket, prefix: options?.prefix });
1741
- return storage.acquireLock(packId, options);
1742
- }
1743
- /**
1744
- * Releases a lock on a packfile.
1745
- *
1746
- * @description
1747
- * Standalone function for releasing a pack lock.
1748
- *
1749
- * Note: This function requires a valid PackLock with a handle to properly
1750
- * release distributed locks. For best results, use the lock.release() method
1751
- * on the PackLock object returned from acquirePackLock.
1752
- *
1753
- * @param bucket - R2 bucket instance
1754
- * @param packId - Pack identifier to unlock
1755
- * @param options - Optional prefix configuration
1756
- *
1757
- * @example
1758
- * ```typescript
1759
- * // Preferred: use lock.release()
1760
- * const lock = await acquirePackLock(bucket, packId);
1761
- * await lock.release();
1762
- *
1763
- * // Alternative: use standalone function (less safe)
1764
- * await releasePackLock(bucket, packId);
1765
- * ```
1766
- */
1767
- export async function releasePackLock(bucket, packId, options) {
1768
- // For backward compatibility, we just delete the lock file directly
1769
- // This is less safe than using the handle-based release, but works for simple cases
1770
- const lockKey = buildKey(options?.prefix ?? '', `locks/${packId}.lock`);
1771
- await bucket.delete(lockKey);
1772
- }
1773
- //# sourceMappingURL=r2-pack.js.map