gitx.do 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/blame.d.ts +259 -0
- package/dist/cli/commands/blame.d.ts.map +1 -0
- package/dist/cli/commands/blame.js +609 -0
- package/dist/cli/commands/blame.js.map +1 -0
- package/dist/cli/commands/branch.d.ts +249 -0
- package/dist/cli/commands/branch.d.ts.map +1 -0
- package/dist/cli/commands/branch.js +693 -0
- package/dist/cli/commands/branch.js.map +1 -0
- package/dist/cli/commands/commit.d.ts +182 -0
- package/dist/cli/commands/commit.d.ts.map +1 -0
- package/dist/cli/commands/commit.js +437 -0
- package/dist/cli/commands/commit.js.map +1 -0
- package/dist/cli/commands/diff.d.ts +464 -0
- package/dist/cli/commands/diff.d.ts.map +1 -0
- package/dist/cli/commands/diff.js +958 -0
- package/dist/cli/commands/diff.js.map +1 -0
- package/dist/cli/commands/log.d.ts +239 -0
- package/dist/cli/commands/log.d.ts.map +1 -0
- package/dist/cli/commands/log.js +535 -0
- package/dist/cli/commands/log.js.map +1 -0
- package/dist/cli/commands/review.d.ts +457 -0
- package/dist/cli/commands/review.d.ts.map +1 -0
- package/dist/cli/commands/review.js +533 -0
- package/dist/cli/commands/review.js.map +1 -0
- package/dist/cli/commands/status.d.ts +269 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +493 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/commands/web.d.ts +199 -0
- package/dist/cli/commands/web.d.ts.map +1 -0
- package/dist/cli/commands/web.js +696 -0
- package/dist/cli/commands/web.js.map +1 -0
- package/dist/cli/fs-adapter.d.ts +656 -0
- package/dist/cli/fs-adapter.d.ts.map +1 -0
- package/dist/cli/fs-adapter.js +1179 -0
- package/dist/cli/fs-adapter.js.map +1 -0
- package/dist/cli/index.d.ts +387 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +523 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/ui/components/DiffView.d.ts +7 -0
- package/dist/cli/ui/components/DiffView.d.ts.map +1 -0
- package/dist/cli/ui/components/DiffView.js +11 -0
- package/dist/cli/ui/components/DiffView.js.map +1 -0
- package/dist/cli/ui/components/ErrorDisplay.d.ts +6 -0
- package/dist/cli/ui/components/ErrorDisplay.d.ts.map +1 -0
- package/dist/cli/ui/components/ErrorDisplay.js +11 -0
- package/dist/cli/ui/components/ErrorDisplay.js.map +1 -0
- package/dist/cli/ui/components/FuzzySearch.d.ts +9 -0
- package/dist/cli/ui/components/FuzzySearch.d.ts.map +1 -0
- package/dist/cli/ui/components/FuzzySearch.js +12 -0
- package/dist/cli/ui/components/FuzzySearch.js.map +1 -0
- package/dist/cli/ui/components/LoadingSpinner.d.ts +6 -0
- package/dist/cli/ui/components/LoadingSpinner.d.ts.map +1 -0
- package/dist/cli/ui/components/LoadingSpinner.js +10 -0
- package/dist/cli/ui/components/LoadingSpinner.js.map +1 -0
- package/dist/cli/ui/components/NavigationList.d.ts +9 -0
- package/dist/cli/ui/components/NavigationList.d.ts.map +1 -0
- package/dist/cli/ui/components/NavigationList.js +11 -0
- package/dist/cli/ui/components/NavigationList.js.map +1 -0
- package/dist/cli/ui/components/ScrollableContent.d.ts +8 -0
- package/dist/cli/ui/components/ScrollableContent.d.ts.map +1 -0
- package/dist/cli/ui/components/ScrollableContent.js +11 -0
- package/dist/cli/ui/components/ScrollableContent.js.map +1 -0
- package/dist/cli/ui/components/index.d.ts +7 -0
- package/dist/cli/ui/components/index.d.ts.map +1 -0
- package/dist/cli/ui/components/index.js +9 -0
- package/dist/cli/ui/components/index.js.map +1 -0
- package/dist/cli/ui/terminal-ui.d.ts +52 -0
- package/dist/cli/ui/terminal-ui.d.ts.map +1 -0
- package/dist/cli/ui/terminal-ui.js +121 -0
- package/dist/cli/ui/terminal-ui.js.map +1 -0
- package/dist/durable-object/object-store.d.ts +401 -23
- package/dist/durable-object/object-store.d.ts.map +1 -1
- package/dist/durable-object/object-store.js +414 -25
- package/dist/durable-object/object-store.js.map +1 -1
- package/dist/durable-object/schema.d.ts +188 -0
- package/dist/durable-object/schema.d.ts.map +1 -1
- package/dist/durable-object/schema.js +160 -0
- package/dist/durable-object/schema.js.map +1 -1
- package/dist/durable-object/wal.d.ts +336 -31
- package/dist/durable-object/wal.d.ts.map +1 -1
- package/dist/durable-object/wal.js +272 -27
- package/dist/durable-object/wal.js.map +1 -1
- package/dist/index.d.ts +379 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +379 -7
- package/dist/index.js.map +1 -1
- package/dist/mcp/adapter.d.ts +579 -38
- package/dist/mcp/adapter.d.ts.map +1 -1
- package/dist/mcp/adapter.js +426 -33
- package/dist/mcp/adapter.js.map +1 -1
- package/dist/mcp/sandbox.d.ts +532 -29
- package/dist/mcp/sandbox.d.ts.map +1 -1
- package/dist/mcp/sandbox.js +389 -22
- package/dist/mcp/sandbox.js.map +1 -1
- package/dist/mcp/sdk-adapter.d.ts +478 -56
- package/dist/mcp/sdk-adapter.d.ts.map +1 -1
- package/dist/mcp/sdk-adapter.js +346 -44
- package/dist/mcp/sdk-adapter.js.map +1 -1
- package/dist/mcp/tools.d.ts +445 -30
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +363 -33
- package/dist/mcp/tools.js.map +1 -1
- package/dist/ops/blame.d.ts +424 -21
- package/dist/ops/blame.d.ts.map +1 -1
- package/dist/ops/blame.js +303 -20
- package/dist/ops/blame.js.map +1 -1
- package/dist/ops/branch.d.ts +583 -32
- package/dist/ops/branch.d.ts.map +1 -1
- package/dist/ops/branch.js +365 -23
- package/dist/ops/branch.js.map +1 -1
- package/dist/ops/commit-traversal.d.ts +164 -24
- package/dist/ops/commit-traversal.d.ts.map +1 -1
- package/dist/ops/commit-traversal.js +68 -2
- package/dist/ops/commit-traversal.js.map +1 -1
- package/dist/ops/commit.d.ts +387 -53
- package/dist/ops/commit.d.ts.map +1 -1
- package/dist/ops/commit.js +249 -29
- package/dist/ops/commit.js.map +1 -1
- package/dist/ops/merge-base.d.ts +195 -21
- package/dist/ops/merge-base.d.ts.map +1 -1
- package/dist/ops/merge-base.js +122 -12
- package/dist/ops/merge-base.js.map +1 -1
- package/dist/ops/merge.d.ts +600 -130
- package/dist/ops/merge.d.ts.map +1 -1
- package/dist/ops/merge.js +408 -60
- package/dist/ops/merge.js.map +1 -1
- package/dist/ops/tag.d.ts +67 -2
- package/dist/ops/tag.d.ts.map +1 -1
- package/dist/ops/tag.js +42 -1
- package/dist/ops/tag.js.map +1 -1
- package/dist/ops/tree-builder.d.ts +102 -6
- package/dist/ops/tree-builder.d.ts.map +1 -1
- package/dist/ops/tree-builder.js +30 -5
- package/dist/ops/tree-builder.js.map +1 -1
- package/dist/ops/tree-diff.d.ts +50 -2
- package/dist/ops/tree-diff.d.ts.map +1 -1
- package/dist/ops/tree-diff.js +50 -2
- package/dist/ops/tree-diff.js.map +1 -1
- package/dist/pack/delta.d.ts +211 -39
- package/dist/pack/delta.d.ts.map +1 -1
- package/dist/pack/delta.js +232 -46
- package/dist/pack/delta.js.map +1 -1
- package/dist/pack/format.d.ts +390 -28
- package/dist/pack/format.d.ts.map +1 -1
- package/dist/pack/format.js +344 -33
- package/dist/pack/format.js.map +1 -1
- package/dist/pack/full-generation.d.ts +313 -28
- package/dist/pack/full-generation.d.ts.map +1 -1
- package/dist/pack/full-generation.js +238 -19
- package/dist/pack/full-generation.js.map +1 -1
- package/dist/pack/generation.d.ts +346 -23
- package/dist/pack/generation.d.ts.map +1 -1
- package/dist/pack/generation.js +269 -21
- package/dist/pack/generation.js.map +1 -1
- package/dist/pack/index.d.ts +407 -86
- package/dist/pack/index.d.ts.map +1 -1
- package/dist/pack/index.js +351 -70
- package/dist/pack/index.js.map +1 -1
- package/dist/refs/branch.d.ts +517 -71
- package/dist/refs/branch.d.ts.map +1 -1
- package/dist/refs/branch.js +410 -26
- package/dist/refs/branch.js.map +1 -1
- package/dist/refs/storage.d.ts +610 -57
- package/dist/refs/storage.d.ts.map +1 -1
- package/dist/refs/storage.js +481 -29
- package/dist/refs/storage.js.map +1 -1
- package/dist/refs/tag.d.ts +677 -67
- package/dist/refs/tag.d.ts.map +1 -1
- package/dist/refs/tag.js +497 -30
- package/dist/refs/tag.js.map +1 -1
- package/dist/storage/lru-cache.d.ts +556 -53
- package/dist/storage/lru-cache.d.ts.map +1 -1
- package/dist/storage/lru-cache.js +439 -36
- package/dist/storage/lru-cache.js.map +1 -1
- package/dist/storage/object-index.d.ts +483 -38
- package/dist/storage/object-index.d.ts.map +1 -1
- package/dist/storage/object-index.js +388 -22
- package/dist/storage/object-index.js.map +1 -1
- package/dist/storage/r2-pack.d.ts +957 -94
- package/dist/storage/r2-pack.d.ts.map +1 -1
- package/dist/storage/r2-pack.js +756 -48
- package/dist/storage/r2-pack.js.map +1 -1
- package/dist/tiered/cdc-pipeline.d.ts +1610 -38
- package/dist/tiered/cdc-pipeline.d.ts.map +1 -1
- package/dist/tiered/cdc-pipeline.js +1131 -22
- package/dist/tiered/cdc-pipeline.js.map +1 -1
- package/dist/tiered/migration.d.ts +903 -41
- package/dist/tiered/migration.d.ts.map +1 -1
- package/dist/tiered/migration.js +646 -24
- package/dist/tiered/migration.js.map +1 -1
- package/dist/tiered/parquet-writer.d.ts +944 -47
- package/dist/tiered/parquet-writer.d.ts.map +1 -1
- package/dist/tiered/parquet-writer.js +667 -39
- package/dist/tiered/parquet-writer.js.map +1 -1
- package/dist/tiered/read-path.d.ts +728 -34
- package/dist/tiered/read-path.d.ts.map +1 -1
- package/dist/tiered/read-path.js +310 -27
- package/dist/tiered/read-path.js.map +1 -1
- package/dist/types/objects.d.ts +457 -0
- package/dist/types/objects.d.ts.map +1 -1
- package/dist/types/objects.js +305 -4
- package/dist/types/objects.js.map +1 -1
- package/dist/types/storage.d.ts +407 -35
- package/dist/types/storage.d.ts.map +1 -1
- package/dist/types/storage.js +27 -3
- package/dist/types/storage.js.map +1 -1
- package/dist/utils/hash.d.ts +133 -12
- package/dist/utils/hash.d.ts.map +1 -1
- package/dist/utils/hash.js +133 -12
- package/dist/utils/hash.js.map +1 -1
- package/dist/utils/sha1.d.ts +102 -9
- package/dist/utils/sha1.d.ts.map +1 -1
- package/dist/utils/sha1.js +114 -11
- package/dist/utils/sha1.js.map +1 -1
- package/dist/wire/capabilities.d.ts +896 -88
- package/dist/wire/capabilities.d.ts.map +1 -1
- package/dist/wire/capabilities.js +566 -62
- package/dist/wire/capabilities.js.map +1 -1
- package/dist/wire/pkt-line.d.ts +293 -15
- package/dist/wire/pkt-line.d.ts.map +1 -1
- package/dist/wire/pkt-line.js +251 -15
- package/dist/wire/pkt-line.js.map +1 -1
- package/dist/wire/receive-pack.d.ts +814 -64
- package/dist/wire/receive-pack.d.ts.map +1 -1
- package/dist/wire/receive-pack.js +542 -41
- package/dist/wire/receive-pack.js.map +1 -1
- package/dist/wire/smart-http.d.ts +575 -97
- package/dist/wire/smart-http.d.ts.map +1 -1
- package/dist/wire/smart-http.js +337 -46
- package/dist/wire/smart-http.js.map +1 -1
- package/dist/wire/upload-pack.d.ts +492 -98
- package/dist/wire/upload-pack.d.ts.map +1 -1
- package/dist/wire/upload-pack.js +347 -59
- package/dist/wire/upload-pack.js.map +1 -1
- package/package.json +10 -2
package/dist/tiered/migration.js
CHANGED
|
@@ -1,15 +1,83 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Tier Migration (Hot -> Warm)
|
|
2
|
+
* @fileoverview Tier Migration Module (Hot -> Warm)
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* - Warm/R2: Packed in R2 object storage (for larger objects or archives)
|
|
4
|
+
* This module handles the migration of Git objects between storage tiers in the
|
|
5
|
+
* gitdo tiered storage architecture. It provides comprehensive functionality for:
|
|
7
6
|
*
|
|
8
|
-
*
|
|
7
|
+
* ## Storage Tiers
|
|
8
|
+
*
|
|
9
|
+
* - **Hot**: SQLite in Durable Object storage - fastest access, limited capacity
|
|
10
|
+
* - **Warm/R2**: Packed objects in R2 object storage - medium latency, larger capacity
|
|
11
|
+
*
|
|
12
|
+
* ## Key Features
|
|
13
|
+
*
|
|
14
|
+
* - **Policy-based Migration**: Configurable policies based on age, access frequency, and size
|
|
15
|
+
* - **Access Tracking**: Monitors object access patterns to inform migration decisions
|
|
16
|
+
* - **Atomic Operations**: Ensures data integrity during migration with rollback support
|
|
17
|
+
* - **Concurrent Access Handling**: Safe reads/writes during in-progress migrations
|
|
18
|
+
* - **Checksum Verification**: Optional integrity verification after migration
|
|
19
|
+
* - **Batch Migration**: Efficient bulk migration with configurable concurrency
|
|
20
|
+
*
|
|
21
|
+
* ## Migration Process
|
|
22
|
+
*
|
|
23
|
+
* 1. Acquire distributed lock on the object
|
|
24
|
+
* 2. Copy data from hot tier to warm tier
|
|
25
|
+
* 3. Verify data integrity (optional checksum verification)
|
|
26
|
+
* 4. Update object location index
|
|
27
|
+
* 5. Delete from hot tier
|
|
28
|
+
* 6. Release lock
|
|
29
|
+
*
|
|
30
|
+
* If any step fails, the migration is rolled back automatically.
|
|
31
|
+
*
|
|
32
|
+
* @module tiered/migration
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
* ```typescript
|
|
36
|
+
* // Create a migrator
|
|
37
|
+
* const migrator = new TierMigrator(storage);
|
|
38
|
+
*
|
|
39
|
+
* // Define migration policy
|
|
40
|
+
* const policy: MigrationPolicy = {
|
|
41
|
+
* maxAgeInHot: 24 * 60 * 60 * 1000, // 24 hours
|
|
42
|
+
* minAccessCount: 5,
|
|
43
|
+
* maxHotSize: 100 * 1024 * 1024 // 100MB
|
|
44
|
+
* };
|
|
45
|
+
*
|
|
46
|
+
* // Find candidates and migrate
|
|
47
|
+
* const candidates = await migrator.findMigrationCandidates(policy);
|
|
48
|
+
* for (const sha of candidates) {
|
|
49
|
+
* await migrator.migrate(sha, 'hot', 'r2', { verifyChecksum: true });
|
|
50
|
+
* }
|
|
51
|
+
* ```
|
|
9
52
|
*/
|
|
10
53
|
/**
|
|
11
|
-
* Error thrown during migration operations
|
|
12
|
-
*
|
|
54
|
+
* Error thrown during migration operations.
|
|
55
|
+
*
|
|
56
|
+
* @description
|
|
57
|
+
* Custom error class for migration failures with detailed information
|
|
58
|
+
* about the failure context. Also implements MigrationResult-like properties
|
|
59
|
+
* for compatibility with result handling code.
|
|
60
|
+
*
|
|
61
|
+
* Error codes:
|
|
62
|
+
* - `NOT_FOUND`: Object does not exist in source tier
|
|
63
|
+
* - `ALREADY_IN_TARGET`: Object is already in the target tier
|
|
64
|
+
* - `LOCK_TIMEOUT`: Could not acquire lock within timeout
|
|
65
|
+
* - `WRITE_FAILED`: Failed to write to target tier
|
|
66
|
+
* - `CHECKSUM_MISMATCH`: Data verification failed after migration
|
|
67
|
+
* - `UPDATE_FAILED`: Failed to update object index
|
|
68
|
+
*
|
|
69
|
+
* @example
|
|
70
|
+
* ```typescript
|
|
71
|
+
* try {
|
|
72
|
+
* await migrator.migrate(sha, 'hot', 'r2');
|
|
73
|
+
* } catch (error) {
|
|
74
|
+
* if (error instanceof MigrationError) {
|
|
75
|
+
* console.log(`Migration failed: ${error.code}`);
|
|
76
|
+
* console.log(`Object: ${error.sha}`);
|
|
77
|
+
* console.log(`${error.sourceTier} -> ${error.targetTier}`);
|
|
78
|
+
* }
|
|
79
|
+
* }
|
|
80
|
+
* ```
|
|
13
81
|
*/
|
|
14
82
|
export class MigrationError extends Error {
|
|
15
83
|
code;
|
|
@@ -17,9 +85,22 @@ export class MigrationError extends Error {
|
|
|
17
85
|
sourceTier;
|
|
18
86
|
targetTier;
|
|
19
87
|
cause;
|
|
88
|
+
/** Always false for error objects */
|
|
20
89
|
success = false;
|
|
90
|
+
/** Whether rollback was performed */
|
|
21
91
|
rolledBack = true;
|
|
92
|
+
/** Reason for rollback (from cause error) */
|
|
22
93
|
rollbackReason;
|
|
94
|
+
/**
|
|
95
|
+
* Creates a new MigrationError.
|
|
96
|
+
*
|
|
97
|
+
* @param message - Human-readable error message
|
|
98
|
+
* @param code - Error code for programmatic handling
|
|
99
|
+
* @param sha - SHA of the object being migrated
|
|
100
|
+
* @param sourceTier - Source storage tier
|
|
101
|
+
* @param targetTier - Target storage tier
|
|
102
|
+
* @param cause - Underlying error that caused this failure
|
|
103
|
+
*/
|
|
23
104
|
constructor(message, code, sha, sourceTier, targetTier, cause) {
|
|
24
105
|
super(message);
|
|
25
106
|
this.code = code;
|
|
@@ -31,20 +112,57 @@ export class MigrationError extends Error {
|
|
|
31
112
|
this.rollbackReason = cause?.message;
|
|
32
113
|
}
|
|
33
114
|
/**
|
|
34
|
-
*
|
|
115
|
+
* Returns this error as a MigrationError reference.
|
|
116
|
+
*
|
|
117
|
+
* @description
|
|
118
|
+
* Provides compatibility with MigrationResult.error property access.
|
|
119
|
+
*
|
|
120
|
+
* @returns This MigrationError instance
|
|
35
121
|
*/
|
|
36
122
|
get error() {
|
|
37
123
|
return this;
|
|
38
124
|
}
|
|
39
125
|
}
|
|
40
126
|
/**
|
|
41
|
-
* Rollback handler for failed migrations
|
|
127
|
+
* Rollback handler for failed migrations.
|
|
128
|
+
*
|
|
129
|
+
* @description
|
|
130
|
+
* Handles cleanup operations when a migration fails, ensuring
|
|
131
|
+
* that partial migrations don't leave the system in an inconsistent state.
|
|
132
|
+
*
|
|
133
|
+
* @example
|
|
134
|
+
* ```typescript
|
|
135
|
+
* const rollback = new MigrationRollback(storage);
|
|
136
|
+
* if (migrationFailed) {
|
|
137
|
+
* await rollback.rollback(job);
|
|
138
|
+
* }
|
|
139
|
+
* ```
|
|
42
140
|
*/
|
|
43
141
|
export class MigrationRollback {
|
|
44
142
|
storage;
|
|
143
|
+
/**
|
|
144
|
+
* Creates a new MigrationRollback handler.
|
|
145
|
+
*
|
|
146
|
+
* @param storage - The tier storage implementation
|
|
147
|
+
*/
|
|
45
148
|
constructor(storage) {
|
|
46
149
|
this.storage = storage;
|
|
47
150
|
}
|
|
151
|
+
/**
|
|
152
|
+
* Rolls back a failed migration job.
|
|
153
|
+
*
|
|
154
|
+
* @description
|
|
155
|
+
* Cleans up any partial data in the warm tier and releases the lock.
|
|
156
|
+
* Updates the job state to 'rolled_back'.
|
|
157
|
+
*
|
|
158
|
+
* @param job - The migration job to roll back
|
|
159
|
+
*
|
|
160
|
+
* @example
|
|
161
|
+
* ```typescript
|
|
162
|
+
* await rollback.rollback(failedJob);
|
|
163
|
+
* console.log(failedJob.state); // 'rolled_back'
|
|
164
|
+
* ```
|
|
165
|
+
*/
|
|
48
166
|
async rollback(job) {
|
|
49
167
|
// Clean up warm tier if data was written there
|
|
50
168
|
await this.storage.deleteFromWarm(job.sha);
|
|
@@ -56,13 +174,56 @@ export class MigrationRollback {
|
|
|
56
174
|
}
|
|
57
175
|
}
|
|
58
176
|
/**
|
|
59
|
-
* Handler for concurrent access during migration
|
|
177
|
+
* Handler for concurrent access during migration.
|
|
178
|
+
*
|
|
179
|
+
* @description
|
|
180
|
+
* Manages read and write operations that occur while an object
|
|
181
|
+
* is being migrated, ensuring data consistency.
|
|
182
|
+
*
|
|
183
|
+
* During migration:
|
|
184
|
+
* - Reads check hot tier first (data still there), then warm tier
|
|
185
|
+
* - Writes go to hot tier and may be queued for replay
|
|
186
|
+
*
|
|
187
|
+
* @example
|
|
188
|
+
* ```typescript
|
|
189
|
+
* const handler = new ConcurrentAccessHandler(storage);
|
|
190
|
+
*
|
|
191
|
+
* // Safe read during migration
|
|
192
|
+
* const data = await handler.handleRead(sha);
|
|
193
|
+
*
|
|
194
|
+
* // Safe write during migration
|
|
195
|
+
* await handler.handleWrite(sha, newData);
|
|
196
|
+
* ```
|
|
60
197
|
*/
|
|
61
198
|
export class ConcurrentAccessHandler {
|
|
62
199
|
storage;
|
|
200
|
+
/**
|
|
201
|
+
* Creates a new ConcurrentAccessHandler.
|
|
202
|
+
*
|
|
203
|
+
* @param storage - The tier storage implementation
|
|
204
|
+
*/
|
|
63
205
|
constructor(storage) {
|
|
64
206
|
this.storage = storage;
|
|
65
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* Handles a read operation during migration.
|
|
210
|
+
*
|
|
211
|
+
* @description
|
|
212
|
+
* Reads from hot tier first (data is still there during migration),
|
|
213
|
+
* then falls back to warm tier if not found.
|
|
214
|
+
*
|
|
215
|
+
* @param sha - The object SHA to read
|
|
216
|
+
*
|
|
217
|
+
* @returns Object data or null if not found
|
|
218
|
+
*
|
|
219
|
+
* @example
|
|
220
|
+
* ```typescript
|
|
221
|
+
* const data = await handler.handleRead(sha);
|
|
222
|
+
* if (data) {
|
|
223
|
+
* // Process the data
|
|
224
|
+
* }
|
|
225
|
+
* ```
|
|
226
|
+
*/
|
|
66
227
|
async handleRead(sha) {
|
|
67
228
|
// During migration, read from hot tier first (data is still there)
|
|
68
229
|
const data = await this.storage.getFromHot(sha);
|
|
@@ -71,21 +232,104 @@ export class ConcurrentAccessHandler {
|
|
|
71
232
|
// Fall back to warm tier
|
|
72
233
|
return this.storage.getFromWarm(sha);
|
|
73
234
|
}
|
|
235
|
+
/**
|
|
236
|
+
* Handles a write operation during migration.
|
|
237
|
+
*
|
|
238
|
+
* @description
|
|
239
|
+
* Writes to the hot tier. The TierMigrator will handle replaying
|
|
240
|
+
* pending writes after migration completes.
|
|
241
|
+
*
|
|
242
|
+
* @param sha - The object SHA to write
|
|
243
|
+
* @param data - The data to write
|
|
244
|
+
*
|
|
245
|
+
* @example
|
|
246
|
+
* ```typescript
|
|
247
|
+
* await handler.handleWrite(sha, newData);
|
|
248
|
+
* ```
|
|
249
|
+
*/
|
|
74
250
|
async handleWrite(sha, data) {
|
|
75
251
|
// Queue write - for now just write to hot tier
|
|
76
252
|
await this.storage.putToHot(sha, data);
|
|
77
253
|
}
|
|
78
254
|
}
|
|
79
255
|
/**
|
|
80
|
-
* Tracks access patterns for objects to inform migration decisions
|
|
256
|
+
* Tracks access patterns for objects to inform migration decisions.
|
|
257
|
+
*
|
|
258
|
+
* @description
|
|
259
|
+
* Records and analyzes access patterns for objects in the storage system.
|
|
260
|
+
* This information is used to make intelligent decisions about which
|
|
261
|
+
* objects should be migrated between tiers.
|
|
262
|
+
*
|
|
263
|
+
* ## Features
|
|
264
|
+
*
|
|
265
|
+
* - Records read/write operations with optional metrics
|
|
266
|
+
* - Calculates access frequency over time
|
|
267
|
+
* - Identifies hot objects (frequently accessed)
|
|
268
|
+
* - Identifies cold objects (rarely accessed)
|
|
269
|
+
* - Supports access count decay for temporal relevance
|
|
270
|
+
* - Persists patterns to storage for durability
|
|
271
|
+
*
|
|
272
|
+
* @example
|
|
273
|
+
* ```typescript
|
|
274
|
+
* const tracker = new AccessTracker(storage);
|
|
275
|
+
*
|
|
276
|
+
* // Record accesses
|
|
277
|
+
* await tracker.recordAccess(sha, 'read', { bytesRead: 1024 });
|
|
278
|
+
* await tracker.recordAccess(sha, 'write');
|
|
279
|
+
*
|
|
280
|
+
* // Get access pattern for an object
|
|
281
|
+
* const pattern = await tracker.getAccessPattern(sha);
|
|
282
|
+
* console.log(`Frequency: ${pattern.accessFrequency}/sec`);
|
|
283
|
+
*
|
|
284
|
+
* // Find hot and cold objects
|
|
285
|
+
* const hotObjects = await tracker.identifyHotObjects({ minAccessCount: 50 });
|
|
286
|
+
* const coldObjects = await tracker.identifyColdObjects({ maxAccessCount: 2 });
|
|
287
|
+
*
|
|
288
|
+
* // Apply decay to gradually forget old patterns
|
|
289
|
+
* await tracker.applyDecay({ decayFactor: 0.5, minAgeForDecayMs: 86400000 });
|
|
290
|
+
* ```
|
|
81
291
|
*/
|
|
82
292
|
export class AccessTracker {
|
|
83
293
|
storage;
|
|
84
294
|
accessPatterns;
|
|
295
|
+
/**
|
|
296
|
+
* Creates a new AccessTracker.
|
|
297
|
+
*
|
|
298
|
+
* @param storage - The tier storage implementation
|
|
299
|
+
*
|
|
300
|
+
* @example
|
|
301
|
+
* ```typescript
|
|
302
|
+
* const tracker = new AccessTracker(storage);
|
|
303
|
+
* ```
|
|
304
|
+
*/
|
|
85
305
|
constructor(storage) {
|
|
86
306
|
this.storage = storage;
|
|
87
307
|
this.accessPatterns = new Map();
|
|
88
308
|
}
|
|
309
|
+
/**
|
|
310
|
+
* Records an access operation for an object.
|
|
311
|
+
*
|
|
312
|
+
* @description
|
|
313
|
+
* Tracks a read or write operation, updating the access pattern
|
|
314
|
+
* for the object. Can include optional metrics like bytes read
|
|
315
|
+
* and latency.
|
|
316
|
+
*
|
|
317
|
+
* @param sha - The object SHA being accessed
|
|
318
|
+
* @param type - Type of access ('read' or 'write')
|
|
319
|
+
* @param metrics - Optional additional metrics
|
|
320
|
+
*
|
|
321
|
+
* @example
|
|
322
|
+
* ```typescript
|
|
323
|
+
* // Basic access recording
|
|
324
|
+
* await tracker.recordAccess(sha, 'read');
|
|
325
|
+
*
|
|
326
|
+
* // With metrics
|
|
327
|
+
* await tracker.recordAccess(sha, 'read', {
|
|
328
|
+
* bytesRead: 2048,
|
|
329
|
+
* latencyMs: 5
|
|
330
|
+
* });
|
|
331
|
+
* ```
|
|
332
|
+
*/
|
|
89
333
|
async recordAccess(sha, type, metrics) {
|
|
90
334
|
let pattern = this.accessPatterns.get(sha);
|
|
91
335
|
if (!pattern) {
|
|
@@ -125,6 +369,25 @@ export class AccessTracker {
|
|
|
125
369
|
this.storage.accessPatterns || new Map();
|
|
126
370
|
(this.storage.accessPatterns).set(sha, pattern);
|
|
127
371
|
}
|
|
372
|
+
/**
|
|
373
|
+
* Gets the access pattern for a specific object.
|
|
374
|
+
*
|
|
375
|
+
* @description
|
|
376
|
+
* Returns detailed access statistics for an object including
|
|
377
|
+
* read/write counts, access frequency, and average latency.
|
|
378
|
+
*
|
|
379
|
+
* @param sha - The object SHA to query
|
|
380
|
+
*
|
|
381
|
+
* @returns Access pattern for the object
|
|
382
|
+
*
|
|
383
|
+
* @example
|
|
384
|
+
* ```typescript
|
|
385
|
+
* const pattern = await tracker.getAccessPattern(sha);
|
|
386
|
+
* console.log(`Reads: ${pattern.readCount}`);
|
|
387
|
+
* console.log(`Writes: ${pattern.writeCount}`);
|
|
388
|
+
* console.log(`Frequency: ${pattern.accessFrequency.toFixed(2)}/sec`);
|
|
389
|
+
* ```
|
|
390
|
+
*/
|
|
128
391
|
async getAccessPattern(sha) {
|
|
129
392
|
const pattern = this.accessPatterns.get(sha);
|
|
130
393
|
const now = Date.now();
|
|
@@ -153,6 +416,25 @@ export class AccessTracker {
|
|
|
153
416
|
avgLatencyMs: pattern.accessCount > 0 ? pattern.totalLatencyMs / pattern.accessCount : 0
|
|
154
417
|
};
|
|
155
418
|
}
|
|
419
|
+
/**
|
|
420
|
+
* Identifies frequently accessed (hot) objects.
|
|
421
|
+
*
|
|
422
|
+
* @description
|
|
423
|
+
* Returns SHAs of objects that meet the hot object criteria,
|
|
424
|
+
* typically objects with high access counts.
|
|
425
|
+
*
|
|
426
|
+
* @param criteria - Criteria for identifying hot objects
|
|
427
|
+
*
|
|
428
|
+
* @returns Array of SHAs for hot objects
|
|
429
|
+
*
|
|
430
|
+
* @example
|
|
431
|
+
* ```typescript
|
|
432
|
+
* const hotObjects = await tracker.identifyHotObjects({
|
|
433
|
+
* minAccessCount: 100
|
|
434
|
+
* });
|
|
435
|
+
* console.log(`Found ${hotObjects.length} hot objects`);
|
|
436
|
+
* ```
|
|
437
|
+
*/
|
|
156
438
|
async identifyHotObjects(criteria) {
|
|
157
439
|
const hotObjects = [];
|
|
158
440
|
const minAccessCount = criteria.minAccessCount ?? 0;
|
|
@@ -164,6 +446,26 @@ export class AccessTracker {
|
|
|
164
446
|
}
|
|
165
447
|
return hotObjects;
|
|
166
448
|
}
|
|
449
|
+
/**
|
|
450
|
+
* Identifies rarely accessed (cold) objects.
|
|
451
|
+
*
|
|
452
|
+
* @description
|
|
453
|
+
* Returns SHAs of objects that meet the cold object criteria,
|
|
454
|
+
* typically objects with low access counts.
|
|
455
|
+
*
|
|
456
|
+
* @param criteria - Criteria for identifying cold objects
|
|
457
|
+
*
|
|
458
|
+
* @returns Array of SHAs for cold objects
|
|
459
|
+
*
|
|
460
|
+
* @example
|
|
461
|
+
* ```typescript
|
|
462
|
+
* const coldObjects = await tracker.identifyColdObjects({
|
|
463
|
+
* maxAccessCount: 2,
|
|
464
|
+
* minAgeMs: 7 * 24 * 60 * 60 * 1000 // 7 days
|
|
465
|
+
* });
|
|
466
|
+
* console.log(`Found ${coldObjects.length} cold objects for migration`);
|
|
467
|
+
* ```
|
|
468
|
+
*/
|
|
167
469
|
async identifyColdObjects(criteria) {
|
|
168
470
|
const coldObjects = [];
|
|
169
471
|
const maxAccessCount = criteria.maxAccessCount ?? Infinity;
|
|
@@ -178,6 +480,25 @@ export class AccessTracker {
|
|
|
178
480
|
}
|
|
179
481
|
return coldObjects;
|
|
180
482
|
}
|
|
483
|
+
/**
|
|
484
|
+
* Applies decay to access counts.
|
|
485
|
+
*
|
|
486
|
+
* @description
|
|
487
|
+
* Reduces access counts by a factor to gradually "forget" old access
|
|
488
|
+
* patterns. This helps the system respond to changing usage patterns
|
|
489
|
+
* over time.
|
|
490
|
+
*
|
|
491
|
+
* @param options - Decay configuration options
|
|
492
|
+
*
|
|
493
|
+
* @example
|
|
494
|
+
* ```typescript
|
|
495
|
+
* // Run daily to decay access counts by 50%
|
|
496
|
+
* await tracker.applyDecay({
|
|
497
|
+
* decayFactor: 0.5,
|
|
498
|
+
* minAgeForDecayMs: 0 // Apply to all
|
|
499
|
+
* });
|
|
500
|
+
* ```
|
|
501
|
+
*/
|
|
181
502
|
async applyDecay(options) {
|
|
182
503
|
const { decayFactor } = options;
|
|
183
504
|
for (const [_sha, pattern] of this.accessPatterns) {
|
|
@@ -185,6 +506,23 @@ export class AccessTracker {
|
|
|
185
506
|
pattern.writeCount = Math.floor(pattern.writeCount * decayFactor);
|
|
186
507
|
}
|
|
187
508
|
}
|
|
509
|
+
/**
|
|
510
|
+
* Gets aggregate access statistics.
|
|
511
|
+
*
|
|
512
|
+
* @description
|
|
513
|
+
* Returns summary statistics about access patterns across all
|
|
514
|
+
* tracked objects.
|
|
515
|
+
*
|
|
516
|
+
* @returns Aggregate access statistics
|
|
517
|
+
*
|
|
518
|
+
* @example
|
|
519
|
+
* ```typescript
|
|
520
|
+
* const stats = await tracker.getAccessStats();
|
|
521
|
+
* console.log(`Total reads: ${stats.totalReads}`);
|
|
522
|
+
* console.log(`Total writes: ${stats.totalWrites}`);
|
|
523
|
+
* console.log(`Unique objects: ${stats.uniqueObjectsAccessed}`);
|
|
524
|
+
* ```
|
|
525
|
+
*/
|
|
188
526
|
async getAccessStats() {
|
|
189
527
|
let totalReads = 0;
|
|
190
528
|
let totalWrites = 0;
|
|
@@ -202,6 +540,19 @@ export class AccessTracker {
|
|
|
202
540
|
uniqueObjectsAccessed: uniqueObjects.size
|
|
203
541
|
};
|
|
204
542
|
}
|
|
543
|
+
/**
|
|
544
|
+
* Loads persisted access patterns from storage.
|
|
545
|
+
*
|
|
546
|
+
* @description
|
|
547
|
+
* Restores access patterns that were previously persisted,
|
|
548
|
+
* useful for recovering state after a restart.
|
|
549
|
+
*
|
|
550
|
+
* @example
|
|
551
|
+
* ```typescript
|
|
552
|
+
* // On startup, restore access patterns
|
|
553
|
+
* await tracker.loadFromStorage();
|
|
554
|
+
* ```
|
|
555
|
+
*/
|
|
205
556
|
async loadFromStorage() {
|
|
206
557
|
// Load persisted access patterns from storage
|
|
207
558
|
const storedPatterns = this.storage.accessPatterns;
|
|
@@ -213,7 +564,54 @@ export class AccessTracker {
|
|
|
213
564
|
}
|
|
214
565
|
}
|
|
215
566
|
/**
|
|
216
|
-
* Main tier migration service
|
|
567
|
+
* Main tier migration service.
|
|
568
|
+
*
|
|
569
|
+
* @description
|
|
570
|
+
* Orchestrates the migration of Git objects between storage tiers.
|
|
571
|
+
* Provides both synchronous single-object migration and asynchronous
|
|
572
|
+
* job-based migration for long-running operations.
|
|
573
|
+
*
|
|
574
|
+
* ## Migration Process
|
|
575
|
+
*
|
|
576
|
+
* 1. Validate object exists and is not already in target tier
|
|
577
|
+
* 2. Acquire distributed lock with configurable timeout
|
|
578
|
+
* 3. Read data from source tier
|
|
579
|
+
* 4. Optionally compute source checksum
|
|
580
|
+
* 5. Write data to target tier
|
|
581
|
+
* 6. Optionally verify checksum matches
|
|
582
|
+
* 7. Update object index to point to new location
|
|
583
|
+
* 8. Delete from source tier
|
|
584
|
+
* 9. Release lock
|
|
585
|
+
*
|
|
586
|
+
* If any step fails, the migration is automatically rolled back.
|
|
587
|
+
*
|
|
588
|
+
* @example
|
|
589
|
+
* ```typescript
|
|
590
|
+
* const migrator = new TierMigrator(storage);
|
|
591
|
+
*
|
|
592
|
+
* // Simple migration
|
|
593
|
+
* const result = await migrator.migrate(sha, 'hot', 'r2');
|
|
594
|
+
* if (result.success) {
|
|
595
|
+
* console.log('Migration successful');
|
|
596
|
+
* }
|
|
597
|
+
*
|
|
598
|
+
* // Migration with verification
|
|
599
|
+
* const verifiedResult = await migrator.migrate(sha, 'hot', 'r2', {
|
|
600
|
+
* verifyChecksum: true,
|
|
601
|
+
* lockTimeout: 10000
|
|
602
|
+
* });
|
|
603
|
+
*
|
|
604
|
+
* // Batch migration
|
|
605
|
+
* const batchResult = await migrator.migrateBatch(shas, 'hot', 'r2', {
|
|
606
|
+
* concurrency: 5
|
|
607
|
+
* });
|
|
608
|
+
* console.log(`Migrated: ${batchResult.successful.length}`);
|
|
609
|
+
*
|
|
610
|
+
* // Long-running migration job
|
|
611
|
+
* const job = await migrator.startMigrationJob(largeSha, 'hot', 'r2');
|
|
612
|
+
* // ... later
|
|
613
|
+
* await migrator.completeMigrationJob(job);
|
|
614
|
+
* ```
|
|
217
615
|
*/
|
|
218
616
|
export class TierMigrator {
|
|
219
617
|
storage;
|
|
@@ -221,6 +619,16 @@ export class TierMigrator {
|
|
|
221
619
|
migrationHistory;
|
|
222
620
|
migratingObjects;
|
|
223
621
|
pendingWrites;
|
|
622
|
+
/**
|
|
623
|
+
* Creates a new TierMigrator.
|
|
624
|
+
*
|
|
625
|
+
* @param storage - The tier storage implementation
|
|
626
|
+
*
|
|
627
|
+
* @example
|
|
628
|
+
* ```typescript
|
|
629
|
+
* const migrator = new TierMigrator(storage);
|
|
630
|
+
* ```
|
|
631
|
+
*/
|
|
224
632
|
constructor(storage) {
|
|
225
633
|
this.storage = storage;
|
|
226
634
|
this.activeJobs = new Map();
|
|
@@ -230,7 +638,27 @@ export class TierMigrator {
|
|
|
230
638
|
// checksumCache reserved for integrity verification during migration
|
|
231
639
|
}
|
|
232
640
|
/**
|
|
233
|
-
*
|
|
641
|
+
* Finds objects that are candidates for migration based on policy.
|
|
642
|
+
*
|
|
643
|
+
* @description
|
|
644
|
+
* Analyzes objects in the hot tier and returns those that meet
|
|
645
|
+
* the migration criteria defined in the policy. Results are sorted
|
|
646
|
+
* by last access time (oldest first).
|
|
647
|
+
*
|
|
648
|
+
* @param policy - Migration policy defining criteria
|
|
649
|
+
*
|
|
650
|
+
* @returns Array of SHAs that are candidates for migration
|
|
651
|
+
*
|
|
652
|
+
* @example
|
|
653
|
+
* ```typescript
|
|
654
|
+
* const candidates = await migrator.findMigrationCandidates({
|
|
655
|
+
* maxAgeInHot: 7 * 24 * 60 * 60 * 1000, // 7 days
|
|
656
|
+
* minAccessCount: 5,
|
|
657
|
+
* maxHotSize: 100 * 1024 * 1024
|
|
658
|
+
* });
|
|
659
|
+
*
|
|
660
|
+
* console.log(`Found ${candidates.length} candidates for migration`);
|
|
661
|
+
* ```
|
|
234
662
|
*/
|
|
235
663
|
async findMigrationCandidates(policy) {
|
|
236
664
|
const now = Date.now();
|
|
@@ -275,7 +703,42 @@ export class TierMigrator {
|
|
|
275
703
|
return filtered.map(c => c.sha);
|
|
276
704
|
}
|
|
277
705
|
/**
|
|
278
|
-
*
|
|
706
|
+
* Migrates a single object between tiers.
|
|
707
|
+
*
|
|
708
|
+
* @description
|
|
709
|
+
* Performs a complete migration of an object from the source tier
|
|
710
|
+
* to the target tier. Handles locking, data transfer, verification,
|
|
711
|
+
* and cleanup.
|
|
712
|
+
*
|
|
713
|
+
* @param sha - The object SHA to migrate
|
|
714
|
+
* @param sourceTier - The source storage tier
|
|
715
|
+
* @param targetTier - The target storage tier
|
|
716
|
+
* @param options - Optional migration settings
|
|
717
|
+
*
|
|
718
|
+
* @returns Migration result with success/failure status
|
|
719
|
+
*
|
|
720
|
+
* @throws {MigrationError} If object not found or already in target tier
|
|
721
|
+
*
|
|
722
|
+
* @example
|
|
723
|
+
* ```typescript
|
|
724
|
+
* // Basic migration
|
|
725
|
+
* const result = await migrator.migrate(sha, 'hot', 'r2');
|
|
726
|
+
*
|
|
727
|
+
* // With checksum verification
|
|
728
|
+
* const verified = await migrator.migrate(sha, 'hot', 'r2', {
|
|
729
|
+
* verifyChecksum: true,
|
|
730
|
+
* lockTimeout: 10000
|
|
731
|
+
* });
|
|
732
|
+
*
|
|
733
|
+
* if (verified.success) {
|
|
734
|
+
* console.log('Migration successful');
|
|
735
|
+
* if (verified.checksumVerified) {
|
|
736
|
+
* console.log('Integrity verified');
|
|
737
|
+
* }
|
|
738
|
+
* } else if (verified.rolledBack) {
|
|
739
|
+
* console.log(`Rolled back: ${verified.rollbackReason}`);
|
|
740
|
+
* }
|
|
741
|
+
* ```
|
|
279
742
|
*/
|
|
280
743
|
async migrate(sha, sourceTier, targetTier, options) {
|
|
281
744
|
// Check if object exists
|
|
@@ -413,7 +876,28 @@ export class TierMigrator {
|
|
|
413
876
|
this.migrationHistory.set(sha, history);
|
|
414
877
|
}
|
|
415
878
|
/**
|
|
416
|
-
*
|
|
879
|
+
* Starts a long-running migration job.
|
|
880
|
+
*
|
|
881
|
+
* @description
|
|
882
|
+
* Initiates a migration job that can be monitored and completed
|
|
883
|
+
* asynchronously. Useful for large objects where progress tracking
|
|
884
|
+
* is important.
|
|
885
|
+
*
|
|
886
|
+
* @param sha - The object SHA to migrate
|
|
887
|
+
* @param sourceTier - The source storage tier
|
|
888
|
+
* @param targetTier - The target storage tier
|
|
889
|
+
*
|
|
890
|
+
* @returns The migration job with tracking information
|
|
891
|
+
*
|
|
892
|
+
* @example
|
|
893
|
+
* ```typescript
|
|
894
|
+
* const job = await migrator.startMigrationJob(largeSha, 'hot', 'r2');
|
|
895
|
+
* console.log(`Job ${job.id} started`);
|
|
896
|
+
* console.log(`Progress: ${job.progress.bytesTransferred}/${job.progress.totalBytes}`);
|
|
897
|
+
*
|
|
898
|
+
* // Complete the job when ready
|
|
899
|
+
* await migrator.completeMigrationJob(job);
|
|
900
|
+
* ```
|
|
417
901
|
*/
|
|
418
902
|
async startMigrationJob(sha, sourceTier, targetTier) {
|
|
419
903
|
// Acquire lock
|
|
@@ -449,7 +933,22 @@ export class TierMigrator {
|
|
|
449
933
|
return job;
|
|
450
934
|
}
|
|
451
935
|
/**
|
|
452
|
-
*
|
|
936
|
+
* Completes a migration job.
|
|
937
|
+
*
|
|
938
|
+
* @description
|
|
939
|
+
* Finalizes a migration job by updating the index and cleaning up
|
|
940
|
+
* the source tier. Also processes any pending writes that occurred
|
|
941
|
+
* during the migration.
|
|
942
|
+
*
|
|
943
|
+
* @param job - The migration job to complete
|
|
944
|
+
*
|
|
945
|
+
* @example
|
|
946
|
+
* ```typescript
|
|
947
|
+
* const job = await migrator.startMigrationJob(sha, 'hot', 'r2');
|
|
948
|
+
* // ... wait for progress or do other work
|
|
949
|
+
* await migrator.completeMigrationJob(job);
|
|
950
|
+
* console.log(`Job completed at ${new Date(job.completedAt!)}`);
|
|
951
|
+
* ```
|
|
453
952
|
*/
|
|
454
953
|
async completeMigrationJob(job) {
|
|
455
954
|
const jobWithMeta = job;
|
|
@@ -481,7 +980,22 @@ export class TierMigrator {
|
|
|
481
980
|
this.recordHistory(job.sha, job.sourceTier, job.targetTier, 'completed');
|
|
482
981
|
}
|
|
483
982
|
/**
|
|
484
|
-
*
|
|
983
|
+
* Rolls back a migration job.
|
|
984
|
+
*
|
|
985
|
+
* @description
|
|
986
|
+
* Cancels a migration job and cleans up any partial data in the
|
|
987
|
+
* target tier.
|
|
988
|
+
*
|
|
989
|
+
* @param job - The migration job to roll back
|
|
990
|
+
*
|
|
991
|
+
* @example
|
|
992
|
+
* ```typescript
|
|
993
|
+
* const job = await migrator.startMigrationJob(sha, 'hot', 'r2');
|
|
994
|
+
* if (someCondition) {
|
|
995
|
+
* await migrator.rollbackMigrationJob(job);
|
|
996
|
+
* console.log('Migration rolled back');
|
|
997
|
+
* }
|
|
998
|
+
* ```
|
|
485
999
|
*/
|
|
486
1000
|
async rollbackMigrationJob(job) {
|
|
487
1001
|
// Clean up warm tier
|
|
@@ -496,7 +1010,19 @@ export class TierMigrator {
|
|
|
496
1010
|
this.recordHistory(job.sha, job.sourceTier, job.targetTier, 'rolled_back');
|
|
497
1011
|
}
|
|
498
1012
|
/**
|
|
499
|
-
*
|
|
1013
|
+
* Cancels a migration job by ID.
|
|
1014
|
+
*
|
|
1015
|
+
* @description
|
|
1016
|
+
* Stops a migration job and cleans up resources.
|
|
1017
|
+
*
|
|
1018
|
+
* @param jobId - The job ID to cancel
|
|
1019
|
+
*
|
|
1020
|
+
* @example
|
|
1021
|
+
* ```typescript
|
|
1022
|
+
* const job = await migrator.startMigrationJob(sha, 'hot', 'r2');
|
|
1023
|
+
* // Later...
|
|
1024
|
+
* await migrator.cancelMigrationJob(job.id);
|
|
1025
|
+
* ```
|
|
500
1026
|
*/
|
|
501
1027
|
async cancelMigrationJob(jobId) {
|
|
502
1028
|
const job = this.activeJobs.get(jobId);
|
|
@@ -513,19 +1039,71 @@ export class TierMigrator {
|
|
|
513
1039
|
this.activeJobs.delete(jobId);
|
|
514
1040
|
}
|
|
515
1041
|
/**
|
|
516
|
-
*
|
|
1042
|
+
* Gets all active migration jobs.
|
|
1043
|
+
*
|
|
1044
|
+
* @description
|
|
1045
|
+
* Returns jobs that are currently in progress.
|
|
1046
|
+
*
|
|
1047
|
+
* @returns Array of active migration jobs
|
|
1048
|
+
*
|
|
1049
|
+
* @example
|
|
1050
|
+
* ```typescript
|
|
1051
|
+
* const activeJobs = await migrator.getActiveMigrationJobs();
|
|
1052
|
+
* for (const job of activeJobs) {
|
|
1053
|
+
* console.log(`${job.id}: ${job.sha} - ${job.progress.bytesTransferred}/${job.progress.totalBytes}`);
|
|
1054
|
+
* }
|
|
1055
|
+
* ```
|
|
517
1056
|
*/
|
|
518
1057
|
async getActiveMigrationJobs() {
|
|
519
1058
|
return Array.from(this.activeJobs.values()).filter(j => j.state === 'in_progress');
|
|
520
1059
|
}
|
|
521
1060
|
/**
|
|
522
|
-
*
|
|
1061
|
+
* Gets migration history for an object.
|
|
1062
|
+
*
|
|
1063
|
+
* @description
|
|
1064
|
+
* Returns the history of migration events for a specific object.
|
|
1065
|
+
*
|
|
1066
|
+
* @param sha - The object SHA to query
|
|
1067
|
+
*
|
|
1068
|
+
* @returns Array of migration history entries
|
|
1069
|
+
*
|
|
1070
|
+
* @example
|
|
1071
|
+
* ```typescript
|
|
1072
|
+
* const history = await migrator.getMigrationHistory(sha);
|
|
1073
|
+
* for (const entry of history) {
|
|
1074
|
+
* console.log(`${new Date(entry.timestamp)}: ${entry.state}`);
|
|
1075
|
+
* }
|
|
1076
|
+
* ```
|
|
523
1077
|
*/
|
|
524
1078
|
async getMigrationHistory(sha) {
|
|
525
1079
|
return this.migrationHistory.get(sha) ?? [];
|
|
526
1080
|
}
|
|
527
1081
|
/**
|
|
528
|
-
*
|
|
1082
|
+
* Migrates multiple objects in a batch.
|
|
1083
|
+
*
|
|
1084
|
+
* @description
|
|
1085
|
+
* Efficiently migrates multiple objects with configurable concurrency.
|
|
1086
|
+
* Failed migrations don't affect other objects in the batch.
|
|
1087
|
+
*
|
|
1088
|
+
* @param shas - Array of object SHAs to migrate
|
|
1089
|
+
* @param sourceTier - The source storage tier
|
|
1090
|
+
* @param targetTier - The target storage tier
|
|
1091
|
+
* @param options - Optional batch migration settings
|
|
1092
|
+
*
|
|
1093
|
+
* @returns Result with successful and failed SHAs
|
|
1094
|
+
*
|
|
1095
|
+
* @example
|
|
1096
|
+
* ```typescript
|
|
1097
|
+
* const result = await migrator.migrateBatch(
|
|
1098
|
+
* candidates,
|
|
1099
|
+
* 'hot',
|
|
1100
|
+
* 'r2',
|
|
1101
|
+
* { concurrency: 5 }
|
|
1102
|
+
* );
|
|
1103
|
+
*
|
|
1104
|
+
* console.log(`Migrated: ${result.successful.length}`);
|
|
1105
|
+
* console.log(`Failed: ${result.failed.length}`);
|
|
1106
|
+
* ```
|
|
529
1107
|
*/
|
|
530
1108
|
async migrateBatch(shas, sourceTier, targetTier, options) {
|
|
531
1109
|
const concurrency = options?.concurrency ?? shas.length;
|
|
@@ -555,7 +1133,23 @@ export class TierMigrator {
|
|
|
555
1133
|
return { successful, failed };
|
|
556
1134
|
}
|
|
557
1135
|
/**
|
|
558
|
-
*
|
|
1136
|
+
* Reads object data during an in-progress migration.
|
|
1137
|
+
*
|
|
1138
|
+
* @description
|
|
1139
|
+
* Safely reads data for an object that may be in the process of
|
|
1140
|
+
* being migrated. Checks hot tier first, then warm tier.
|
|
1141
|
+
*
|
|
1142
|
+
* @param sha - The object SHA to read
|
|
1143
|
+
*
|
|
1144
|
+
* @returns Object data or null if not found
|
|
1145
|
+
*
|
|
1146
|
+
* @example
|
|
1147
|
+
* ```typescript
|
|
1148
|
+
* const data = await migrator.readDuringMigration(sha);
|
|
1149
|
+
* if (data) {
|
|
1150
|
+
* // Process the data regardless of which tier it's in
|
|
1151
|
+
* }
|
|
1152
|
+
* ```
|
|
559
1153
|
*/
|
|
560
1154
|
async readDuringMigration(sha) {
|
|
561
1155
|
// During migration, data should still be in hot tier
|
|
@@ -566,7 +1160,21 @@ export class TierMigrator {
|
|
|
566
1160
|
return this.storage.getFromWarm(sha);
|
|
567
1161
|
}
|
|
568
1162
|
/**
|
|
569
|
-
*
|
|
1163
|
+
* Writes object data during an in-progress migration.
|
|
1164
|
+
*
|
|
1165
|
+
* @description
|
|
1166
|
+
* Safely handles writes for an object that may be in the process
|
|
1167
|
+
* of being migrated. If the object is being migrated, the write
|
|
1168
|
+
* is queued and replayed after migration completes.
|
|
1169
|
+
*
|
|
1170
|
+
* @param sha - The object SHA to write
|
|
1171
|
+
* @param data - The data to write
|
|
1172
|
+
*
|
|
1173
|
+
* @example
|
|
1174
|
+
* ```typescript
|
|
1175
|
+
* // This is safe to call even during migration
|
|
1176
|
+
* await migrator.writeDuringMigration(sha, newData);
|
|
1177
|
+
* ```
|
|
570
1178
|
*/
|
|
571
1179
|
async writeDuringMigration(sha, data) {
|
|
572
1180
|
// If object is being migrated, queue the write
|
|
@@ -581,7 +1189,21 @@ export class TierMigrator {
|
|
|
581
1189
|
await this.storage.putToHot(sha, data);
|
|
582
1190
|
}
|
|
583
1191
|
/**
|
|
584
|
-
*
|
|
1192
|
+
* Computes SHA-256 checksum for data verification.
|
|
1193
|
+
*
|
|
1194
|
+
* @description
|
|
1195
|
+
* Calculates a SHA-256 hash of the data for integrity verification
|
|
1196
|
+
* during migration.
|
|
1197
|
+
*
|
|
1198
|
+
* @param data - The data to hash
|
|
1199
|
+
*
|
|
1200
|
+
* @returns Hex-encoded SHA-256 hash
|
|
1201
|
+
*
|
|
1202
|
+
* @example
|
|
1203
|
+
* ```typescript
|
|
1204
|
+
* const checksum = await migrator.computeChecksum(data);
|
|
1205
|
+
* console.log(`Checksum: ${checksum}`);
|
|
1206
|
+
* ```
|
|
585
1207
|
*/
|
|
586
1208
|
async computeChecksum(data) {
|
|
587
1209
|
const hashBuffer = await crypto.subtle.digest('SHA-256', data);
|