gitx.do 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/blame.d.ts +259 -0
- package/dist/cli/commands/blame.d.ts.map +1 -0
- package/dist/cli/commands/blame.js +609 -0
- package/dist/cli/commands/blame.js.map +1 -0
- package/dist/cli/commands/branch.d.ts +249 -0
- package/dist/cli/commands/branch.d.ts.map +1 -0
- package/dist/cli/commands/branch.js +693 -0
- package/dist/cli/commands/branch.js.map +1 -0
- package/dist/cli/commands/commit.d.ts +182 -0
- package/dist/cli/commands/commit.d.ts.map +1 -0
- package/dist/cli/commands/commit.js +437 -0
- package/dist/cli/commands/commit.js.map +1 -0
- package/dist/cli/commands/diff.d.ts +464 -0
- package/dist/cli/commands/diff.d.ts.map +1 -0
- package/dist/cli/commands/diff.js +958 -0
- package/dist/cli/commands/diff.js.map +1 -0
- package/dist/cli/commands/log.d.ts +239 -0
- package/dist/cli/commands/log.d.ts.map +1 -0
- package/dist/cli/commands/log.js +535 -0
- package/dist/cli/commands/log.js.map +1 -0
- package/dist/cli/commands/review.d.ts +457 -0
- package/dist/cli/commands/review.d.ts.map +1 -0
- package/dist/cli/commands/review.js +533 -0
- package/dist/cli/commands/review.js.map +1 -0
- package/dist/cli/commands/status.d.ts +269 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +493 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/commands/web.d.ts +199 -0
- package/dist/cli/commands/web.d.ts.map +1 -0
- package/dist/cli/commands/web.js +696 -0
- package/dist/cli/commands/web.js.map +1 -0
- package/dist/cli/fs-adapter.d.ts +656 -0
- package/dist/cli/fs-adapter.d.ts.map +1 -0
- package/dist/cli/fs-adapter.js +1179 -0
- package/dist/cli/fs-adapter.js.map +1 -0
- package/dist/cli/index.d.ts +387 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +523 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/ui/components/DiffView.d.ts +7 -0
- package/dist/cli/ui/components/DiffView.d.ts.map +1 -0
- package/dist/cli/ui/components/DiffView.js +11 -0
- package/dist/cli/ui/components/DiffView.js.map +1 -0
- package/dist/cli/ui/components/ErrorDisplay.d.ts +6 -0
- package/dist/cli/ui/components/ErrorDisplay.d.ts.map +1 -0
- package/dist/cli/ui/components/ErrorDisplay.js +11 -0
- package/dist/cli/ui/components/ErrorDisplay.js.map +1 -0
- package/dist/cli/ui/components/FuzzySearch.d.ts +9 -0
- package/dist/cli/ui/components/FuzzySearch.d.ts.map +1 -0
- package/dist/cli/ui/components/FuzzySearch.js +12 -0
- package/dist/cli/ui/components/FuzzySearch.js.map +1 -0
- package/dist/cli/ui/components/LoadingSpinner.d.ts +6 -0
- package/dist/cli/ui/components/LoadingSpinner.d.ts.map +1 -0
- package/dist/cli/ui/components/LoadingSpinner.js +10 -0
- package/dist/cli/ui/components/LoadingSpinner.js.map +1 -0
- package/dist/cli/ui/components/NavigationList.d.ts +9 -0
- package/dist/cli/ui/components/NavigationList.d.ts.map +1 -0
- package/dist/cli/ui/components/NavigationList.js +11 -0
- package/dist/cli/ui/components/NavigationList.js.map +1 -0
- package/dist/cli/ui/components/ScrollableContent.d.ts +8 -0
- package/dist/cli/ui/components/ScrollableContent.d.ts.map +1 -0
- package/dist/cli/ui/components/ScrollableContent.js +11 -0
- package/dist/cli/ui/components/ScrollableContent.js.map +1 -0
- package/dist/cli/ui/components/index.d.ts +7 -0
- package/dist/cli/ui/components/index.d.ts.map +1 -0
- package/dist/cli/ui/components/index.js +9 -0
- package/dist/cli/ui/components/index.js.map +1 -0
- package/dist/cli/ui/terminal-ui.d.ts +52 -0
- package/dist/cli/ui/terminal-ui.d.ts.map +1 -0
- package/dist/cli/ui/terminal-ui.js +121 -0
- package/dist/cli/ui/terminal-ui.js.map +1 -0
- package/dist/durable-object/object-store.d.ts +401 -23
- package/dist/durable-object/object-store.d.ts.map +1 -1
- package/dist/durable-object/object-store.js +414 -25
- package/dist/durable-object/object-store.js.map +1 -1
- package/dist/durable-object/schema.d.ts +188 -0
- package/dist/durable-object/schema.d.ts.map +1 -1
- package/dist/durable-object/schema.js +160 -0
- package/dist/durable-object/schema.js.map +1 -1
- package/dist/durable-object/wal.d.ts +336 -31
- package/dist/durable-object/wal.d.ts.map +1 -1
- package/dist/durable-object/wal.js +272 -27
- package/dist/durable-object/wal.js.map +1 -1
- package/dist/index.d.ts +379 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +379 -7
- package/dist/index.js.map +1 -1
- package/dist/mcp/adapter.d.ts +579 -38
- package/dist/mcp/adapter.d.ts.map +1 -1
- package/dist/mcp/adapter.js +426 -33
- package/dist/mcp/adapter.js.map +1 -1
- package/dist/mcp/sandbox.d.ts +532 -29
- package/dist/mcp/sandbox.d.ts.map +1 -1
- package/dist/mcp/sandbox.js +389 -22
- package/dist/mcp/sandbox.js.map +1 -1
- package/dist/mcp/sdk-adapter.d.ts +478 -56
- package/dist/mcp/sdk-adapter.d.ts.map +1 -1
- package/dist/mcp/sdk-adapter.js +346 -44
- package/dist/mcp/sdk-adapter.js.map +1 -1
- package/dist/mcp/tools.d.ts +445 -30
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +363 -33
- package/dist/mcp/tools.js.map +1 -1
- package/dist/ops/blame.d.ts +424 -21
- package/dist/ops/blame.d.ts.map +1 -1
- package/dist/ops/blame.js +303 -20
- package/dist/ops/blame.js.map +1 -1
- package/dist/ops/branch.d.ts +583 -32
- package/dist/ops/branch.d.ts.map +1 -1
- package/dist/ops/branch.js +365 -23
- package/dist/ops/branch.js.map +1 -1
- package/dist/ops/commit-traversal.d.ts +164 -24
- package/dist/ops/commit-traversal.d.ts.map +1 -1
- package/dist/ops/commit-traversal.js +68 -2
- package/dist/ops/commit-traversal.js.map +1 -1
- package/dist/ops/commit.d.ts +387 -53
- package/dist/ops/commit.d.ts.map +1 -1
- package/dist/ops/commit.js +249 -29
- package/dist/ops/commit.js.map +1 -1
- package/dist/ops/merge-base.d.ts +195 -21
- package/dist/ops/merge-base.d.ts.map +1 -1
- package/dist/ops/merge-base.js +122 -12
- package/dist/ops/merge-base.js.map +1 -1
- package/dist/ops/merge.d.ts +600 -130
- package/dist/ops/merge.d.ts.map +1 -1
- package/dist/ops/merge.js +408 -60
- package/dist/ops/merge.js.map +1 -1
- package/dist/ops/tag.d.ts +67 -2
- package/dist/ops/tag.d.ts.map +1 -1
- package/dist/ops/tag.js +42 -1
- package/dist/ops/tag.js.map +1 -1
- package/dist/ops/tree-builder.d.ts +102 -6
- package/dist/ops/tree-builder.d.ts.map +1 -1
- package/dist/ops/tree-builder.js +30 -5
- package/dist/ops/tree-builder.js.map +1 -1
- package/dist/ops/tree-diff.d.ts +50 -2
- package/dist/ops/tree-diff.d.ts.map +1 -1
- package/dist/ops/tree-diff.js +50 -2
- package/dist/ops/tree-diff.js.map +1 -1
- package/dist/pack/delta.d.ts +211 -39
- package/dist/pack/delta.d.ts.map +1 -1
- package/dist/pack/delta.js +232 -46
- package/dist/pack/delta.js.map +1 -1
- package/dist/pack/format.d.ts +390 -28
- package/dist/pack/format.d.ts.map +1 -1
- package/dist/pack/format.js +344 -33
- package/dist/pack/format.js.map +1 -1
- package/dist/pack/full-generation.d.ts +313 -28
- package/dist/pack/full-generation.d.ts.map +1 -1
- package/dist/pack/full-generation.js +238 -19
- package/dist/pack/full-generation.js.map +1 -1
- package/dist/pack/generation.d.ts +346 -23
- package/dist/pack/generation.d.ts.map +1 -1
- package/dist/pack/generation.js +269 -21
- package/dist/pack/generation.js.map +1 -1
- package/dist/pack/index.d.ts +407 -86
- package/dist/pack/index.d.ts.map +1 -1
- package/dist/pack/index.js +351 -70
- package/dist/pack/index.js.map +1 -1
- package/dist/refs/branch.d.ts +517 -71
- package/dist/refs/branch.d.ts.map +1 -1
- package/dist/refs/branch.js +410 -26
- package/dist/refs/branch.js.map +1 -1
- package/dist/refs/storage.d.ts +610 -57
- package/dist/refs/storage.d.ts.map +1 -1
- package/dist/refs/storage.js +481 -29
- package/dist/refs/storage.js.map +1 -1
- package/dist/refs/tag.d.ts +677 -67
- package/dist/refs/tag.d.ts.map +1 -1
- package/dist/refs/tag.js +497 -30
- package/dist/refs/tag.js.map +1 -1
- package/dist/storage/lru-cache.d.ts +556 -53
- package/dist/storage/lru-cache.d.ts.map +1 -1
- package/dist/storage/lru-cache.js +439 -36
- package/dist/storage/lru-cache.js.map +1 -1
- package/dist/storage/object-index.d.ts +483 -38
- package/dist/storage/object-index.d.ts.map +1 -1
- package/dist/storage/object-index.js +388 -22
- package/dist/storage/object-index.js.map +1 -1
- package/dist/storage/r2-pack.d.ts +957 -94
- package/dist/storage/r2-pack.d.ts.map +1 -1
- package/dist/storage/r2-pack.js +756 -48
- package/dist/storage/r2-pack.js.map +1 -1
- package/dist/tiered/cdc-pipeline.d.ts +1610 -38
- package/dist/tiered/cdc-pipeline.d.ts.map +1 -1
- package/dist/tiered/cdc-pipeline.js +1131 -22
- package/dist/tiered/cdc-pipeline.js.map +1 -1
- package/dist/tiered/migration.d.ts +903 -41
- package/dist/tiered/migration.d.ts.map +1 -1
- package/dist/tiered/migration.js +646 -24
- package/dist/tiered/migration.js.map +1 -1
- package/dist/tiered/parquet-writer.d.ts +944 -47
- package/dist/tiered/parquet-writer.d.ts.map +1 -1
- package/dist/tiered/parquet-writer.js +667 -39
- package/dist/tiered/parquet-writer.js.map +1 -1
- package/dist/tiered/read-path.d.ts +728 -34
- package/dist/tiered/read-path.d.ts.map +1 -1
- package/dist/tiered/read-path.js +310 -27
- package/dist/tiered/read-path.js.map +1 -1
- package/dist/types/objects.d.ts +457 -0
- package/dist/types/objects.d.ts.map +1 -1
- package/dist/types/objects.js +305 -4
- package/dist/types/objects.js.map +1 -1
- package/dist/types/storage.d.ts +407 -35
- package/dist/types/storage.d.ts.map +1 -1
- package/dist/types/storage.js +27 -3
- package/dist/types/storage.js.map +1 -1
- package/dist/utils/hash.d.ts +133 -12
- package/dist/utils/hash.d.ts.map +1 -1
- package/dist/utils/hash.js +133 -12
- package/dist/utils/hash.js.map +1 -1
- package/dist/utils/sha1.d.ts +102 -9
- package/dist/utils/sha1.d.ts.map +1 -1
- package/dist/utils/sha1.js +114 -11
- package/dist/utils/sha1.js.map +1 -1
- package/dist/wire/capabilities.d.ts +896 -88
- package/dist/wire/capabilities.d.ts.map +1 -1
- package/dist/wire/capabilities.js +566 -62
- package/dist/wire/capabilities.js.map +1 -1
- package/dist/wire/pkt-line.d.ts +293 -15
- package/dist/wire/pkt-line.d.ts.map +1 -1
- package/dist/wire/pkt-line.js +251 -15
- package/dist/wire/pkt-line.js.map +1 -1
- package/dist/wire/receive-pack.d.ts +814 -64
- package/dist/wire/receive-pack.d.ts.map +1 -1
- package/dist/wire/receive-pack.js +542 -41
- package/dist/wire/receive-pack.js.map +1 -1
- package/dist/wire/smart-http.d.ts +575 -97
- package/dist/wire/smart-http.d.ts.map +1 -1
- package/dist/wire/smart-http.js +337 -46
- package/dist/wire/smart-http.js.map +1 -1
- package/dist/wire/upload-pack.d.ts +492 -98
- package/dist/wire/upload-pack.d.ts.map +1 -1
- package/dist/wire/upload-pack.js +347 -59
- package/dist/wire/upload-pack.js.map +1 -1
- package/package.json +10 -2
|
@@ -1,135 +1,443 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Tier Migration (Hot -> Warm)
|
|
2
|
+
* @fileoverview Tier Migration Module (Hot -> Warm)
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* - Warm/R2: Packed in R2 object storage (for larger objects or archives)
|
|
4
|
+
* This module handles the migration of Git objects between storage tiers in the
|
|
5
|
+
* gitdo tiered storage architecture. It provides comprehensive functionality for:
|
|
7
6
|
*
|
|
8
|
-
*
|
|
7
|
+
* ## Storage Tiers
|
|
8
|
+
*
|
|
9
|
+
* - **Hot**: SQLite in Durable Object storage - fastest access, limited capacity
|
|
10
|
+
* - **Warm/R2**: Packed objects in R2 object storage - medium latency, larger capacity
|
|
11
|
+
*
|
|
12
|
+
* ## Key Features
|
|
13
|
+
*
|
|
14
|
+
* - **Policy-based Migration**: Configurable policies based on age, access frequency, and size
|
|
15
|
+
* - **Access Tracking**: Monitors object access patterns to inform migration decisions
|
|
16
|
+
* - **Atomic Operations**: Ensures data integrity during migration with rollback support
|
|
17
|
+
* - **Concurrent Access Handling**: Safe reads/writes during in-progress migrations
|
|
18
|
+
* - **Checksum Verification**: Optional integrity verification after migration
|
|
19
|
+
* - **Batch Migration**: Efficient bulk migration with configurable concurrency
|
|
20
|
+
*
|
|
21
|
+
* ## Migration Process
|
|
22
|
+
*
|
|
23
|
+
* 1. Acquire distributed lock on the object
|
|
24
|
+
* 2. Copy data from hot tier to warm tier
|
|
25
|
+
* 3. Verify data integrity (optional checksum verification)
|
|
26
|
+
* 4. Update object location index
|
|
27
|
+
* 5. Delete from hot tier
|
|
28
|
+
* 6. Release lock
|
|
29
|
+
*
|
|
30
|
+
* If any step fails, the migration is rolled back automatically.
|
|
31
|
+
*
|
|
32
|
+
* @module tiered/migration
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
* ```typescript
|
|
36
|
+
* // Create a migrator
|
|
37
|
+
* const migrator = new TierMigrator(storage);
|
|
38
|
+
*
|
|
39
|
+
* // Define migration policy
|
|
40
|
+
* const policy: MigrationPolicy = {
|
|
41
|
+
* maxAgeInHot: 24 * 60 * 60 * 1000, // 24 hours
|
|
42
|
+
* minAccessCount: 5,
|
|
43
|
+
* maxHotSize: 100 * 1024 * 1024 // 100MB
|
|
44
|
+
* };
|
|
45
|
+
*
|
|
46
|
+
* // Find candidates and migrate
|
|
47
|
+
* const candidates = await migrator.findMigrationCandidates(policy);
|
|
48
|
+
* for (const sha of candidates) {
|
|
49
|
+
* await migrator.migrate(sha, 'hot', 'r2', { verifyChecksum: true });
|
|
50
|
+
* }
|
|
51
|
+
* ```
|
|
9
52
|
*/
|
|
10
53
|
import { StorageTier } from '../storage/object-index';
|
|
11
54
|
/**
|
|
12
|
-
* Migration policy configuration
|
|
55
|
+
* Migration policy configuration.
|
|
56
|
+
*
|
|
57
|
+
* @description
|
|
58
|
+
* Defines the criteria for determining when objects should be migrated
|
|
59
|
+
* from the hot tier to the warm tier. Objects meeting ANY of these criteria
|
|
60
|
+
* may be migrated.
|
|
61
|
+
*
|
|
62
|
+
* @example
|
|
63
|
+
* ```typescript
|
|
64
|
+
* const policy: MigrationPolicy = {
|
|
65
|
+
* maxAgeInHot: 7 * 24 * 60 * 60 * 1000, // 7 days
|
|
66
|
+
* minAccessCount: 10, // Stay hot if accessed 10+ times
|
|
67
|
+
* maxHotSize: 500 * 1024 * 1024 // Trigger migration if hot > 500MB
|
|
68
|
+
* };
|
|
69
|
+
* ```
|
|
13
70
|
*/
|
|
14
71
|
export interface MigrationPolicy {
|
|
15
|
-
/**
|
|
72
|
+
/**
|
|
73
|
+
* Maximum age in hot tier before migration (milliseconds).
|
|
74
|
+
* Objects older than this may be migrated to warm tier.
|
|
75
|
+
* Use Infinity to disable age-based migration.
|
|
76
|
+
*/
|
|
16
77
|
maxAgeInHot: number;
|
|
17
|
-
/**
|
|
78
|
+
/**
|
|
79
|
+
* Minimum access count to stay in hot tier.
|
|
80
|
+
* Objects with fewer accesses may be migrated.
|
|
81
|
+
* Use 0 to disable access-count-based migration.
|
|
82
|
+
*/
|
|
18
83
|
minAccessCount: number;
|
|
19
|
-
/**
|
|
84
|
+
/**
|
|
85
|
+
* Maximum total size of hot tier (bytes).
|
|
86
|
+
* When exceeded, oldest/coldest objects are migrated.
|
|
87
|
+
*/
|
|
20
88
|
maxHotSize: number;
|
|
21
89
|
}
|
|
22
90
|
/**
|
|
23
|
-
* Migration job state
|
|
91
|
+
* Migration job state enumeration.
|
|
92
|
+
*
|
|
93
|
+
* @description
|
|
94
|
+
* Represents the lifecycle states of a migration job:
|
|
95
|
+
* - `pending`: Job created but not started
|
|
96
|
+
* - `in_progress`: Migration actively running
|
|
97
|
+
* - `completed`: Migration finished successfully
|
|
98
|
+
* - `failed`: Migration failed (may be retried)
|
|
99
|
+
* - `cancelled`: Job was cancelled by user
|
|
100
|
+
* - `rolled_back`: Migration failed and changes were reverted
|
|
24
101
|
*/
|
|
25
102
|
export type MigrationState = 'pending' | 'in_progress' | 'completed' | 'failed' | 'cancelled' | 'rolled_back';
|
|
26
103
|
/**
|
|
27
|
-
* Migration job progress
|
|
104
|
+
* Migration job progress tracking.
|
|
105
|
+
*
|
|
106
|
+
* @description
|
|
107
|
+
* Tracks the progress of data transfer during a migration job.
|
|
108
|
+
* Useful for long-running migrations of large objects.
|
|
109
|
+
*
|
|
110
|
+
* @example
|
|
111
|
+
* ```typescript
|
|
112
|
+
* const job = await migrator.startMigrationJob(sha, 'hot', 'r2');
|
|
113
|
+
* console.log(`Progress: ${job.progress.bytesTransferred}/${job.progress.totalBytes}`);
|
|
114
|
+
* ```
|
|
28
115
|
*/
|
|
29
116
|
export interface MigrationProgress {
|
|
117
|
+
/** Number of bytes transferred so far */
|
|
30
118
|
bytesTransferred: number;
|
|
119
|
+
/** Total number of bytes to transfer */
|
|
31
120
|
totalBytes: number;
|
|
32
121
|
}
|
|
33
122
|
/**
|
|
34
|
-
* Migration job tracking
|
|
123
|
+
* Migration job tracking information.
|
|
124
|
+
*
|
|
125
|
+
* @description
|
|
126
|
+
* Contains all information about an active or completed migration job,
|
|
127
|
+
* including progress tracking, timing, and state.
|
|
128
|
+
*
|
|
129
|
+
* @example
|
|
130
|
+
* ```typescript
|
|
131
|
+
* const job = await migrator.startMigrationJob(sha, 'hot', 'r2');
|
|
132
|
+
* console.log(`Job ${job.id} started at ${new Date(job.startedAt)}`);
|
|
133
|
+
* console.log(`State: ${job.state}`);
|
|
134
|
+
* ```
|
|
35
135
|
*/
|
|
36
136
|
export interface MigrationJob {
|
|
137
|
+
/** Unique identifier for the migration job */
|
|
37
138
|
id: string;
|
|
139
|
+
/** SHA of the object being migrated */
|
|
38
140
|
sha: string;
|
|
141
|
+
/** Source storage tier */
|
|
39
142
|
sourceTier: StorageTier;
|
|
143
|
+
/** Target storage tier */
|
|
40
144
|
targetTier: StorageTier;
|
|
145
|
+
/** Current state of the migration */
|
|
41
146
|
state: MigrationState;
|
|
147
|
+
/** Whether a lock was successfully acquired */
|
|
42
148
|
lockAcquired: boolean;
|
|
149
|
+
/** Progress tracking for the migration */
|
|
43
150
|
progress: MigrationProgress;
|
|
151
|
+
/** Timestamp when the job started (ms since epoch) */
|
|
44
152
|
startedAt: number;
|
|
153
|
+
/** Timestamp when the job completed (ms since epoch), if completed */
|
|
45
154
|
completedAt?: number;
|
|
46
155
|
}
|
|
47
156
|
/**
|
|
48
|
-
*
|
|
157
|
+
* Result of a migration operation.
|
|
158
|
+
*
|
|
159
|
+
* @description
|
|
160
|
+
* Contains the outcome of a single object migration, including success/failure
|
|
161
|
+
* status, verification results, and error information if applicable.
|
|
162
|
+
*
|
|
163
|
+
* @example
|
|
164
|
+
* ```typescript
|
|
165
|
+
* const result = await migrator.migrate(sha, 'hot', 'r2', { verifyChecksum: true });
|
|
166
|
+
* if (result.success) {
|
|
167
|
+
* console.log('Migration successful');
|
|
168
|
+
* if (result.checksumVerified) {
|
|
169
|
+
* console.log('Data integrity verified');
|
|
170
|
+
* }
|
|
171
|
+
* } else if (result.skipped) {
|
|
172
|
+
* console.log('Object was already migrated');
|
|
173
|
+
* } else if (result.rolledBack) {
|
|
174
|
+
* console.log(`Migration rolled back: ${result.rollbackReason}`);
|
|
175
|
+
* }
|
|
176
|
+
* ```
|
|
49
177
|
*/
|
|
50
178
|
export interface MigrationResult {
|
|
179
|
+
/** Whether the migration completed successfully */
|
|
51
180
|
success: boolean;
|
|
181
|
+
/** Whether the migration was skipped (e.g., already in target tier) */
|
|
52
182
|
skipped?: boolean;
|
|
183
|
+
/** Whether the migration was rolled back due to an error */
|
|
53
184
|
rolledBack?: boolean;
|
|
185
|
+
/** Whether checksum verification passed (if requested) */
|
|
54
186
|
checksumVerified?: boolean;
|
|
187
|
+
/** Error details if migration failed */
|
|
55
188
|
error?: MigrationError;
|
|
189
|
+
/** Reason for rollback if rolledBack is true */
|
|
56
190
|
rollbackReason?: string;
|
|
57
191
|
}
|
|
58
192
|
/**
|
|
59
|
-
*
|
|
193
|
+
* Result of a batch migration operation.
|
|
194
|
+
*
|
|
195
|
+
* @description
|
|
196
|
+
* Contains arrays of successful and failed SHA hashes after a batch migration.
|
|
197
|
+
*
|
|
198
|
+
* @example
|
|
199
|
+
* ```typescript
|
|
200
|
+
* const result = await migrator.migrateBatch(shas, 'hot', 'r2');
|
|
201
|
+
* console.log(`Migrated: ${result.successful.length}`);
|
|
202
|
+
* console.log(`Failed: ${result.failed.length}`);
|
|
203
|
+
* ```
|
|
60
204
|
*/
|
|
61
205
|
export interface BatchMigrationResult {
|
|
206
|
+
/** SHAs of objects that were successfully migrated */
|
|
62
207
|
successful: string[];
|
|
208
|
+
/** SHAs of objects that failed to migrate */
|
|
63
209
|
failed: string[];
|
|
64
210
|
}
|
|
65
211
|
/**
|
|
66
|
-
*
|
|
212
|
+
* Options for batch migration operations.
|
|
213
|
+
*
|
|
214
|
+
* @description
|
|
215
|
+
* Configures batch migration behavior, particularly concurrency.
|
|
216
|
+
*
|
|
217
|
+
* @example
|
|
218
|
+
* ```typescript
|
|
219
|
+
* // Migrate with limited concurrency to avoid overwhelming storage
|
|
220
|
+
* const result = await migrator.migrateBatch(shas, 'hot', 'r2', {
|
|
221
|
+
* concurrency: 5
|
|
222
|
+
* });
|
|
223
|
+
* ```
|
|
67
224
|
*/
|
|
68
225
|
export interface BatchMigrationOptions {
|
|
226
|
+
/**
|
|
227
|
+
* Number of concurrent migrations.
|
|
228
|
+
* @default shas.length (all at once)
|
|
229
|
+
*/
|
|
69
230
|
concurrency?: number;
|
|
70
231
|
}
|
|
71
232
|
/**
|
|
72
|
-
*
|
|
233
|
+
* Options for individual migration operations.
|
|
234
|
+
*
|
|
235
|
+
* @description
|
|
236
|
+
* Configures behavior of a single migration operation.
|
|
237
|
+
*
|
|
238
|
+
* @example
|
|
239
|
+
* ```typescript
|
|
240
|
+
* const result = await migrator.migrate(sha, 'hot', 'r2', {
|
|
241
|
+
* verifyChecksum: true,
|
|
242
|
+
* lockTimeout: 10000
|
|
243
|
+
* });
|
|
244
|
+
* ```
|
|
73
245
|
*/
|
|
74
246
|
export interface MigrateOptions {
|
|
247
|
+
/**
|
|
248
|
+
* Whether to verify checksums after migration.
|
|
249
|
+
* Adds overhead but ensures data integrity.
|
|
250
|
+
* @default false
|
|
251
|
+
*/
|
|
75
252
|
verifyChecksum?: boolean;
|
|
253
|
+
/**
|
|
254
|
+
* Timeout in milliseconds for acquiring the lock.
|
|
255
|
+
* @default 5000
|
|
256
|
+
*/
|
|
76
257
|
lockTimeout?: number;
|
|
77
258
|
}
|
|
78
259
|
/**
|
|
79
|
-
* Migration history entry
|
|
260
|
+
* Migration history entry.
|
|
261
|
+
*
|
|
262
|
+
* @description
|
|
263
|
+
* Records a single migration event for an object, useful for
|
|
264
|
+
* auditing and debugging migration issues.
|
|
265
|
+
*
|
|
266
|
+
* @example
|
|
267
|
+
* ```typescript
|
|
268
|
+
* const history = await migrator.getMigrationHistory(sha);
|
|
269
|
+
* for (const entry of history) {
|
|
270
|
+
* console.log(`${entry.timestamp}: ${entry.sourceTier} -> ${entry.targetTier}: ${entry.state}`);
|
|
271
|
+
* }
|
|
272
|
+
* ```
|
|
80
273
|
*/
|
|
81
274
|
export interface MigrationHistoryEntry {
|
|
275
|
+
/** SHA of the migrated object */
|
|
82
276
|
sha: string;
|
|
277
|
+
/** Source storage tier */
|
|
83
278
|
sourceTier: StorageTier;
|
|
279
|
+
/** Target storage tier */
|
|
84
280
|
targetTier: StorageTier;
|
|
281
|
+
/** Final state of the migration */
|
|
85
282
|
state: MigrationState;
|
|
283
|
+
/** Timestamp of the migration event */
|
|
86
284
|
timestamp: number;
|
|
87
285
|
}
|
|
88
286
|
/**
|
|
89
|
-
* Access pattern for an object
|
|
287
|
+
* Access pattern information for an object.
|
|
288
|
+
*
|
|
289
|
+
* @description
|
|
290
|
+
* Contains detailed access statistics for an object, used to make
|
|
291
|
+
* informed migration decisions.
|
|
292
|
+
*
|
|
293
|
+
* @example
|
|
294
|
+
* ```typescript
|
|
295
|
+
* const pattern = await tracker.getAccessPattern(sha);
|
|
296
|
+
* console.log(`Reads: ${pattern.readCount}, Writes: ${pattern.writeCount}`);
|
|
297
|
+
* console.log(`Access frequency: ${pattern.accessFrequency.toFixed(2)}/sec`);
|
|
298
|
+
* ```
|
|
90
299
|
*/
|
|
91
300
|
export interface AccessPattern {
|
|
301
|
+
/** SHA of the object */
|
|
92
302
|
sha: string;
|
|
303
|
+
/** Number of read operations */
|
|
93
304
|
readCount: number;
|
|
305
|
+
/** Number of write operations */
|
|
94
306
|
writeCount: number;
|
|
307
|
+
/** Timestamp of last access (ms since epoch) */
|
|
95
308
|
lastAccessedAt: number;
|
|
309
|
+
/** Access frequency (accesses per second) */
|
|
96
310
|
accessFrequency: number;
|
|
311
|
+
/** Total bytes read from this object */
|
|
97
312
|
totalBytesRead?: number;
|
|
313
|
+
/** Average latency in milliseconds for accessing this object */
|
|
98
314
|
avgLatencyMs?: number;
|
|
99
315
|
}
|
|
100
316
|
/**
|
|
101
|
-
*
|
|
317
|
+
* Aggregate access statistics.
|
|
318
|
+
*
|
|
319
|
+
* @description
|
|
320
|
+
* Provides summary statistics about access patterns across all tracked objects.
|
|
321
|
+
*
|
|
322
|
+
* @example
|
|
323
|
+
* ```typescript
|
|
324
|
+
* const stats = await tracker.getAccessStats();
|
|
325
|
+
* console.log(`Total reads: ${stats.totalReads}`);
|
|
326
|
+
* console.log(`Unique objects: ${stats.uniqueObjectsAccessed}`);
|
|
327
|
+
* ```
|
|
102
328
|
*/
|
|
103
329
|
export interface AccessStats {
|
|
330
|
+
/** Total number of read operations across all objects */
|
|
104
331
|
totalReads: number;
|
|
332
|
+
/** Total number of write operations across all objects */
|
|
105
333
|
totalWrites: number;
|
|
334
|
+
/** Number of unique objects that have been accessed */
|
|
106
335
|
uniqueObjectsAccessed: number;
|
|
107
336
|
}
|
|
108
337
|
/**
|
|
109
|
-
*
|
|
338
|
+
* Criteria for identifying hot or cold objects.
|
|
339
|
+
*
|
|
340
|
+
* @description
|
|
341
|
+
* Specifies filters for querying objects based on access patterns.
|
|
342
|
+
*
|
|
343
|
+
* @example
|
|
344
|
+
* ```typescript
|
|
345
|
+
* // Find frequently accessed objects
|
|
346
|
+
* const hotObjects = await tracker.identifyHotObjects({
|
|
347
|
+
* minAccessCount: 100
|
|
348
|
+
* });
|
|
349
|
+
*
|
|
350
|
+
* // Find rarely accessed objects
|
|
351
|
+
* const coldObjects = await tracker.identifyColdObjects({
|
|
352
|
+
* maxAccessCount: 2,
|
|
353
|
+
* minAgeMs: 7 * 24 * 60 * 60 * 1000 // 7 days old
|
|
354
|
+
* });
|
|
355
|
+
* ```
|
|
110
356
|
*/
|
|
111
357
|
export interface ObjectIdentificationCriteria {
|
|
358
|
+
/** Minimum total access count (reads + writes) */
|
|
112
359
|
minAccessCount?: number;
|
|
360
|
+
/** Maximum total access count (reads + writes) */
|
|
113
361
|
maxAccessCount?: number;
|
|
362
|
+
/** Minimum age in milliseconds since last access */
|
|
114
363
|
minAgeMs?: number;
|
|
115
364
|
}
|
|
116
365
|
/**
|
|
117
|
-
*
|
|
366
|
+
* Options for applying access count decay.
|
|
367
|
+
*
|
|
368
|
+
* @description
|
|
369
|
+
* Configures how access counts are decayed over time to gradually
|
|
370
|
+
* "forget" old access patterns and respond to changing usage.
|
|
371
|
+
*
|
|
372
|
+
* @example
|
|
373
|
+
* ```typescript
|
|
374
|
+
* // Apply 50% decay to all access counts
|
|
375
|
+
* await tracker.applyDecay({
|
|
376
|
+
* decayFactor: 0.5,
|
|
377
|
+
* minAgeForDecayMs: 24 * 60 * 60 * 1000 // Only decay after 24 hours
|
|
378
|
+
* });
|
|
379
|
+
* ```
|
|
118
380
|
*/
|
|
119
381
|
export interface DecayOptions {
|
|
382
|
+
/**
|
|
383
|
+
* Multiplier applied to access counts (0-1).
|
|
384
|
+
* 0.5 means counts are halved.
|
|
385
|
+
*/
|
|
120
386
|
decayFactor: number;
|
|
387
|
+
/**
|
|
388
|
+
* Minimum age in milliseconds before decay is applied.
|
|
389
|
+
* Prevents decay of recently accessed objects.
|
|
390
|
+
*/
|
|
121
391
|
minAgeForDecayMs: number;
|
|
122
392
|
}
|
|
123
393
|
/**
|
|
124
|
-
* Custom access
|
|
394
|
+
* Custom metrics for access recording.
|
|
395
|
+
*
|
|
396
|
+
* @description
|
|
397
|
+
* Additional metrics that can be recorded with each access operation.
|
|
398
|
+
*
|
|
399
|
+
* @example
|
|
400
|
+
* ```typescript
|
|
401
|
+
* await tracker.recordAccess(sha, 'read', {
|
|
402
|
+
* bytesRead: 1024,
|
|
403
|
+
* latencyMs: 15
|
|
404
|
+
* });
|
|
405
|
+
* ```
|
|
125
406
|
*/
|
|
126
407
|
export interface AccessMetrics {
|
|
408
|
+
/** Number of bytes read in this operation */
|
|
127
409
|
bytesRead?: number;
|
|
410
|
+
/** Latency of this operation in milliseconds */
|
|
128
411
|
latencyMs?: number;
|
|
129
412
|
}
|
|
130
413
|
/**
|
|
131
|
-
* Error thrown during migration operations
|
|
132
|
-
*
|
|
414
|
+
* Error thrown during migration operations.
|
|
415
|
+
*
|
|
416
|
+
* @description
|
|
417
|
+
* Custom error class for migration failures with detailed information
|
|
418
|
+
* about the failure context. Also implements MigrationResult-like properties
|
|
419
|
+
* for compatibility with result handling code.
|
|
420
|
+
*
|
|
421
|
+
* Error codes:
|
|
422
|
+
* - `NOT_FOUND`: Object does not exist in source tier
|
|
423
|
+
* - `ALREADY_IN_TARGET`: Object is already in the target tier
|
|
424
|
+
* - `LOCK_TIMEOUT`: Could not acquire lock within timeout
|
|
425
|
+
* - `WRITE_FAILED`: Failed to write to target tier
|
|
426
|
+
* - `CHECKSUM_MISMATCH`: Data verification failed after migration
|
|
427
|
+
* - `UPDATE_FAILED`: Failed to update object index
|
|
428
|
+
*
|
|
429
|
+
* @example
|
|
430
|
+
* ```typescript
|
|
431
|
+
* try {
|
|
432
|
+
* await migrator.migrate(sha, 'hot', 'r2');
|
|
433
|
+
* } catch (error) {
|
|
434
|
+
* if (error instanceof MigrationError) {
|
|
435
|
+
* console.log(`Migration failed: ${error.code}`);
|
|
436
|
+
* console.log(`Object: ${error.sha}`);
|
|
437
|
+
* console.log(`${error.sourceTier} -> ${error.targetTier}`);
|
|
438
|
+
* }
|
|
439
|
+
* }
|
|
440
|
+
* ```
|
|
133
441
|
*/
|
|
134
442
|
export declare class MigrationError extends Error {
|
|
135
443
|
readonly code: string;
|
|
@@ -137,50 +445,380 @@ export declare class MigrationError extends Error {
|
|
|
137
445
|
readonly sourceTier: StorageTier;
|
|
138
446
|
readonly targetTier: StorageTier;
|
|
139
447
|
readonly cause?: Error | undefined;
|
|
448
|
+
/** Always false for error objects */
|
|
140
449
|
readonly success: boolean;
|
|
450
|
+
/** Whether rollback was performed */
|
|
141
451
|
readonly rolledBack: boolean;
|
|
452
|
+
/** Reason for rollback (from cause error) */
|
|
142
453
|
readonly rollbackReason?: string;
|
|
454
|
+
/**
|
|
455
|
+
* Creates a new MigrationError.
|
|
456
|
+
*
|
|
457
|
+
* @param message - Human-readable error message
|
|
458
|
+
* @param code - Error code for programmatic handling
|
|
459
|
+
* @param sha - SHA of the object being migrated
|
|
460
|
+
* @param sourceTier - Source storage tier
|
|
461
|
+
* @param targetTier - Target storage tier
|
|
462
|
+
* @param cause - Underlying error that caused this failure
|
|
463
|
+
*/
|
|
143
464
|
constructor(message: string, code: string, sha: string, sourceTier: StorageTier, targetTier: StorageTier, cause?: Error | undefined);
|
|
144
465
|
/**
|
|
145
|
-
*
|
|
466
|
+
* Returns this error as a MigrationError reference.
|
|
467
|
+
*
|
|
468
|
+
* @description
|
|
469
|
+
* Provides compatibility with MigrationResult.error property access.
|
|
470
|
+
*
|
|
471
|
+
* @returns This MigrationError instance
|
|
146
472
|
*/
|
|
147
473
|
get error(): MigrationError;
|
|
148
474
|
}
|
|
149
475
|
/**
|
|
150
|
-
* Rollback handler for failed migrations
|
|
476
|
+
* Rollback handler for failed migrations.
|
|
477
|
+
*
|
|
478
|
+
* @description
|
|
479
|
+
* Handles cleanup operations when a migration fails, ensuring
|
|
480
|
+
* that partial migrations don't leave the system in an inconsistent state.
|
|
481
|
+
*
|
|
482
|
+
* @example
|
|
483
|
+
* ```typescript
|
|
484
|
+
* const rollback = new MigrationRollback(storage);
|
|
485
|
+
* if (migrationFailed) {
|
|
486
|
+
* await rollback.rollback(job);
|
|
487
|
+
* }
|
|
488
|
+
* ```
|
|
151
489
|
*/
|
|
152
490
|
export declare class MigrationRollback {
|
|
153
491
|
private storage;
|
|
492
|
+
/**
|
|
493
|
+
* Creates a new MigrationRollback handler.
|
|
494
|
+
*
|
|
495
|
+
* @param storage - The tier storage implementation
|
|
496
|
+
*/
|
|
154
497
|
constructor(storage: unknown);
|
|
498
|
+
/**
|
|
499
|
+
* Rolls back a failed migration job.
|
|
500
|
+
*
|
|
501
|
+
* @description
|
|
502
|
+
* Cleans up any partial data in the warm tier and releases the lock.
|
|
503
|
+
* Updates the job state to 'rolled_back'.
|
|
504
|
+
*
|
|
505
|
+
* @param job - The migration job to roll back
|
|
506
|
+
*
|
|
507
|
+
* @example
|
|
508
|
+
* ```typescript
|
|
509
|
+
* await rollback.rollback(failedJob);
|
|
510
|
+
* console.log(failedJob.state); // 'rolled_back'
|
|
511
|
+
* ```
|
|
512
|
+
*/
|
|
155
513
|
rollback(job: MigrationJob): Promise<void>;
|
|
156
514
|
}
|
|
157
515
|
/**
|
|
158
|
-
* Handler for concurrent access during migration
|
|
516
|
+
* Handler for concurrent access during migration.
|
|
517
|
+
*
|
|
518
|
+
* @description
|
|
519
|
+
* Manages read and write operations that occur while an object
|
|
520
|
+
* is being migrated, ensuring data consistency.
|
|
521
|
+
*
|
|
522
|
+
* During migration:
|
|
523
|
+
* - Reads check hot tier first (data still there), then warm tier
|
|
524
|
+
* - Writes go to hot tier and may be queued for replay
|
|
525
|
+
*
|
|
526
|
+
* @example
|
|
527
|
+
* ```typescript
|
|
528
|
+
* const handler = new ConcurrentAccessHandler(storage);
|
|
529
|
+
*
|
|
530
|
+
* // Safe read during migration
|
|
531
|
+
* const data = await handler.handleRead(sha);
|
|
532
|
+
*
|
|
533
|
+
* // Safe write during migration
|
|
534
|
+
* await handler.handleWrite(sha, newData);
|
|
535
|
+
* ```
|
|
159
536
|
*/
|
|
160
537
|
export declare class ConcurrentAccessHandler {
|
|
161
538
|
private storage;
|
|
539
|
+
/**
|
|
540
|
+
* Creates a new ConcurrentAccessHandler.
|
|
541
|
+
*
|
|
542
|
+
* @param storage - The tier storage implementation
|
|
543
|
+
*/
|
|
162
544
|
constructor(storage: unknown);
|
|
545
|
+
/**
|
|
546
|
+
* Handles a read operation during migration.
|
|
547
|
+
*
|
|
548
|
+
* @description
|
|
549
|
+
* Reads from hot tier first (data is still there during migration),
|
|
550
|
+
* then falls back to warm tier if not found.
|
|
551
|
+
*
|
|
552
|
+
* @param sha - The object SHA to read
|
|
553
|
+
*
|
|
554
|
+
* @returns Object data or null if not found
|
|
555
|
+
*
|
|
556
|
+
* @example
|
|
557
|
+
* ```typescript
|
|
558
|
+
* const data = await handler.handleRead(sha);
|
|
559
|
+
* if (data) {
|
|
560
|
+
* // Process the data
|
|
561
|
+
* }
|
|
562
|
+
* ```
|
|
563
|
+
*/
|
|
163
564
|
handleRead(sha: string): Promise<Uint8Array | null>;
|
|
565
|
+
/**
|
|
566
|
+
* Handles a write operation during migration.
|
|
567
|
+
*
|
|
568
|
+
* @description
|
|
569
|
+
* Writes to the hot tier. The TierMigrator will handle replaying
|
|
570
|
+
* pending writes after migration completes.
|
|
571
|
+
*
|
|
572
|
+
* @param sha - The object SHA to write
|
|
573
|
+
* @param data - The data to write
|
|
574
|
+
*
|
|
575
|
+
* @example
|
|
576
|
+
* ```typescript
|
|
577
|
+
* await handler.handleWrite(sha, newData);
|
|
578
|
+
* ```
|
|
579
|
+
*/
|
|
164
580
|
handleWrite(sha: string, data: Uint8Array): Promise<void>;
|
|
165
581
|
}
|
|
166
582
|
/**
|
|
167
|
-
* Tracks access patterns for objects to inform migration decisions
|
|
583
|
+
* Tracks access patterns for objects to inform migration decisions.
|
|
584
|
+
*
|
|
585
|
+
* @description
|
|
586
|
+
* Records and analyzes access patterns for objects in the storage system.
|
|
587
|
+
* This information is used to make intelligent decisions about which
|
|
588
|
+
* objects should be migrated between tiers.
|
|
589
|
+
*
|
|
590
|
+
* ## Features
|
|
591
|
+
*
|
|
592
|
+
* - Records read/write operations with optional metrics
|
|
593
|
+
* - Calculates access frequency over time
|
|
594
|
+
* - Identifies hot objects (frequently accessed)
|
|
595
|
+
* - Identifies cold objects (rarely accessed)
|
|
596
|
+
* - Supports access count decay for temporal relevance
|
|
597
|
+
* - Persists patterns to storage for durability
|
|
598
|
+
*
|
|
599
|
+
* @example
|
|
600
|
+
* ```typescript
|
|
601
|
+
* const tracker = new AccessTracker(storage);
|
|
602
|
+
*
|
|
603
|
+
* // Record accesses
|
|
604
|
+
* await tracker.recordAccess(sha, 'read', { bytesRead: 1024 });
|
|
605
|
+
* await tracker.recordAccess(sha, 'write');
|
|
606
|
+
*
|
|
607
|
+
* // Get access pattern for an object
|
|
608
|
+
* const pattern = await tracker.getAccessPattern(sha);
|
|
609
|
+
* console.log(`Frequency: ${pattern.accessFrequency}/sec`);
|
|
610
|
+
*
|
|
611
|
+
* // Find hot and cold objects
|
|
612
|
+
* const hotObjects = await tracker.identifyHotObjects({ minAccessCount: 50 });
|
|
613
|
+
* const coldObjects = await tracker.identifyColdObjects({ maxAccessCount: 2 });
|
|
614
|
+
*
|
|
615
|
+
* // Apply decay to gradually forget old patterns
|
|
616
|
+
* await tracker.applyDecay({ decayFactor: 0.5, minAgeForDecayMs: 86400000 });
|
|
617
|
+
* ```
|
|
168
618
|
*/
|
|
169
619
|
export declare class AccessTracker {
|
|
170
620
|
private storage;
|
|
171
621
|
private accessPatterns;
|
|
622
|
+
/**
|
|
623
|
+
* Creates a new AccessTracker.
|
|
624
|
+
*
|
|
625
|
+
* @param storage - The tier storage implementation
|
|
626
|
+
*
|
|
627
|
+
* @example
|
|
628
|
+
* ```typescript
|
|
629
|
+
* const tracker = new AccessTracker(storage);
|
|
630
|
+
* ```
|
|
631
|
+
*/
|
|
172
632
|
constructor(storage: unknown);
|
|
633
|
+
/**
|
|
634
|
+
* Records an access operation for an object.
|
|
635
|
+
*
|
|
636
|
+
* @description
|
|
637
|
+
* Tracks a read or write operation, updating the access pattern
|
|
638
|
+
* for the object. Can include optional metrics like bytes read
|
|
639
|
+
* and latency.
|
|
640
|
+
*
|
|
641
|
+
* @param sha - The object SHA being accessed
|
|
642
|
+
* @param type - Type of access ('read' or 'write')
|
|
643
|
+
* @param metrics - Optional additional metrics
|
|
644
|
+
*
|
|
645
|
+
* @example
|
|
646
|
+
* ```typescript
|
|
647
|
+
* // Basic access recording
|
|
648
|
+
* await tracker.recordAccess(sha, 'read');
|
|
649
|
+
*
|
|
650
|
+
* // With metrics
|
|
651
|
+
* await tracker.recordAccess(sha, 'read', {
|
|
652
|
+
* bytesRead: 2048,
|
|
653
|
+
* latencyMs: 5
|
|
654
|
+
* });
|
|
655
|
+
* ```
|
|
656
|
+
*/
|
|
173
657
|
recordAccess(sha: string, type: 'read' | 'write', metrics?: AccessMetrics): Promise<void>;
|
|
174
658
|
private persistPattern;
|
|
659
|
+
/**
|
|
660
|
+
* Gets the access pattern for a specific object.
|
|
661
|
+
*
|
|
662
|
+
* @description
|
|
663
|
+
* Returns detailed access statistics for an object including
|
|
664
|
+
* read/write counts, access frequency, and average latency.
|
|
665
|
+
*
|
|
666
|
+
* @param sha - The object SHA to query
|
|
667
|
+
*
|
|
668
|
+
* @returns Access pattern for the object
|
|
669
|
+
*
|
|
670
|
+
* @example
|
|
671
|
+
* ```typescript
|
|
672
|
+
* const pattern = await tracker.getAccessPattern(sha);
|
|
673
|
+
* console.log(`Reads: ${pattern.readCount}`);
|
|
674
|
+
* console.log(`Writes: ${pattern.writeCount}`);
|
|
675
|
+
* console.log(`Frequency: ${pattern.accessFrequency.toFixed(2)}/sec`);
|
|
676
|
+
* ```
|
|
677
|
+
*/
|
|
175
678
|
getAccessPattern(sha: string): Promise<AccessPattern>;
|
|
679
|
+
/**
|
|
680
|
+
* Identifies frequently accessed (hot) objects.
|
|
681
|
+
*
|
|
682
|
+
* @description
|
|
683
|
+
* Returns SHAs of objects that meet the hot object criteria,
|
|
684
|
+
* typically objects with high access counts.
|
|
685
|
+
*
|
|
686
|
+
* @param criteria - Criteria for identifying hot objects
|
|
687
|
+
*
|
|
688
|
+
* @returns Array of SHAs for hot objects
|
|
689
|
+
*
|
|
690
|
+
* @example
|
|
691
|
+
* ```typescript
|
|
692
|
+
* const hotObjects = await tracker.identifyHotObjects({
|
|
693
|
+
* minAccessCount: 100
|
|
694
|
+
* });
|
|
695
|
+
* console.log(`Found ${hotObjects.length} hot objects`);
|
|
696
|
+
* ```
|
|
697
|
+
*/
|
|
176
698
|
identifyHotObjects(criteria: ObjectIdentificationCriteria): Promise<string[]>;
|
|
699
|
+
/**
|
|
700
|
+
* Identifies rarely accessed (cold) objects.
|
|
701
|
+
*
|
|
702
|
+
* @description
|
|
703
|
+
* Returns SHAs of objects that meet the cold object criteria,
|
|
704
|
+
* typically objects with low access counts.
|
|
705
|
+
*
|
|
706
|
+
* @param criteria - Criteria for identifying cold objects
|
|
707
|
+
*
|
|
708
|
+
* @returns Array of SHAs for cold objects
|
|
709
|
+
*
|
|
710
|
+
* @example
|
|
711
|
+
* ```typescript
|
|
712
|
+
* const coldObjects = await tracker.identifyColdObjects({
|
|
713
|
+
* maxAccessCount: 2,
|
|
714
|
+
* minAgeMs: 7 * 24 * 60 * 60 * 1000 // 7 days
|
|
715
|
+
* });
|
|
716
|
+
* console.log(`Found ${coldObjects.length} cold objects for migration`);
|
|
717
|
+
* ```
|
|
718
|
+
*/
|
|
177
719
|
identifyColdObjects(criteria: ObjectIdentificationCriteria): Promise<string[]>;
|
|
720
|
+
/**
|
|
721
|
+
* Applies decay to access counts.
|
|
722
|
+
*
|
|
723
|
+
* @description
|
|
724
|
+
* Reduces access counts by a factor to gradually "forget" old access
|
|
725
|
+
* patterns. This helps the system respond to changing usage patterns
|
|
726
|
+
* over time.
|
|
727
|
+
*
|
|
728
|
+
* @param options - Decay configuration options
|
|
729
|
+
*
|
|
730
|
+
* @example
|
|
731
|
+
* ```typescript
|
|
732
|
+
* // Run daily to decay access counts by 50%
|
|
733
|
+
* await tracker.applyDecay({
|
|
734
|
+
* decayFactor: 0.5,
|
|
735
|
+
* minAgeForDecayMs: 0 // Apply to all
|
|
736
|
+
* });
|
|
737
|
+
* ```
|
|
738
|
+
*/
|
|
178
739
|
applyDecay(options: DecayOptions): Promise<void>;
|
|
740
|
+
/**
|
|
741
|
+
* Gets aggregate access statistics.
|
|
742
|
+
*
|
|
743
|
+
* @description
|
|
744
|
+
* Returns summary statistics about access patterns across all
|
|
745
|
+
* tracked objects.
|
|
746
|
+
*
|
|
747
|
+
* @returns Aggregate access statistics
|
|
748
|
+
*
|
|
749
|
+
* @example
|
|
750
|
+
* ```typescript
|
|
751
|
+
* const stats = await tracker.getAccessStats();
|
|
752
|
+
* console.log(`Total reads: ${stats.totalReads}`);
|
|
753
|
+
* console.log(`Total writes: ${stats.totalWrites}`);
|
|
754
|
+
* console.log(`Unique objects: ${stats.uniqueObjectsAccessed}`);
|
|
755
|
+
* ```
|
|
756
|
+
*/
|
|
179
757
|
getAccessStats(): Promise<AccessStats>;
|
|
758
|
+
/**
|
|
759
|
+
* Loads persisted access patterns from storage.
|
|
760
|
+
*
|
|
761
|
+
* @description
|
|
762
|
+
* Restores access patterns that were previously persisted,
|
|
763
|
+
* useful for recovering state after a restart.
|
|
764
|
+
*
|
|
765
|
+
* @example
|
|
766
|
+
* ```typescript
|
|
767
|
+
* // On startup, restore access patterns
|
|
768
|
+
* await tracker.loadFromStorage();
|
|
769
|
+
* ```
|
|
770
|
+
*/
|
|
180
771
|
loadFromStorage(): Promise<void>;
|
|
181
772
|
}
|
|
182
773
|
/**
|
|
183
|
-
* Main tier migration service
|
|
774
|
+
* Main tier migration service.
|
|
775
|
+
*
|
|
776
|
+
* @description
|
|
777
|
+
* Orchestrates the migration of Git objects between storage tiers.
|
|
778
|
+
* Provides both synchronous single-object migration and asynchronous
|
|
779
|
+
* job-based migration for long-running operations.
|
|
780
|
+
*
|
|
781
|
+
* ## Migration Process
|
|
782
|
+
*
|
|
783
|
+
* 1. Validate object exists and is not already in target tier
|
|
784
|
+
* 2. Acquire distributed lock with configurable timeout
|
|
785
|
+
* 3. Read data from source tier
|
|
786
|
+
* 4. Optionally compute source checksum
|
|
787
|
+
* 5. Write data to target tier
|
|
788
|
+
* 6. Optionally verify checksum matches
|
|
789
|
+
* 7. Update object index to point to new location
|
|
790
|
+
* 8. Delete from source tier
|
|
791
|
+
* 9. Release lock
|
|
792
|
+
*
|
|
793
|
+
* If any step fails, the migration is automatically rolled back.
|
|
794
|
+
*
|
|
795
|
+
* @example
|
|
796
|
+
* ```typescript
|
|
797
|
+
* const migrator = new TierMigrator(storage);
|
|
798
|
+
*
|
|
799
|
+
* // Simple migration
|
|
800
|
+
* const result = await migrator.migrate(sha, 'hot', 'r2');
|
|
801
|
+
* if (result.success) {
|
|
802
|
+
* console.log('Migration successful');
|
|
803
|
+
* }
|
|
804
|
+
*
|
|
805
|
+
* // Migration with verification
|
|
806
|
+
* const verifiedResult = await migrator.migrate(sha, 'hot', 'r2', {
|
|
807
|
+
* verifyChecksum: true,
|
|
808
|
+
* lockTimeout: 10000
|
|
809
|
+
* });
|
|
810
|
+
*
|
|
811
|
+
* // Batch migration
|
|
812
|
+
* const batchResult = await migrator.migrateBatch(shas, 'hot', 'r2', {
|
|
813
|
+
* concurrency: 5
|
|
814
|
+
* });
|
|
815
|
+
* console.log(`Migrated: ${batchResult.successful.length}`);
|
|
816
|
+
*
|
|
817
|
+
* // Long-running migration job
|
|
818
|
+
* const job = await migrator.startMigrationJob(largeSha, 'hot', 'r2');
|
|
819
|
+
* // ... later
|
|
820
|
+
* await migrator.completeMigrationJob(job);
|
|
821
|
+
* ```
|
|
184
822
|
*/
|
|
185
823
|
export declare class TierMigrator {
|
|
186
824
|
private storage;
|
|
@@ -188,54 +826,278 @@ export declare class TierMigrator {
|
|
|
188
826
|
private migrationHistory;
|
|
189
827
|
private migratingObjects;
|
|
190
828
|
private pendingWrites;
|
|
829
|
+
/**
|
|
830
|
+
* Creates a new TierMigrator.
|
|
831
|
+
*
|
|
832
|
+
* @param storage - The tier storage implementation
|
|
833
|
+
*
|
|
834
|
+
* @example
|
|
835
|
+
* ```typescript
|
|
836
|
+
* const migrator = new TierMigrator(storage);
|
|
837
|
+
* ```
|
|
838
|
+
*/
|
|
191
839
|
constructor(storage: unknown);
|
|
192
840
|
/**
|
|
193
|
-
*
|
|
841
|
+
* Finds objects that are candidates for migration based on policy.
|
|
842
|
+
*
|
|
843
|
+
* @description
|
|
844
|
+
* Analyzes objects in the hot tier and returns those that meet
|
|
845
|
+
* the migration criteria defined in the policy. Results are sorted
|
|
846
|
+
* by last access time (oldest first).
|
|
847
|
+
*
|
|
848
|
+
* @param policy - Migration policy defining criteria
|
|
849
|
+
*
|
|
850
|
+
* @returns Array of SHAs that are candidates for migration
|
|
851
|
+
*
|
|
852
|
+
* @example
|
|
853
|
+
* ```typescript
|
|
854
|
+
* const candidates = await migrator.findMigrationCandidates({
|
|
855
|
+
* maxAgeInHot: 7 * 24 * 60 * 60 * 1000, // 7 days
|
|
856
|
+
* minAccessCount: 5,
|
|
857
|
+
* maxHotSize: 100 * 1024 * 1024
|
|
858
|
+
* });
|
|
859
|
+
*
|
|
860
|
+
* console.log(`Found ${candidates.length} candidates for migration`);
|
|
861
|
+
* ```
|
|
194
862
|
*/
|
|
195
863
|
findMigrationCandidates(policy: MigrationPolicy): Promise<string[]>;
|
|
196
864
|
/**
|
|
197
|
-
*
|
|
865
|
+
* Migrates a single object between tiers.
|
|
866
|
+
*
|
|
867
|
+
* @description
|
|
868
|
+
* Performs a complete migration of an object from the source tier
|
|
869
|
+
* to the target tier. Handles locking, data transfer, verification,
|
|
870
|
+
* and cleanup.
|
|
871
|
+
*
|
|
872
|
+
* @param sha - The object SHA to migrate
|
|
873
|
+
* @param sourceTier - The source storage tier
|
|
874
|
+
* @param targetTier - The target storage tier
|
|
875
|
+
* @param options - Optional migration settings
|
|
876
|
+
*
|
|
877
|
+
* @returns Migration result with success/failure status
|
|
878
|
+
*
|
|
879
|
+
* @throws {MigrationError} If object not found or already in target tier
|
|
880
|
+
*
|
|
881
|
+
* @example
|
|
882
|
+
* ```typescript
|
|
883
|
+
* // Basic migration
|
|
884
|
+
* const result = await migrator.migrate(sha, 'hot', 'r2');
|
|
885
|
+
*
|
|
886
|
+
* // With checksum verification
|
|
887
|
+
* const verified = await migrator.migrate(sha, 'hot', 'r2', {
|
|
888
|
+
* verifyChecksum: true,
|
|
889
|
+
* lockTimeout: 10000
|
|
890
|
+
* });
|
|
891
|
+
*
|
|
892
|
+
* if (verified.success) {
|
|
893
|
+
* console.log('Migration successful');
|
|
894
|
+
* if (verified.checksumVerified) {
|
|
895
|
+
* console.log('Integrity verified');
|
|
896
|
+
* }
|
|
897
|
+
* } else if (verified.rolledBack) {
|
|
898
|
+
* console.log(`Rolled back: ${verified.rollbackReason}`);
|
|
899
|
+
* }
|
|
900
|
+
* ```
|
|
198
901
|
*/
|
|
199
902
|
migrate(sha: string, sourceTier: StorageTier, targetTier: StorageTier, options?: MigrateOptions): Promise<MigrationResult>;
|
|
200
903
|
private recordHistory;
|
|
201
904
|
/**
|
|
202
|
-
*
|
|
905
|
+
* Starts a long-running migration job.
|
|
906
|
+
*
|
|
907
|
+
* @description
|
|
908
|
+
* Initiates a migration job that can be monitored and completed
|
|
909
|
+
* asynchronously. Useful for large objects where progress tracking
|
|
910
|
+
* is important.
|
|
911
|
+
*
|
|
912
|
+
* @param sha - The object SHA to migrate
|
|
913
|
+
* @param sourceTier - The source storage tier
|
|
914
|
+
* @param targetTier - The target storage tier
|
|
915
|
+
*
|
|
916
|
+
* @returns The migration job with tracking information
|
|
917
|
+
*
|
|
918
|
+
* @example
|
|
919
|
+
* ```typescript
|
|
920
|
+
* const job = await migrator.startMigrationJob(largeSha, 'hot', 'r2');
|
|
921
|
+
* console.log(`Job ${job.id} started`);
|
|
922
|
+
* console.log(`Progress: ${job.progress.bytesTransferred}/${job.progress.totalBytes}`);
|
|
923
|
+
*
|
|
924
|
+
* // Complete the job when ready
|
|
925
|
+
* await migrator.completeMigrationJob(job);
|
|
926
|
+
* ```
|
|
203
927
|
*/
|
|
204
928
|
startMigrationJob(sha: string, sourceTier: StorageTier, targetTier: StorageTier): Promise<MigrationJob>;
|
|
205
929
|
/**
|
|
206
|
-
*
|
|
930
|
+
* Completes a migration job.
|
|
931
|
+
*
|
|
932
|
+
* @description
|
|
933
|
+
* Finalizes a migration job by updating the index and cleaning up
|
|
934
|
+
* the source tier. Also processes any pending writes that occurred
|
|
935
|
+
* during the migration.
|
|
936
|
+
*
|
|
937
|
+
* @param job - The migration job to complete
|
|
938
|
+
*
|
|
939
|
+
* @example
|
|
940
|
+
* ```typescript
|
|
941
|
+
* const job = await migrator.startMigrationJob(sha, 'hot', 'r2');
|
|
942
|
+
* // ... wait for progress or do other work
|
|
943
|
+
* await migrator.completeMigrationJob(job);
|
|
944
|
+
* console.log(`Job completed at ${new Date(job.completedAt!)}`);
|
|
945
|
+
* ```
|
|
207
946
|
*/
|
|
208
947
|
completeMigrationJob(job: MigrationJob): Promise<void>;
|
|
209
948
|
/**
|
|
210
|
-
*
|
|
949
|
+
* Rolls back a migration job.
|
|
950
|
+
*
|
|
951
|
+
* @description
|
|
952
|
+
* Cancels a migration job and cleans up any partial data in the
|
|
953
|
+
* target tier.
|
|
954
|
+
*
|
|
955
|
+
* @param job - The migration job to roll back
|
|
956
|
+
*
|
|
957
|
+
* @example
|
|
958
|
+
* ```typescript
|
|
959
|
+
* const job = await migrator.startMigrationJob(sha, 'hot', 'r2');
|
|
960
|
+
* if (someCondition) {
|
|
961
|
+
* await migrator.rollbackMigrationJob(job);
|
|
962
|
+
* console.log('Migration rolled back');
|
|
963
|
+
* }
|
|
964
|
+
* ```
|
|
211
965
|
*/
|
|
212
966
|
rollbackMigrationJob(job: MigrationJob): Promise<void>;
|
|
213
967
|
/**
|
|
214
|
-
*
|
|
968
|
+
* Cancels a migration job by ID.
|
|
969
|
+
*
|
|
970
|
+
* @description
|
|
971
|
+
* Stops a migration job and cleans up resources.
|
|
972
|
+
*
|
|
973
|
+
* @param jobId - The job ID to cancel
|
|
974
|
+
*
|
|
975
|
+
* @example
|
|
976
|
+
* ```typescript
|
|
977
|
+
* const job = await migrator.startMigrationJob(sha, 'hot', 'r2');
|
|
978
|
+
* // Later...
|
|
979
|
+
* await migrator.cancelMigrationJob(job.id);
|
|
980
|
+
* ```
|
|
215
981
|
*/
|
|
216
982
|
cancelMigrationJob(jobId: string): Promise<void>;
|
|
217
983
|
/**
|
|
218
|
-
*
|
|
984
|
+
* Gets all active migration jobs.
|
|
985
|
+
*
|
|
986
|
+
* @description
|
|
987
|
+
* Returns jobs that are currently in progress.
|
|
988
|
+
*
|
|
989
|
+
* @returns Array of active migration jobs
|
|
990
|
+
*
|
|
991
|
+
* @example
|
|
992
|
+
* ```typescript
|
|
993
|
+
* const activeJobs = await migrator.getActiveMigrationJobs();
|
|
994
|
+
* for (const job of activeJobs) {
|
|
995
|
+
* console.log(`${job.id}: ${job.sha} - ${job.progress.bytesTransferred}/${job.progress.totalBytes}`);
|
|
996
|
+
* }
|
|
997
|
+
* ```
|
|
219
998
|
*/
|
|
220
999
|
getActiveMigrationJobs(): Promise<MigrationJob[]>;
|
|
221
1000
|
/**
|
|
222
|
-
*
|
|
1001
|
+
* Gets migration history for an object.
|
|
1002
|
+
*
|
|
1003
|
+
* @description
|
|
1004
|
+
* Returns the history of migration events for a specific object.
|
|
1005
|
+
*
|
|
1006
|
+
* @param sha - The object SHA to query
|
|
1007
|
+
*
|
|
1008
|
+
* @returns Array of migration history entries
|
|
1009
|
+
*
|
|
1010
|
+
* @example
|
|
1011
|
+
* ```typescript
|
|
1012
|
+
* const history = await migrator.getMigrationHistory(sha);
|
|
1013
|
+
* for (const entry of history) {
|
|
1014
|
+
* console.log(`${new Date(entry.timestamp)}: ${entry.state}`);
|
|
1015
|
+
* }
|
|
1016
|
+
* ```
|
|
223
1017
|
*/
|
|
224
1018
|
getMigrationHistory(sha: string): Promise<MigrationHistoryEntry[]>;
|
|
225
1019
|
/**
|
|
226
|
-
*
|
|
1020
|
+
* Migrates multiple objects in a batch.
|
|
1021
|
+
*
|
|
1022
|
+
* @description
|
|
1023
|
+
* Efficiently migrates multiple objects with configurable concurrency.
|
|
1024
|
+
* Failed migrations don't affect other objects in the batch.
|
|
1025
|
+
*
|
|
1026
|
+
* @param shas - Array of object SHAs to migrate
|
|
1027
|
+
* @param sourceTier - The source storage tier
|
|
1028
|
+
* @param targetTier - The target storage tier
|
|
1029
|
+
* @param options - Optional batch migration settings
|
|
1030
|
+
*
|
|
1031
|
+
* @returns Result with successful and failed SHAs
|
|
1032
|
+
*
|
|
1033
|
+
* @example
|
|
1034
|
+
* ```typescript
|
|
1035
|
+
* const result = await migrator.migrateBatch(
|
|
1036
|
+
* candidates,
|
|
1037
|
+
* 'hot',
|
|
1038
|
+
* 'r2',
|
|
1039
|
+
* { concurrency: 5 }
|
|
1040
|
+
* );
|
|
1041
|
+
*
|
|
1042
|
+
* console.log(`Migrated: ${result.successful.length}`);
|
|
1043
|
+
* console.log(`Failed: ${result.failed.length}`);
|
|
1044
|
+
* ```
|
|
227
1045
|
*/
|
|
228
1046
|
migrateBatch(shas: string[], sourceTier: StorageTier, targetTier: StorageTier, options?: BatchMigrationOptions): Promise<BatchMigrationResult>;
|
|
229
1047
|
/**
|
|
230
|
-
*
|
|
1048
|
+
* Reads object data during an in-progress migration.
|
|
1049
|
+
*
|
|
1050
|
+
* @description
|
|
1051
|
+
* Safely reads data for an object that may be in the process of
|
|
1052
|
+
* being migrated. Checks hot tier first, then warm tier.
|
|
1053
|
+
*
|
|
1054
|
+
* @param sha - The object SHA to read
|
|
1055
|
+
*
|
|
1056
|
+
* @returns Object data or null if not found
|
|
1057
|
+
*
|
|
1058
|
+
* @example
|
|
1059
|
+
* ```typescript
|
|
1060
|
+
* const data = await migrator.readDuringMigration(sha);
|
|
1061
|
+
* if (data) {
|
|
1062
|
+
* // Process the data regardless of which tier it's in
|
|
1063
|
+
* }
|
|
1064
|
+
* ```
|
|
231
1065
|
*/
|
|
232
1066
|
readDuringMigration(sha: string): Promise<Uint8Array | null>;
|
|
233
1067
|
/**
|
|
234
|
-
*
|
|
1068
|
+
* Writes object data during an in-progress migration.
|
|
1069
|
+
*
|
|
1070
|
+
* @description
|
|
1071
|
+
* Safely handles writes for an object that may be in the process
|
|
1072
|
+
* of being migrated. If the object is being migrated, the write
|
|
1073
|
+
* is queued and replayed after migration completes.
|
|
1074
|
+
*
|
|
1075
|
+
* @param sha - The object SHA to write
|
|
1076
|
+
* @param data - The data to write
|
|
1077
|
+
*
|
|
1078
|
+
* @example
|
|
1079
|
+
* ```typescript
|
|
1080
|
+
* // This is safe to call even during migration
|
|
1081
|
+
* await migrator.writeDuringMigration(sha, newData);
|
|
1082
|
+
* ```
|
|
235
1083
|
*/
|
|
236
1084
|
writeDuringMigration(sha: string, data: Uint8Array): Promise<void>;
|
|
237
1085
|
/**
|
|
238
|
-
*
|
|
1086
|
+
* Computes SHA-256 checksum for data verification.
|
|
1087
|
+
*
|
|
1088
|
+
* @description
|
|
1089
|
+
* Calculates a SHA-256 hash of the data for integrity verification
|
|
1090
|
+
* during migration.
|
|
1091
|
+
*
|
|
1092
|
+
* @param data - The data to hash
|
|
1093
|
+
*
|
|
1094
|
+
* @returns Hex-encoded SHA-256 hash
|
|
1095
|
+
*
|
|
1096
|
+
* @example
|
|
1097
|
+
* ```typescript
|
|
1098
|
+
* const checksum = await migrator.computeChecksum(data);
|
|
1099
|
+
* console.log(`Checksum: ${checksum}`);
|
|
1100
|
+
* ```
|
|
239
1101
|
*/
|
|
240
1102
|
computeChecksum(data: Uint8Array): Promise<string>;
|
|
241
1103
|
}
|