@qvac/registry-client 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -17
- package/bin/cli.js +33 -16
- package/index.d.ts +62 -21
- package/lib/client.js +242 -119
- package/lib/config.js +8 -5
- package/lib/logger.js +7 -7
- package/lib/profiler.js +16 -7
- package/package.json +9 -5
- package/utils/env.js +1 -1
- package/utils/error.js +19 -16
- package/utils/retry.js +8 -3
package/README.md
CHANGED
|
@@ -26,7 +26,7 @@ Ensure the registry core key is available via environment variables or provided
|
|
|
26
26
|
'use strict'
|
|
27
27
|
const { QVACRegistryClient } = require('@qvac/registry-client')
|
|
28
28
|
|
|
29
|
-
async function main
|
|
29
|
+
async function main() {
|
|
30
30
|
const client = new QVACRegistryClient({
|
|
31
31
|
registryCoreKey: process.env.QVAC_REGISTRY_CORE_KEY
|
|
32
32
|
})
|
|
@@ -120,16 +120,19 @@ result.artifact.stream.pipe(fs.createWriteStream('./model.ggml'))
|
|
|
120
120
|
This method only waits for the network layer (Corestore + Hyperswarm) — it does not wait for the metadata core to sync, making it faster for known blob coordinates.
|
|
121
121
|
|
|
122
122
|
```javascript
|
|
123
|
-
const result = await client.downloadBlob(
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
123
|
+
const result = await client.downloadBlob(
|
|
124
|
+
{
|
|
125
|
+
coreKey: 'ey46cahego89xox118uhyryakz47bcs8bbxu97tnnpmuwmgi5wmo',
|
|
126
|
+
blockOffset: 0,
|
|
127
|
+
blockLength: 665,
|
|
128
|
+
byteOffset: 0,
|
|
129
|
+
byteLength: 43537433
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
outputFile: './downloaded/ggml-tiny-q8_0.bin',
|
|
133
|
+
timeout: 60000
|
|
134
|
+
}
|
|
135
|
+
)
|
|
133
136
|
console.log('Downloaded to:', result.artifact.path)
|
|
134
137
|
```
|
|
135
138
|
|
|
@@ -332,11 +335,11 @@ The client uses custom error codes in the range 19001-20000. All errors extend `
|
|
|
332
335
|
|
|
333
336
|
### Error Codes
|
|
334
337
|
|
|
335
|
-
| Code
|
|
336
|
-
|
|
337
|
-
| 19001 | FAILED_TO_CONNECT | Connection to registry failed
|
|
338
|
-
| 19002 | FAILED_TO_CLOSE
|
|
339
|
-
| 19003 | MODEL_NOT_FOUND
|
|
338
|
+
| Code | Name | Description | When Thrown |
|
|
339
|
+
| ----- | ----------------- | -------------------------------- | ------------------------------------------------------ |
|
|
340
|
+
| 19001 | FAILED_TO_CONNECT | Connection to registry failed | Missing core key, network issues during initialization |
|
|
341
|
+
| 19002 | FAILED_TO_CLOSE | Failed to close registry cleanly | Resource cleanup errors during shutdown |
|
|
342
|
+
| 19003 | MODEL_NOT_FOUND | Model not found or invalid | Model doesn't exist or missing blob binding |
|
|
340
343
|
|
|
341
344
|
### Error Handling Example
|
|
342
345
|
|
|
@@ -344,7 +347,7 @@ The client uses custom error codes in the range 19001-20000. All errors extend `
|
|
|
344
347
|
const { QVACRegistryClient } = require('@qvac/registry-client')
|
|
345
348
|
const { QvacErrorRegistryClient } = require('@qvac/registry-client/utils/error')
|
|
346
349
|
|
|
347
|
-
async function handleErrors
|
|
350
|
+
async function handleErrors() {
|
|
348
351
|
const client = new QVACRegistryClient({
|
|
349
352
|
registryCoreKey: process.env.QVAC_REGISTRY_CORE_KEY
|
|
350
353
|
})
|
package/bin/cli.js
CHANGED
|
@@ -11,7 +11,7 @@ const VERSION = require('../package.json').version
|
|
|
11
11
|
|
|
12
12
|
// --- Helpers ---
|
|
13
13
|
|
|
14
|
-
function createClient
|
|
14
|
+
function createClient(parentFlags) {
|
|
15
15
|
const opts = {}
|
|
16
16
|
if (parentFlags.key) opts.registryCoreKey = parentFlags.key
|
|
17
17
|
if (parentFlags.storage) opts.storage = parentFlags.storage
|
|
@@ -23,13 +23,13 @@ function createClient (parentFlags) {
|
|
|
23
23
|
return new QVACRegistryClient(opts)
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
function getRootFlags
|
|
26
|
+
function getRootFlags(cmd) {
|
|
27
27
|
let current = cmd.command || cmd
|
|
28
28
|
while (current.parent) current = current.parent
|
|
29
29
|
return current.flags || {}
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
function formatSize
|
|
32
|
+
function formatSize(bytes) {
|
|
33
33
|
if (!bytes) return 'N/A'
|
|
34
34
|
if (bytes >= 1024 * 1024 * 1024) return (bytes / (1024 * 1024 * 1024)).toFixed(2) + ' GB'
|
|
35
35
|
if (bytes >= 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(2) + ' MB'
|
|
@@ -37,14 +37,14 @@ function formatSize (bytes) {
|
|
|
37
37
|
return bytes + ' B'
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
function printModelCompact
|
|
40
|
+
function printModelCompact(model) {
|
|
41
41
|
const parts = [model.path, model.source]
|
|
42
42
|
if (model.quantization) parts.push(model.quantization)
|
|
43
43
|
if (model.params) parts.push(model.params)
|
|
44
44
|
console.log(parts.join('\t'))
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
function printModelFull
|
|
47
|
+
function printModelFull(model, opts = {}) {
|
|
48
48
|
const blob = model.blobBinding || {}
|
|
49
49
|
const coreKey = blob.coreKey ? IdEnc.normalize(blob.coreKey) : 'N/A'
|
|
50
50
|
|
|
@@ -55,7 +55,9 @@ function printModelFull (model, opts = {}) {
|
|
|
55
55
|
if (model.quantization) console.log(` quantization: ${model.quantization}`)
|
|
56
56
|
if (model.params) console.log(` params: ${model.params}`)
|
|
57
57
|
console.log(` size: ${formatSize(blob.byteLength || model.sizeBytes)}`)
|
|
58
|
-
if (model.licenseId || model.license)
|
|
58
|
+
if (model.licenseId || model.license) {
|
|
59
|
+
console.log(` license: ${model.licenseId || model.license}`)
|
|
60
|
+
}
|
|
59
61
|
if (blob.sha256 || model.sha256) console.log(` sha256: ${blob.sha256 || model.sha256}`)
|
|
60
62
|
if (opts.verbose && model.description) console.log(` description: ${model.description}`)
|
|
61
63
|
if (opts.verbose) console.log(` blob core: ${coreKey}`)
|
|
@@ -63,7 +65,7 @@ function printModelFull (model, opts = {}) {
|
|
|
63
65
|
console.log()
|
|
64
66
|
}
|
|
65
67
|
|
|
66
|
-
async function withClient
|
|
68
|
+
async function withClient(cmd, fn) {
|
|
67
69
|
const rootFlags = getRootFlags(cmd)
|
|
68
70
|
const client = createClient(rootFlags)
|
|
69
71
|
try {
|
|
@@ -75,7 +77,8 @@ async function withClient (cmd, fn) {
|
|
|
75
77
|
|
|
76
78
|
// --- Commands ---
|
|
77
79
|
|
|
78
|
-
const listCmd = command(
|
|
80
|
+
const listCmd = command(
|
|
81
|
+
'list',
|
|
79
82
|
summary('List models from the registry'),
|
|
80
83
|
description('List all models or filter by name, engine, or quantization.'),
|
|
81
84
|
flag('--name|-n [name]', 'Filter by model name (partial match)'),
|
|
@@ -103,10 +106,13 @@ const listCmd = command('list',
|
|
|
103
106
|
}
|
|
104
107
|
|
|
105
108
|
if (flags.json) {
|
|
106
|
-
const out = models.map(m => ({
|
|
109
|
+
const out = models.map((m) => ({
|
|
107
110
|
...m,
|
|
108
111
|
blobBinding: m.blobBinding
|
|
109
|
-
? {
|
|
112
|
+
? {
|
|
113
|
+
...m.blobBinding,
|
|
114
|
+
coreKey: m.blobBinding.coreKey ? IdEnc.normalize(m.blobBinding.coreKey) : undefined
|
|
115
|
+
}
|
|
110
116
|
: undefined
|
|
111
117
|
}))
|
|
112
118
|
console.log(JSON.stringify(out, null, 2))
|
|
@@ -129,7 +135,8 @@ const listCmd = command('list',
|
|
|
129
135
|
}
|
|
130
136
|
)
|
|
131
137
|
|
|
132
|
-
const getCmd = command(
|
|
138
|
+
const getCmd = command(
|
|
139
|
+
'get',
|
|
133
140
|
summary('Get a specific model by path and source'),
|
|
134
141
|
arg('<path>', 'Model path (e.g. hf/org/repo/file.gguf)'),
|
|
135
142
|
arg('<source>', 'Model source (e.g. hf, s3)'),
|
|
@@ -149,7 +156,12 @@ const getCmd = command('get',
|
|
|
149
156
|
const out = {
|
|
150
157
|
...model,
|
|
151
158
|
blobBinding: model.blobBinding
|
|
152
|
-
? {
|
|
159
|
+
? {
|
|
160
|
+
...model.blobBinding,
|
|
161
|
+
coreKey: model.blobBinding.coreKey
|
|
162
|
+
? IdEnc.normalize(model.blobBinding.coreKey)
|
|
163
|
+
: undefined
|
|
164
|
+
}
|
|
153
165
|
: undefined
|
|
154
166
|
}
|
|
155
167
|
console.log(JSON.stringify(out, null, 2))
|
|
@@ -161,7 +173,8 @@ const getCmd = command('get',
|
|
|
161
173
|
}
|
|
162
174
|
)
|
|
163
175
|
|
|
164
|
-
const downloadCmd = command(
|
|
176
|
+
const downloadCmd = command(
|
|
177
|
+
'download',
|
|
165
178
|
summary('Download a model from the registry'),
|
|
166
179
|
arg('<path>', 'Model path'),
|
|
167
180
|
arg('<source>', 'Model source (e.g. hf, s3)'),
|
|
@@ -192,9 +205,12 @@ const downloadCmd = command('download',
|
|
|
192
205
|
}
|
|
193
206
|
)
|
|
194
207
|
|
|
195
|
-
const profileCmd = command(
|
|
208
|
+
const profileCmd = command(
|
|
209
|
+
'profile',
|
|
196
210
|
summary('Profile download performance for a model'),
|
|
197
|
-
description(
|
|
211
|
+
description(
|
|
212
|
+
'Downloads a model to a temp directory while collecting UDX network stats, connection info, and hypercore metrics. Similar to hyperdrive-profiler but for registry blobs.'
|
|
213
|
+
),
|
|
198
214
|
arg('<path>', 'Model path'),
|
|
199
215
|
arg('[source]', 'Model source filter (e.g. hf, s3)'),
|
|
200
216
|
flag('--interval|-i [seconds]', 'Stats print interval in seconds (default: 5)'),
|
|
@@ -232,7 +248,8 @@ const profileCmd = command('profile',
|
|
|
232
248
|
|
|
233
249
|
// --- Root command ---
|
|
234
250
|
|
|
235
|
-
const cmd = command(
|
|
251
|
+
const cmd = command(
|
|
252
|
+
'qvac-registry',
|
|
236
253
|
header(`qvac-registry v${VERSION}`),
|
|
237
254
|
summary('QVAC Model Registry CLI'),
|
|
238
255
|
footer('Set QVAC_REGISTRY_CORE_KEY env var or use --key to specify the registry core key.'),
|
package/index.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ export interface QVACBlobBinding {
|
|
|
10
10
|
blockLength: number
|
|
11
11
|
byteOffset: number
|
|
12
12
|
byteLength: number
|
|
13
|
+
sha256: string
|
|
13
14
|
}
|
|
14
15
|
|
|
15
16
|
export interface QVACModelEntry {
|
|
@@ -26,6 +27,29 @@ export interface QVACModelEntry {
|
|
|
26
27
|
notes?: string
|
|
27
28
|
tags?: string[]
|
|
28
29
|
blobBinding: QVACBlobBinding
|
|
30
|
+
/**
|
|
31
|
+
* Raw GGUF key-value metadata extracted at ingest, serialised as a JSON string.
|
|
32
|
+
*
|
|
33
|
+
* - Only present for GGUF artifacts; absent for other formats and when
|
|
34
|
+
* extraction failed (e.g. some GGUF v3 files).
|
|
35
|
+
* - Tensor infos are discarded and every `tokenizer.*` key is stripped.
|
|
36
|
+
* - For sharded models only the first shard (`-00001-of-`) is described.
|
|
37
|
+
* - Numeric values that exceeded the safe integer range are stringified.
|
|
38
|
+
*/
|
|
39
|
+
ggufMetadata?: string
|
|
40
|
+
/**
|
|
41
|
+
* Points at a weightless description of this artifact: the tensor list an
|
|
42
|
+
* engine's dry-run fitter reads, with none of the weights.
|
|
43
|
+
*
|
|
44
|
+
* - For a GGUF it is a short GGUF holding the tensor list and the settings
|
|
45
|
+
* without the tokenizer tables; for a safetensors it is the JSON header.
|
|
46
|
+
* - Present for GGUF and safetensors artifacts, every shard of a split model
|
|
47
|
+
* included; absent for other formats and when it could not be built.
|
|
48
|
+
* - Written to the writer's active blob core, which is not always the one
|
|
49
|
+
* holding the weights; `downloadBlob` takes the binding directly.
|
|
50
|
+
* - `sha256` covers the description, not the artifact it came from.
|
|
51
|
+
*/
|
|
52
|
+
fitBlobBinding?: QVACBlobBinding
|
|
29
53
|
}
|
|
30
54
|
|
|
31
55
|
export interface QVACDownloadedArtifactStream {
|
|
@@ -48,7 +72,12 @@ export interface QVACDownloadOptions {
|
|
|
48
72
|
peerTimeout?: number
|
|
49
73
|
maxRetries?: number
|
|
50
74
|
outputFile?: string
|
|
51
|
-
onProgress?: (progress: {
|
|
75
|
+
onProgress?: (progress: {
|
|
76
|
+
downloaded: number
|
|
77
|
+
total: number
|
|
78
|
+
cachedBlocks: number
|
|
79
|
+
totalBlocks: number
|
|
80
|
+
}) => void
|
|
52
81
|
signal?: AbortSignal
|
|
53
82
|
}
|
|
54
83
|
|
|
@@ -56,7 +85,12 @@ export interface QVACBlobDownloadOptions {
|
|
|
56
85
|
timeout?: number
|
|
57
86
|
maxRetries?: number
|
|
58
87
|
outputFile?: string
|
|
59
|
-
onProgress?: (progress: {
|
|
88
|
+
onProgress?: (progress: {
|
|
89
|
+
downloaded: number
|
|
90
|
+
total: number
|
|
91
|
+
cachedBlocks: number
|
|
92
|
+
totalBlocks: number
|
|
93
|
+
}) => void
|
|
60
94
|
signal?: AbortSignal
|
|
61
95
|
}
|
|
62
96
|
|
|
@@ -91,35 +125,42 @@ export interface FindByParams {
|
|
|
91
125
|
|
|
92
126
|
export interface LifecycleSwarmHandle {
|
|
93
127
|
readonly suspended: boolean
|
|
94
|
-
suspend
|
|
95
|
-
resume
|
|
128
|
+
suspend(opts?: LifecycleLogOptions): Promise<void>
|
|
129
|
+
resume(opts?: LifecycleLogOptions): Promise<void>
|
|
96
130
|
}
|
|
97
131
|
|
|
98
132
|
export interface LifecycleStoreHandle {
|
|
99
|
-
suspend
|
|
100
|
-
resume
|
|
133
|
+
suspend(opts?: LifecycleLogOptions): Promise<void>
|
|
134
|
+
resume(): Promise<void>
|
|
101
135
|
}
|
|
102
136
|
|
|
103
137
|
export class QVACRegistryClient extends ReadyResource {
|
|
104
|
-
constructor
|
|
138
|
+
constructor(opts?: QVACRegistryClientOptions)
|
|
105
139
|
|
|
106
140
|
/** Valid only while the client remains open. Cached handles become stale after close(). */
|
|
107
141
|
readonly corestore: LifecycleStoreHandle | null
|
|
108
142
|
/** Valid only while the client remains open. Cached handles become stale after close(). */
|
|
109
143
|
readonly hyperswarm: LifecycleSwarmHandle | null
|
|
110
144
|
|
|
111
|
-
ready
|
|
112
|
-
close
|
|
113
|
-
suspend
|
|
114
|
-
resume
|
|
115
|
-
|
|
116
|
-
getModel
|
|
117
|
-
downloadModel
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
145
|
+
ready(): Promise<void>
|
|
146
|
+
close(): Promise<void>
|
|
147
|
+
suspend(opts?: LifecycleLogOptions): Promise<void>
|
|
148
|
+
resume(opts?: LifecycleLogOptions): Promise<void>
|
|
149
|
+
|
|
150
|
+
getModel(path: string, source: string): Promise<QVACModelEntry | null>
|
|
151
|
+
downloadModel(
|
|
152
|
+
path: string,
|
|
153
|
+
source: string,
|
|
154
|
+
options?: QVACDownloadOptions
|
|
155
|
+
): Promise<QVACDownloadResult>
|
|
156
|
+
downloadBlob(
|
|
157
|
+
blobBinding: QVACBlobBinding,
|
|
158
|
+
options?: QVACBlobDownloadOptions
|
|
159
|
+
): Promise<QVACBlobDownloadResult>
|
|
160
|
+
|
|
161
|
+
findBy(params?: FindByParams): Promise<QVACModelEntry[]>
|
|
162
|
+
findModels(query?: QVACModelQuery): Promise<QVACModelEntry[]>
|
|
163
|
+
findModelsByEngine(query?: QVACModelQuery): Promise<QVACModelEntry[]>
|
|
164
|
+
findModelsByName(query?: QVACModelQuery): Promise<QVACModelEntry[]>
|
|
165
|
+
findModelsByQuantization(query?: QVACModelQuery): Promise<QVACModelEntry[]>
|
|
125
166
|
}
|
package/lib/client.js
CHANGED
|
@@ -15,9 +15,17 @@ const fs = require('#fs')
|
|
|
15
15
|
|
|
16
16
|
const DEFAULT_DOWNLOAD_MAX_RETRIES = 3
|
|
17
17
|
const RETRIABLE_DOWNLOAD_CODES = ['REQUEST_TIMEOUT']
|
|
18
|
+
const INFLIGHT_DRAIN_TIMEOUT_MS = 5000
|
|
19
|
+
const INFLIGHT_DRAIN_POLL_MS = 10
|
|
20
|
+
|
|
21
|
+
// While the app is backgrounded the swarm is suspended; a retry must wait for
|
|
22
|
+
// resume rather than burn its (small) retry budget timing out against a dead
|
|
23
|
+
// swarm. Bounded so a never-resumed runtime still fails instead of hanging.
|
|
24
|
+
const RESUME_WAIT_MAX_MS = 5 * 60 * 1000
|
|
25
|
+
const RESUME_WAIT_POLL_MS = 200
|
|
18
26
|
|
|
19
27
|
class QVACRegistryClient extends ReadyResource {
|
|
20
|
-
constructor
|
|
28
|
+
constructor(opts = {}) {
|
|
21
29
|
super()
|
|
22
30
|
|
|
23
31
|
this.logger = new Logger(opts.logger)
|
|
@@ -40,7 +48,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
40
48
|
this.ready()
|
|
41
49
|
}
|
|
42
50
|
|
|
43
|
-
async _open
|
|
51
|
+
async _open() {
|
|
44
52
|
this.logger.debug('_open called')
|
|
45
53
|
|
|
46
54
|
this.logger.debug('Opening corestore')
|
|
@@ -59,10 +67,13 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
59
67
|
await this._metadataReady
|
|
60
68
|
}
|
|
61
69
|
|
|
62
|
-
async _connectMetadataCore
|
|
70
|
+
async _connectMetadataCore() {
|
|
63
71
|
if (!this.registryCoreKey) {
|
|
64
72
|
this.logger.error('Missing registry core key for read mode')
|
|
65
|
-
throw new QvacErrorRegistryClient({
|
|
73
|
+
throw new QvacErrorRegistryClient({
|
|
74
|
+
code: ERR_CODES.FAILED_TO_CONNECT,
|
|
75
|
+
adds: 'Missing registry core key. Set QVAC_REGISTRY_CORE_KEY environment variable.'
|
|
76
|
+
})
|
|
66
77
|
}
|
|
67
78
|
|
|
68
79
|
const viewKey = IdEnc.decode(this.registryCoreKey)
|
|
@@ -90,15 +101,18 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
90
101
|
})
|
|
91
102
|
}
|
|
92
103
|
|
|
93
|
-
async _ensureMetadata
|
|
104
|
+
async _ensureMetadata() {
|
|
94
105
|
await this.ready()
|
|
95
106
|
await this._metadataReady
|
|
96
107
|
if (!this.db) {
|
|
97
|
-
throw new QvacErrorRegistryClient({
|
|
108
|
+
throw new QvacErrorRegistryClient({
|
|
109
|
+
code: ERR_CODES.FAILED_TO_CONNECT,
|
|
110
|
+
adds: 'Registry database not available.'
|
|
111
|
+
})
|
|
98
112
|
}
|
|
99
113
|
}
|
|
100
114
|
|
|
101
|
-
async getModel
|
|
115
|
+
async getModel(path, source) {
|
|
102
116
|
this._validateString(path, 'path')
|
|
103
117
|
this._validateString(source, 'source')
|
|
104
118
|
|
|
@@ -115,7 +129,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
115
129
|
}
|
|
116
130
|
}
|
|
117
131
|
|
|
118
|
-
async findModels
|
|
132
|
+
async findModels(query = {}, opts = {}) {
|
|
119
133
|
await this._ensureMetadata()
|
|
120
134
|
const { includeDeprecated = false } = opts
|
|
121
135
|
this.logger.debug('findModels called', { query, includeDeprecated })
|
|
@@ -123,25 +137,25 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
123
137
|
let models = await this.db.findModelsByPath(query).toArray()
|
|
124
138
|
|
|
125
139
|
if (!includeDeprecated) {
|
|
126
|
-
models = models.filter(m => !m.deprecated)
|
|
140
|
+
models = models.filter((m) => !m.deprecated)
|
|
127
141
|
}
|
|
128
142
|
|
|
129
143
|
return models
|
|
130
144
|
}
|
|
131
145
|
|
|
132
|
-
async findModelsByEngine
|
|
146
|
+
async findModelsByEngine(query = {}) {
|
|
133
147
|
await this._ensureMetadata()
|
|
134
148
|
this.logger.debug('findModelsByEngine called', { query })
|
|
135
149
|
return this.db.findModelsByEngine(query).toArray()
|
|
136
150
|
}
|
|
137
151
|
|
|
138
|
-
async findModelsByName
|
|
152
|
+
async findModelsByName(query = {}) {
|
|
139
153
|
await this._ensureMetadata()
|
|
140
154
|
this.logger.debug('findModelsByName called', { query })
|
|
141
155
|
return this.db.findModelsByName(query).toArray()
|
|
142
156
|
}
|
|
143
157
|
|
|
144
|
-
async findModelsByQuantization
|
|
158
|
+
async findModelsByQuantization(query = {}) {
|
|
145
159
|
await this._ensureMetadata()
|
|
146
160
|
this.logger.debug('findModelsByQuantization called', { query })
|
|
147
161
|
return this.db.findModelsByQuantization(query).toArray()
|
|
@@ -157,19 +171,19 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
157
171
|
* @param {boolean} [params.includeDeprecated=false] - Include deprecated models
|
|
158
172
|
* @returns {Promise<Array>} Array of matching models
|
|
159
173
|
*/
|
|
160
|
-
async findBy
|
|
174
|
+
async findBy(params = {}) {
|
|
161
175
|
await this._ensureMetadata()
|
|
162
176
|
this.logger.debug('findBy called', { params })
|
|
163
177
|
return this.db.findBy(params)
|
|
164
178
|
}
|
|
165
179
|
|
|
166
|
-
_validateString
|
|
180
|
+
_validateString(value, name) {
|
|
167
181
|
if (typeof value !== 'string' || value.length === 0) {
|
|
168
182
|
throw new Error(`Invalid ${name}: ${value}`)
|
|
169
183
|
}
|
|
170
184
|
}
|
|
171
185
|
|
|
172
|
-
async _checkBlobProgress
|
|
186
|
+
async _checkBlobProgress(core, blobPointer) {
|
|
173
187
|
const totalBlocks = blobPointer.blockLength
|
|
174
188
|
const totalBytes = blobPointer.byteLength
|
|
175
189
|
|
|
@@ -186,7 +200,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
186
200
|
return { cachedBlocks, totalBlocks, totalBytes }
|
|
187
201
|
}
|
|
188
202
|
|
|
189
|
-
async _getBlobsCore
|
|
203
|
+
async _getBlobsCore(blobsCoreKey) {
|
|
190
204
|
let keyBuffer
|
|
191
205
|
if (Buffer.isBuffer(blobsCoreKey)) {
|
|
192
206
|
keyBuffer = blobsCoreKey
|
|
@@ -209,7 +223,82 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
209
223
|
return { core, blobs }
|
|
210
224
|
}
|
|
211
225
|
|
|
212
|
-
|
|
226
|
+
/**
|
|
227
|
+
* Blocks while the swarm is suspended (app backgrounded), so a retry does not
|
|
228
|
+
* fire against a swarm that cannot connect yet and exhaust the retry budget.
|
|
229
|
+
* Bounded by RESUME_WAIT_MAX_MS; returns (and lets the retry proceed/fail) if
|
|
230
|
+
* the runtime never resumes.
|
|
231
|
+
*/
|
|
232
|
+
async _waitForSwarmResumed(signal) {
|
|
233
|
+
if (!this.hyperswarm || !this.hyperswarm.suspended) return
|
|
234
|
+
|
|
235
|
+
const start = Date.now()
|
|
236
|
+
while (this.hyperswarm.suspended) {
|
|
237
|
+
if (signal && signal.aborted) throw new Error('Download cancelled')
|
|
238
|
+
if (Date.now() - start > RESUME_WAIT_MAX_MS) {
|
|
239
|
+
this.logger.warn('Swarm still suspended after resume wait; retrying anyway')
|
|
240
|
+
return
|
|
241
|
+
}
|
|
242
|
+
await new Promise((resolve) => setTimeout(resolve, RESUME_WAIT_POLL_MS))
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Blocks until at least one peer is replicating the core, so a retry after a
|
|
248
|
+
* network drop waits for the network to actually return instead of firing
|
|
249
|
+
* (and timing out) against zero peers and burning the retry budget. Bounded
|
|
250
|
+
* by RESUME_WAIT_MAX_MS. No-op when peer info is unavailable.
|
|
251
|
+
*/
|
|
252
|
+
async _waitForPeers(core, signal) {
|
|
253
|
+
if (!core || !Array.isArray(core.peers)) return
|
|
254
|
+
if (core.peers.length > 0) return
|
|
255
|
+
|
|
256
|
+
const start = Date.now()
|
|
257
|
+
while (core.peers.length === 0) {
|
|
258
|
+
if (signal && signal.aborted) throw new Error('Download cancelled')
|
|
259
|
+
if (Date.now() - start > RESUME_WAIT_MAX_MS) {
|
|
260
|
+
this.logger.warn('No peers after reconnect wait; retrying anyway')
|
|
261
|
+
return
|
|
262
|
+
}
|
|
263
|
+
await new Promise((resolve) => setTimeout(resolve, RESUME_WAIT_POLL_MS))
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Re-establish peers for a blobs core before retrying a download. After the
|
|
269
|
+
* app backgrounds (or the network drops) the swarm connection for this core
|
|
270
|
+
* is gone; this waits for the swarm to resume and for a peer to be replicating
|
|
271
|
+
* the core, then re-runs the join + findingPeers + update sequence and awaits
|
|
272
|
+
* it. Resume itself is cheap: the next attempt reuses the blocks already
|
|
273
|
+
* cached in the core (the output file is re-streamed from those blocks, not
|
|
274
|
+
* appended to).
|
|
275
|
+
*
|
|
276
|
+
* Honours `signal`: a foreground cancel during the (bounded but up to
|
|
277
|
+
* RESUME_WAIT_MAX_MS) swarm/peer waits aborts promptly instead of blocking
|
|
278
|
+
* until peers return or the cap elapses.
|
|
279
|
+
*/
|
|
280
|
+
async _reconnectCore(core, signal) {
|
|
281
|
+
if (!core || !this.hyperswarm) return
|
|
282
|
+
if (signal && signal.aborted) throw new Error('Download cancelled')
|
|
283
|
+
|
|
284
|
+
await this._waitForSwarmResumed(signal)
|
|
285
|
+
|
|
286
|
+
this.logger.debug('Re-establishing peers before download retry', {
|
|
287
|
+
discoveryKey: IdEnc.normalize(core.discoveryKey)
|
|
288
|
+
})
|
|
289
|
+
|
|
290
|
+
const done = core.findingPeers()
|
|
291
|
+
this.hyperswarm.join(core.discoveryKey, { client: true, server: false })
|
|
292
|
+
try {
|
|
293
|
+
await this.hyperswarm.flush()
|
|
294
|
+
} finally {
|
|
295
|
+
done()
|
|
296
|
+
}
|
|
297
|
+
await this._waitForPeers(core, signal)
|
|
298
|
+
await core.update()
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
async downloadModel(path, source, options = {}) {
|
|
213
302
|
this._validateString(path, 'path')
|
|
214
303
|
this._validateString(source, 'source')
|
|
215
304
|
|
|
@@ -217,7 +306,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
217
306
|
throw new Error(`Invalid options: ${typeof options}`)
|
|
218
307
|
}
|
|
219
308
|
|
|
220
|
-
let core, blobs
|
|
309
|
+
let core, blobs, blockStart, blockEnd, rangeDownload
|
|
221
310
|
|
|
222
311
|
try {
|
|
223
312
|
this.logger.info('Downloading model', { path, source })
|
|
@@ -225,11 +314,17 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
225
314
|
|
|
226
315
|
const model = await this.getModel(path, source)
|
|
227
316
|
if (!model) {
|
|
228
|
-
throw new QvacErrorRegistryClient({
|
|
317
|
+
throw new QvacErrorRegistryClient({
|
|
318
|
+
code: ERR_CODES.MODEL_NOT_FOUND,
|
|
319
|
+
adds: `Model not found: ${path} (source: ${source})`
|
|
320
|
+
})
|
|
229
321
|
}
|
|
230
322
|
|
|
231
323
|
if (!model.blobBinding || !model.blobBinding.coreKey) {
|
|
232
|
-
throw new QvacErrorRegistryClient({
|
|
324
|
+
throw new QvacErrorRegistryClient({
|
|
325
|
+
code: ERR_CODES.MODEL_NOT_FOUND,
|
|
326
|
+
adds: 'Model missing blob binding'
|
|
327
|
+
})
|
|
233
328
|
}
|
|
234
329
|
|
|
235
330
|
this.logger.debug('Model metadata retrieved', { model })
|
|
@@ -251,31 +346,32 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
251
346
|
|
|
252
347
|
const totalSize = model.blobBinding.byteLength
|
|
253
348
|
|
|
254
|
-
|
|
349
|
+
rangeDownload = core.download({
|
|
255
350
|
start: model.blobBinding.blockOffset,
|
|
256
351
|
length: model.blobBinding.blockLength
|
|
257
352
|
})
|
|
258
353
|
|
|
259
|
-
|
|
260
|
-
|
|
354
|
+
blockStart = model.blobBinding.blockOffset
|
|
355
|
+
blockEnd = blockStart + model.blobBinding.blockLength
|
|
261
356
|
|
|
262
357
|
let artifact
|
|
263
358
|
if (options.outputFile) {
|
|
264
359
|
await withRetry(
|
|
265
360
|
() => this._streamBlobToFile(blobs, core, model.blobBinding, options.outputFile, options),
|
|
266
361
|
{
|
|
267
|
-
maxRetries: options.maxRetries
|
|
362
|
+
maxRetries: options.maxRetries ?? DEFAULT_DOWNLOAD_MAX_RETRIES,
|
|
268
363
|
retryCodes: RETRIABLE_DOWNLOAD_CODES,
|
|
269
|
-
|
|
364
|
+
// Wait for the swarm to resume + reconnect peers before retrying,
|
|
365
|
+
// so the retry doesn't immediately time out again against a dead
|
|
366
|
+
// swarm (e.g. after the app backgrounded). The core's cached blocks
|
|
367
|
+
// are not cleared until success, so the retry re-streams cheaply.
|
|
368
|
+
beforeRetry: () => this._reconnectCore(core, options.signal),
|
|
270
369
|
logger: this.logger
|
|
271
370
|
}
|
|
272
371
|
)
|
|
273
372
|
artifact = { path: options.outputFile, totalSize }
|
|
274
373
|
|
|
275
|
-
|
|
276
|
-
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
277
|
-
if (blobs) await blobs.close()
|
|
278
|
-
if (core) await core.close()
|
|
374
|
+
await this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd)
|
|
279
375
|
} else {
|
|
280
376
|
const stream = blobs.createReadStream(model.blobBinding, {
|
|
281
377
|
wait: true,
|
|
@@ -283,27 +379,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
283
379
|
})
|
|
284
380
|
artifact = { stream, totalSize }
|
|
285
381
|
|
|
286
|
-
|
|
287
|
-
rangeDownload.destroy()
|
|
288
|
-
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
289
|
-
if (blobs) {
|
|
290
|
-
try {
|
|
291
|
-
await blobs.close()
|
|
292
|
-
} catch (cleanupError) {
|
|
293
|
-
this.logger.warn('Error closing blob instance', { error: cleanupError.message })
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
if (core) {
|
|
297
|
-
try {
|
|
298
|
-
await core.close()
|
|
299
|
-
} catch (cleanupError) {
|
|
300
|
-
this.logger.warn('Error closing blob core', { error: cleanupError.message })
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
this.logger.debug('Blob resources closed after stream end')
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
stream.once('end', cleanup)
|
|
382
|
+
this._releaseOnStreamEnd(stream, core, blobs, rangeDownload, blockStart, blockEnd)
|
|
307
383
|
}
|
|
308
384
|
|
|
309
385
|
this.logger.info('Model downloaded successfully')
|
|
@@ -315,44 +391,38 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
315
391
|
} catch (error) {
|
|
316
392
|
this.logger.error('Error downloading model', error)
|
|
317
393
|
|
|
318
|
-
|
|
319
|
-
try {
|
|
320
|
-
await blobs.close()
|
|
321
|
-
} catch (cleanupError) {
|
|
322
|
-
this.logger.warn('Error closing blob instance on error', { error: cleanupError.message })
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
if (core) {
|
|
326
|
-
try {
|
|
327
|
-
await core.close()
|
|
328
|
-
} catch (cleanupError) {
|
|
329
|
-
this.logger.warn('Error closing blob core on error', { error: cleanupError.message })
|
|
330
|
-
}
|
|
331
|
-
}
|
|
394
|
+
await this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd)
|
|
332
395
|
|
|
333
396
|
throw error
|
|
334
397
|
}
|
|
335
398
|
}
|
|
336
399
|
|
|
337
|
-
async downloadBlob
|
|
400
|
+
async downloadBlob(blobBinding, options = {}) {
|
|
338
401
|
if (!blobBinding || !blobBinding.coreKey) {
|
|
339
402
|
throw new Error('Invalid blobBinding: coreKey is required')
|
|
340
403
|
}
|
|
341
|
-
if (
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
404
|
+
if (
|
|
405
|
+
typeof blobBinding.blockOffset !== 'number' ||
|
|
406
|
+
typeof blobBinding.blockLength !== 'number' ||
|
|
407
|
+
typeof blobBinding.byteLength !== 'number'
|
|
408
|
+
) {
|
|
409
|
+
throw new Error(
|
|
410
|
+
'Invalid blobBinding: blockOffset, blockLength, and byteLength are required numbers'
|
|
411
|
+
)
|
|
345
412
|
}
|
|
346
413
|
|
|
347
414
|
if (options && typeof options !== 'object') {
|
|
348
415
|
throw new Error(`Invalid options: ${typeof options}`)
|
|
349
416
|
}
|
|
350
417
|
|
|
351
|
-
let core, blobs
|
|
418
|
+
let core, blobs, blockStart, blockEnd, rangeDownload
|
|
352
419
|
|
|
353
420
|
try {
|
|
354
421
|
this.logger.info('Downloading blob directly', {
|
|
355
|
-
coreKey:
|
|
422
|
+
coreKey:
|
|
423
|
+
typeof blobBinding.coreKey === 'string'
|
|
424
|
+
? blobBinding.coreKey.slice(0, 12) + '...'
|
|
425
|
+
: '(buffer)',
|
|
356
426
|
blockOffset: blobBinding.blockOffset,
|
|
357
427
|
blockLength: blobBinding.blockLength,
|
|
358
428
|
byteLength: blobBinding.byteLength
|
|
@@ -383,10 +453,10 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
383
453
|
}
|
|
384
454
|
const totalSize = blobBinding.byteLength
|
|
385
455
|
|
|
386
|
-
|
|
387
|
-
|
|
456
|
+
blockStart = pointer.blockOffset
|
|
457
|
+
blockEnd = blockStart + pointer.blockLength
|
|
388
458
|
|
|
389
|
-
|
|
459
|
+
rangeDownload = core.download({
|
|
390
460
|
start: pointer.blockOffset,
|
|
391
461
|
length: pointer.blockLength
|
|
392
462
|
})
|
|
@@ -396,18 +466,17 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
396
466
|
await withRetry(
|
|
397
467
|
() => this._streamBlobToFile(blobs, core, pointer, options.outputFile, options),
|
|
398
468
|
{
|
|
399
|
-
maxRetries: options.maxRetries
|
|
469
|
+
maxRetries: options.maxRetries ?? DEFAULT_DOWNLOAD_MAX_RETRIES,
|
|
400
470
|
retryCodes: RETRIABLE_DOWNLOAD_CODES,
|
|
401
|
-
|
|
471
|
+
// Wait for swarm resume + peer reconnect before retrying (see
|
|
472
|
+
// downloadModel). Cached blocks are reused; the file is re-streamed.
|
|
473
|
+
beforeRetry: () => this._reconnectCore(core, options.signal),
|
|
402
474
|
logger: this.logger
|
|
403
475
|
}
|
|
404
476
|
)
|
|
405
477
|
artifact = { path: options.outputFile, totalSize }
|
|
406
478
|
|
|
407
|
-
|
|
408
|
-
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
409
|
-
if (blobs) await blobs.close()
|
|
410
|
-
if (core) await core.close()
|
|
479
|
+
await this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd)
|
|
411
480
|
} else {
|
|
412
481
|
const stream = blobs.createReadStream(pointer, {
|
|
413
482
|
wait: true,
|
|
@@ -415,23 +484,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
415
484
|
})
|
|
416
485
|
artifact = { stream, totalSize }
|
|
417
486
|
|
|
418
|
-
|
|
419
|
-
rangeDownload.destroy()
|
|
420
|
-
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
421
|
-
if (blobs) {
|
|
422
|
-
try { await blobs.close() } catch (e) {
|
|
423
|
-
this.logger.warn('Error closing blob instance', { error: e.message })
|
|
424
|
-
}
|
|
425
|
-
}
|
|
426
|
-
if (core) {
|
|
427
|
-
try { await core.close() } catch (e) {
|
|
428
|
-
this.logger.warn('Error closing blob core', { error: e.message })
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
this.logger.debug('Blob resources closed after stream end')
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
stream.once('end', cleanup)
|
|
487
|
+
this._releaseOnStreamEnd(stream, core, blobs, rangeDownload, blockStart, blockEnd)
|
|
435
488
|
}
|
|
436
489
|
|
|
437
490
|
this.logger.info('Blob download complete (direct)')
|
|
@@ -440,33 +493,96 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
440
493
|
} catch (error) {
|
|
441
494
|
this.logger.error('Error downloading blob directly', error)
|
|
442
495
|
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
496
|
+
await this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd)
|
|
497
|
+
|
|
498
|
+
throw error
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
_releaseOnStreamEnd(stream, core, blobs, rangeDownload, blockStart, blockEnd) {
|
|
503
|
+
let released = false
|
|
504
|
+
|
|
505
|
+
// 'close' also covers a destroyed or errored stream; on 'end' alone a
|
|
506
|
+
// cancelled stream download would never free its blocks.
|
|
507
|
+
const release = () => {
|
|
508
|
+
if (released) return
|
|
509
|
+
released = true
|
|
510
|
+
return this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd).catch((e) =>
|
|
511
|
+
this.logger.warn('Error releasing blob resources', { error: e.message })
|
|
512
|
+
)
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
stream.once('end', release)
|
|
516
|
+
stream.once('close', release)
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
async _releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd) {
|
|
520
|
+
// Hypercore can commit in-flight responses after a range is destroyed.
|
|
521
|
+
if (rangeDownload) rangeDownload.destroy()
|
|
522
|
+
|
|
523
|
+
if (core && blockStart !== undefined) {
|
|
524
|
+
if (rangeDownload) await this._waitForInflightBlocks(core)
|
|
525
|
+
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
526
|
+
}
|
|
527
|
+
if (blobs) {
|
|
528
|
+
try {
|
|
529
|
+
await blobs.close()
|
|
530
|
+
} catch (e) {
|
|
531
|
+
this.logger.warn('Error closing blob instance', { error: e.message })
|
|
447
532
|
}
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
533
|
+
}
|
|
534
|
+
if (core) {
|
|
535
|
+
try {
|
|
536
|
+
await core.close()
|
|
537
|
+
} catch (e) {
|
|
538
|
+
this.logger.warn('Error closing blob core', { error: e.message })
|
|
452
539
|
}
|
|
540
|
+
}
|
|
453
541
|
|
|
454
|
-
|
|
542
|
+
this.logger.debug('Blob resources released')
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
async _waitForInflightBlocks(core) {
|
|
546
|
+
if (!Array.isArray(core.peers)) return
|
|
547
|
+
|
|
548
|
+
const timeoutMs = this._inflightDrainTimeoutMs ?? INFLIGHT_DRAIN_TIMEOUT_MS
|
|
549
|
+
const pollMs = this._inflightDrainPollMs ?? INFLIGHT_DRAIN_POLL_MS
|
|
550
|
+
const deadline = Date.now() + timeoutMs
|
|
551
|
+
while (core.peers.some((peer) => peer.inflight > 0 || peer.dataProcessing > 0)) {
|
|
552
|
+
if (Date.now() >= deadline) {
|
|
553
|
+
this.logger.warn('Timed out waiting for in-flight blob blocks before clearing', {
|
|
554
|
+
timeoutMs,
|
|
555
|
+
peers: core.peers.map((peer, index) => ({
|
|
556
|
+
peer: index,
|
|
557
|
+
inflight: peer.inflight,
|
|
558
|
+
dataProcessing: peer.dataProcessing
|
|
559
|
+
}))
|
|
560
|
+
})
|
|
561
|
+
return
|
|
562
|
+
}
|
|
563
|
+
await new Promise((resolve) => setTimeout(resolve, pollMs))
|
|
455
564
|
}
|
|
456
565
|
}
|
|
457
566
|
|
|
458
|
-
async _clearBlobBlocks
|
|
567
|
+
async _clearBlobBlocks(core, start, end) {
|
|
459
568
|
try {
|
|
460
569
|
const cleared = await core.clear(start, end, { diff: true })
|
|
461
570
|
await core.compact()
|
|
462
|
-
this.logger.info('Cleared blob blocks from corestore', {
|
|
571
|
+
this.logger.info('Cleared blob blocks from corestore', {
|
|
572
|
+
start,
|
|
573
|
+
end,
|
|
574
|
+
blocks: cleared ? cleared.blocks : end - start
|
|
575
|
+
})
|
|
463
576
|
} catch (err) {
|
|
464
577
|
this.logger.warn('Failed to clear blob blocks from corestore', { error: err.message })
|
|
465
578
|
}
|
|
466
579
|
}
|
|
467
580
|
|
|
468
|
-
async _streamBlobToFile
|
|
469
|
-
const { cachedBlocks, totalBlocks, totalBytes } = await this._checkBlobProgress(
|
|
581
|
+
async _streamBlobToFile(blobs, core, blobPointer, filePath, options) {
|
|
582
|
+
const { cachedBlocks, totalBlocks, totalBytes } = await this._checkBlobProgress(
|
|
583
|
+
core,
|
|
584
|
+
blobPointer
|
|
585
|
+
)
|
|
470
586
|
|
|
471
587
|
this.logger.debug('Blob progress before download', {
|
|
472
588
|
cachedBlocks,
|
|
@@ -488,7 +604,10 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
488
604
|
}
|
|
489
605
|
|
|
490
606
|
const progressHandler = (index, bytes) => {
|
|
491
|
-
if (
|
|
607
|
+
if (
|
|
608
|
+
index >= blobPointer.blockOffset &&
|
|
609
|
+
index < blobPointer.blockOffset + blobPointer.blockLength
|
|
610
|
+
) {
|
|
492
611
|
downloadedBytes += bytes
|
|
493
612
|
const capped = Math.min(downloadedBytes, totalBytes)
|
|
494
613
|
if (options.onProgress) {
|
|
@@ -533,11 +652,15 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
533
652
|
reject(new Error('Download cancelled'))
|
|
534
653
|
return
|
|
535
654
|
}
|
|
536
|
-
options.signal.addEventListener(
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
655
|
+
options.signal.addEventListener(
|
|
656
|
+
'abort',
|
|
657
|
+
() => {
|
|
658
|
+
stream.destroy()
|
|
659
|
+
writeStream.destroy()
|
|
660
|
+
reject(new Error('Download cancelled'))
|
|
661
|
+
},
|
|
662
|
+
{ once: true }
|
|
663
|
+
)
|
|
541
664
|
}
|
|
542
665
|
})
|
|
543
666
|
} finally {
|
|
@@ -546,7 +669,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
546
669
|
}
|
|
547
670
|
}
|
|
548
671
|
|
|
549
|
-
async suspend
|
|
672
|
+
async suspend(opts = {}) {
|
|
550
673
|
this.logger.debug('suspend called')
|
|
551
674
|
|
|
552
675
|
if (!this.opened || this.closing) {
|
|
@@ -564,7 +687,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
564
687
|
this.logger.debug('QVACRegistryClient suspended')
|
|
565
688
|
}
|
|
566
689
|
|
|
567
|
-
async resume
|
|
690
|
+
async resume(opts = {}) {
|
|
568
691
|
this.logger.debug('resume called')
|
|
569
692
|
|
|
570
693
|
if (!this.opened || this.closing) {
|
|
@@ -582,7 +705,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
582
705
|
this.logger.debug('QVACRegistryClient resumed')
|
|
583
706
|
}
|
|
584
707
|
|
|
585
|
-
async _close
|
|
708
|
+
async _close() {
|
|
586
709
|
this.logger.debug('_close called')
|
|
587
710
|
|
|
588
711
|
if (this._metadataReady) {
|
package/lib/config.js
CHANGED
|
@@ -9,12 +9,12 @@ const path = require('#path')
|
|
|
9
9
|
const DEFAULT_REGISTRY_CORE_KEY = 'uf1fm44uzockp6azhcdiqt1esjgm65fwtimsh946e8kwysdes9ko'
|
|
10
10
|
|
|
11
11
|
class RegistryConfig {
|
|
12
|
-
constructor
|
|
12
|
+
constructor(opts = {}) {
|
|
13
13
|
this.logger = new Logger(opts.logger)
|
|
14
14
|
this.logger.debug('RegistryConfig initialized', { opts })
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
getRegistryStorage
|
|
17
|
+
getRegistryStorage(providedPath) {
|
|
18
18
|
if (providedPath) {
|
|
19
19
|
this.logger.debug('getRegistryStorage called with providedPath', { providedPath })
|
|
20
20
|
return providedPath
|
|
@@ -26,14 +26,17 @@ class RegistryConfig {
|
|
|
26
26
|
return result
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
_createTempStoragePath
|
|
29
|
+
_createTempStoragePath() {
|
|
30
30
|
const tmpBase = os.tmpdir()
|
|
31
|
-
const storageDir = path.join(
|
|
31
|
+
const storageDir = path.join(
|
|
32
|
+
tmpBase,
|
|
33
|
+
`qvac-registry-client-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
|
|
34
|
+
)
|
|
32
35
|
this.logger.debug('Created temp storage path', { storageDir })
|
|
33
36
|
return storageDir
|
|
34
37
|
}
|
|
35
38
|
|
|
36
|
-
getRegistryCoreKey
|
|
39
|
+
getRegistryCoreKey(providedKey) {
|
|
37
40
|
if (providedKey) {
|
|
38
41
|
this.logger.debug('getRegistryCoreKey called with providedKey', { providedKey })
|
|
39
42
|
return providedKey
|
package/lib/logger.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
class Logger {
|
|
4
|
-
constructor
|
|
4
|
+
constructor(opts = {}) {
|
|
5
5
|
this.level = opts.level || 'info'
|
|
6
6
|
this.name = opts.name || 'QVACRegistryClient'
|
|
7
7
|
this.enabled = opts.enabled !== false
|
|
@@ -14,30 +14,30 @@ class Logger {
|
|
|
14
14
|
}
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
_shouldLog
|
|
17
|
+
_shouldLog(level) {
|
|
18
18
|
if (!this.enabled) return false
|
|
19
19
|
return this.levels[level] >= this.levels[this.level]
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
_log
|
|
22
|
+
_log(level, ...args) {
|
|
23
23
|
if (!this._shouldLog(level)) return
|
|
24
24
|
const prefix = `[${this.name}] [${level.toUpperCase()}]`
|
|
25
25
|
console[level](prefix, ...args)
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
debug
|
|
28
|
+
debug(...args) {
|
|
29
29
|
this._log('debug', ...args)
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
info
|
|
32
|
+
info(...args) {
|
|
33
33
|
this._log('info', ...args)
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
warn
|
|
36
|
+
warn(...args) {
|
|
37
37
|
this._log('warn', ...args)
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
error
|
|
40
|
+
error(...args) {
|
|
41
41
|
this._log('error', ...args)
|
|
42
42
|
}
|
|
43
43
|
}
|
package/lib/profiler.js
CHANGED
|
@@ -13,7 +13,7 @@ const { RegistryDatabase } = require('@qvac/registry-schema')
|
|
|
13
13
|
* Normalize a blob core key into a Buffer regardless of input format.
|
|
14
14
|
* Handles raw Buffer, { data: [...] } objects from HyperDB, and z32/hex strings.
|
|
15
15
|
*/
|
|
16
|
-
function decodeCoreKey
|
|
16
|
+
function decodeCoreKey(key) {
|
|
17
17
|
if (Buffer.isBuffer(key)) return key
|
|
18
18
|
if (typeof key === 'object' && key !== null && key.data) {
|
|
19
19
|
return Buffer.from(key.data)
|
|
@@ -25,7 +25,7 @@ function decodeCoreKey (key) {
|
|
|
25
25
|
* Collect a snapshot of network, connection, and hypercore stats.
|
|
26
26
|
* All fields are plain values suitable for logging or formatting.
|
|
27
27
|
*/
|
|
28
|
-
function collectStats
|
|
28
|
+
function collectStats(swarmStats, hypercoreStats, blobCore, elapsedSec) {
|
|
29
29
|
const bytesRx = swarmStats.dhtStats.udxBytesReceived
|
|
30
30
|
const bytesTx = swarmStats.dhtStats.udxBytesTransmitted
|
|
31
31
|
|
|
@@ -71,7 +71,7 @@ function collectStats (swarmStats, hypercoreStats, blobCore, elapsedSec) {
|
|
|
71
71
|
/**
|
|
72
72
|
* Format a stats snapshot into a human-readable string.
|
|
73
73
|
*/
|
|
74
|
-
function formatStats
|
|
74
|
+
function formatStats(stats) {
|
|
75
75
|
const n = stats.network
|
|
76
76
|
const c = stats.connection
|
|
77
77
|
const h = stats.hypercore
|
|
@@ -88,7 +88,14 @@ function formatStats (stats) {
|
|
|
88
88
|
lines += ' Attempted: ' + c.attempted + '\n'
|
|
89
89
|
lines += ' Opened: ' + c.opened + '\n'
|
|
90
90
|
lines += ' Closed: ' + c.closed + '\n'
|
|
91
|
-
lines +=
|
|
91
|
+
lines +=
|
|
92
|
+
' Issues: rto=' +
|
|
93
|
+
c.rtos +
|
|
94
|
+
' fast-recoveries=' +
|
|
95
|
+
c.fastRecoveries +
|
|
96
|
+
' retransmits=' +
|
|
97
|
+
c.retransmits +
|
|
98
|
+
'\n'
|
|
92
99
|
lines += 'Hypercore\n'
|
|
93
100
|
lines += ' Blob core: ' + h.contiguousLength + ' / ' + h.length + ' (contiguous / length)\n'
|
|
94
101
|
lines += ' Hotswaps: ' + h.hotswaps + '\n'
|
|
@@ -106,7 +113,7 @@ function formatStats (stats) {
|
|
|
106
113
|
/**
|
|
107
114
|
* Format a final download summary into a human-readable string.
|
|
108
115
|
*/
|
|
109
|
-
function formatSummary
|
|
116
|
+
function formatSummary(opts) {
|
|
110
117
|
let lines = '='.repeat(50) + '\n'
|
|
111
118
|
lines += 'FINAL SUMMARY\n'
|
|
112
119
|
lines += '='.repeat(50) + '\n'
|
|
@@ -133,7 +140,7 @@ function formatSummary (opts) {
|
|
|
133
140
|
* @param {function} [opts.onLog] - Called with (message) for log output; defaults to console.log
|
|
134
141
|
* @returns {Promise<object>} Summary with timing and stats
|
|
135
142
|
*/
|
|
136
|
-
async function profileDownload
|
|
143
|
+
async function profileDownload(opts) {
|
|
137
144
|
// Lazy-loaded: these Node builtins don't exist in Bare runtime,
|
|
138
145
|
// so they must not be required at the top level or unit tests break.
|
|
139
146
|
const os = require('#os')
|
|
@@ -177,7 +184,9 @@ async function profileDownload (opts) {
|
|
|
177
184
|
const cleanupResources = async () => {
|
|
178
185
|
await swarm.destroy()
|
|
179
186
|
await store.close()
|
|
180
|
-
try {
|
|
187
|
+
try {
|
|
188
|
+
fs.rmSync(tmpdir, { recursive: true, force: true })
|
|
189
|
+
} catch {}
|
|
181
190
|
}
|
|
182
191
|
|
|
183
192
|
try {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/registry-client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "QVAC Registry client library for read-only queries via Hyperswarm",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -47,13 +47,15 @@
|
|
|
47
47
|
"test:unit": "brittle tests/unit/*.test.js",
|
|
48
48
|
"test:unit:bare": "brittle-bare tests/unit/*.test.js",
|
|
49
49
|
"test": "npm run test:unit && npm run test:integration",
|
|
50
|
-
"
|
|
51
|
-
"
|
|
50
|
+
"format": "prettier --check .",
|
|
51
|
+
"format:fix": "prettier --write .",
|
|
52
|
+
"lint": "lunte",
|
|
53
|
+
"lint:fix": "lunte --fix",
|
|
52
54
|
"typecheck": "npx tsc --noEmit"
|
|
53
55
|
},
|
|
54
56
|
"dependencies": {
|
|
55
57
|
"@qvac/error": "^0.1.0",
|
|
56
|
-
"@qvac/registry-schema": "^0.
|
|
58
|
+
"@qvac/registry-schema": "^0.4.0",
|
|
57
59
|
"b4a": "^1.6.7",
|
|
58
60
|
"bare-fs": "^4.5.2",
|
|
59
61
|
"bare-os": "^3.6.2",
|
|
@@ -72,7 +74,9 @@
|
|
|
72
74
|
"devDependencies": {
|
|
73
75
|
"brittle": "^3.4.0",
|
|
74
76
|
"dotenv": "^17.2.3",
|
|
75
|
-
"
|
|
77
|
+
"lunte": "^1.8.4",
|
|
78
|
+
"prettier": "^3.9.6",
|
|
79
|
+
"prettier-config-holepunch": "^2.0.0",
|
|
76
80
|
"test-tmp": "^1.4.0",
|
|
77
81
|
"typescript": "^5.9.3"
|
|
78
82
|
}
|
package/utils/env.js
CHANGED
package/utils/error.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
const { QvacErrorBase, addCodes } = require('@qvac/error')
|
|
4
4
|
const { name, version } = require('../package.json')
|
|
5
5
|
|
|
6
|
-
class QvacErrorRegistryClient extends QvacErrorBase {
|
|
6
|
+
class QvacErrorRegistryClient extends QvacErrorBase {}
|
|
7
7
|
|
|
8
8
|
// This library has error code range from 19,001 to 20,000
|
|
9
9
|
const ERR_CODES = Object.freeze({
|
|
@@ -12,23 +12,26 @@ const ERR_CODES = Object.freeze({
|
|
|
12
12
|
MODEL_NOT_FOUND: 19003
|
|
13
13
|
})
|
|
14
14
|
|
|
15
|
-
addCodes(
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
15
|
+
addCodes(
|
|
16
|
+
{
|
|
17
|
+
[ERR_CODES.FAILED_TO_CONNECT]: {
|
|
18
|
+
name: 'FAILED_TO_CONNECT',
|
|
19
|
+
message: (message) => `Failed to connect to registry, error: ${message}`
|
|
20
|
+
},
|
|
21
|
+
[ERR_CODES.FAILED_TO_CLOSE]: {
|
|
22
|
+
name: 'FAILED_TO_CLOSE',
|
|
23
|
+
message: (message) => `Failed to close registry, error: ${message}`
|
|
24
|
+
},
|
|
25
|
+
[ERR_CODES.MODEL_NOT_FOUND]: {
|
|
26
|
+
name: 'MODEL_NOT_FOUND',
|
|
27
|
+
message: (message) => `Model not found, error: ${message}`
|
|
28
|
+
}
|
|
19
29
|
},
|
|
20
|
-
|
|
21
|
-
name
|
|
22
|
-
|
|
23
|
-
},
|
|
24
|
-
[ERR_CODES.MODEL_NOT_FOUND]: {
|
|
25
|
-
name: 'MODEL_NOT_FOUND',
|
|
26
|
-
message: (message) => `Model not found, error: ${message}`
|
|
30
|
+
{
|
|
31
|
+
name,
|
|
32
|
+
version
|
|
27
33
|
}
|
|
28
|
-
|
|
29
|
-
name,
|
|
30
|
-
version
|
|
31
|
-
})
|
|
34
|
+
)
|
|
32
35
|
|
|
33
36
|
module.exports = {
|
|
34
37
|
ERR_CODES,
|
package/utils/retry.js
CHANGED
|
@@ -8,11 +8,14 @@
|
|
|
8
8
|
* @param {number} [opts.maxRetries=3] - Maximum number of attempts (including the first)
|
|
9
9
|
* @param {string[]} [opts.retryCodes=[]] - Error codes that trigger a retry
|
|
10
10
|
* @param {() => Promise<void>} [opts.onRetry] - Called before each retry attempt (e.g. cleanup)
|
|
11
|
+
* @param {() => Promise<void>} [opts.beforeRetry] - Awaited before the next attempt runs.
|
|
12
|
+
* Use to re-establish prerequisites (e.g. wait for peers to reconnect) so the
|
|
13
|
+
* retry doesn't immediately fail again against a dead connection.
|
|
11
14
|
* @param {{ warn: Function }} [opts.logger] - Logger instance for retry warnings
|
|
12
15
|
* @returns {Promise<unknown>}
|
|
13
16
|
*/
|
|
14
|
-
async function withRetry
|
|
15
|
-
const { maxRetries = 3, retryCodes = [], onRetry, logger } = opts
|
|
17
|
+
async function withRetry(fn, opts = {}) {
|
|
18
|
+
const { maxRetries = 3, retryCodes = [], onRetry, beforeRetry, logger } = opts
|
|
16
19
|
|
|
17
20
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
18
21
|
try {
|
|
@@ -20,8 +23,10 @@ async function withRetry (fn, opts = {}) {
|
|
|
20
23
|
} catch (err) {
|
|
21
24
|
const isRetriable = retryCodes.length > 0 && retryCodes.includes(err && err.code)
|
|
22
25
|
if (!isRetriable || attempt >= maxRetries) throw err
|
|
23
|
-
logger &&
|
|
26
|
+
logger &&
|
|
27
|
+
logger.warn(`Retrying after ${err.code} (attempt ${attempt}/${maxRetries}): ${err.message}`)
|
|
24
28
|
if (onRetry) await onRetry()
|
|
29
|
+
if (beforeRetry) await beforeRetry()
|
|
25
30
|
}
|
|
26
31
|
}
|
|
27
32
|
}
|