@qvac/registry-client 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -17
- package/bin/cli.js +33 -16
- package/index.d.ts +62 -21
- package/lib/client.js +158 -122
- package/lib/config.js +8 -5
- package/lib/logger.js +7 -7
- package/lib/profiler.js +16 -7
- package/package.json +9 -5
- package/utils/env.js +1 -1
- package/utils/error.js +19 -16
- package/utils/retry.js +3 -2
package/README.md
CHANGED
|
@@ -26,7 +26,7 @@ Ensure the registry core key is available via environment variables or provided
|
|
|
26
26
|
'use strict'
|
|
27
27
|
const { QVACRegistryClient } = require('@qvac/registry-client')
|
|
28
28
|
|
|
29
|
-
async function main
|
|
29
|
+
async function main() {
|
|
30
30
|
const client = new QVACRegistryClient({
|
|
31
31
|
registryCoreKey: process.env.QVAC_REGISTRY_CORE_KEY
|
|
32
32
|
})
|
|
@@ -120,16 +120,19 @@ result.artifact.stream.pipe(fs.createWriteStream('./model.ggml'))
|
|
|
120
120
|
This method only waits for the network layer (Corestore + Hyperswarm) — it does not wait for the metadata core to sync, making it faster for known blob coordinates.
|
|
121
121
|
|
|
122
122
|
```javascript
|
|
123
|
-
const result = await client.downloadBlob(
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
123
|
+
const result = await client.downloadBlob(
|
|
124
|
+
{
|
|
125
|
+
coreKey: 'ey46cahego89xox118uhyryakz47bcs8bbxu97tnnpmuwmgi5wmo',
|
|
126
|
+
blockOffset: 0,
|
|
127
|
+
blockLength: 665,
|
|
128
|
+
byteOffset: 0,
|
|
129
|
+
byteLength: 43537433
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
outputFile: './downloaded/ggml-tiny-q8_0.bin',
|
|
133
|
+
timeout: 60000
|
|
134
|
+
}
|
|
135
|
+
)
|
|
133
136
|
console.log('Downloaded to:', result.artifact.path)
|
|
134
137
|
```
|
|
135
138
|
|
|
@@ -332,11 +335,11 @@ The client uses custom error codes in the range 19001-20000. All errors extend `
|
|
|
332
335
|
|
|
333
336
|
### Error Codes
|
|
334
337
|
|
|
335
|
-
| Code
|
|
336
|
-
|
|
337
|
-
| 19001 | FAILED_TO_CONNECT | Connection to registry failed
|
|
338
|
-
| 19002 | FAILED_TO_CLOSE
|
|
339
|
-
| 19003 | MODEL_NOT_FOUND
|
|
338
|
+
| Code | Name | Description | When Thrown |
|
|
339
|
+
| ----- | ----------------- | -------------------------------- | ------------------------------------------------------ |
|
|
340
|
+
| 19001 | FAILED_TO_CONNECT | Connection to registry failed | Missing core key, network issues during initialization |
|
|
341
|
+
| 19002 | FAILED_TO_CLOSE | Failed to close registry cleanly | Resource cleanup errors during shutdown |
|
|
342
|
+
| 19003 | MODEL_NOT_FOUND | Model not found or invalid | Model doesn't exist or missing blob binding |
|
|
340
343
|
|
|
341
344
|
### Error Handling Example
|
|
342
345
|
|
|
@@ -344,7 +347,7 @@ The client uses custom error codes in the range 19001-20000. All errors extend `
|
|
|
344
347
|
const { QVACRegistryClient } = require('@qvac/registry-client')
|
|
345
348
|
const { QvacErrorRegistryClient } = require('@qvac/registry-client/utils/error')
|
|
346
349
|
|
|
347
|
-
async function handleErrors
|
|
350
|
+
async function handleErrors() {
|
|
348
351
|
const client = new QVACRegistryClient({
|
|
349
352
|
registryCoreKey: process.env.QVAC_REGISTRY_CORE_KEY
|
|
350
353
|
})
|
package/bin/cli.js
CHANGED
|
@@ -11,7 +11,7 @@ const VERSION = require('../package.json').version
|
|
|
11
11
|
|
|
12
12
|
// --- Helpers ---
|
|
13
13
|
|
|
14
|
-
function createClient
|
|
14
|
+
function createClient(parentFlags) {
|
|
15
15
|
const opts = {}
|
|
16
16
|
if (parentFlags.key) opts.registryCoreKey = parentFlags.key
|
|
17
17
|
if (parentFlags.storage) opts.storage = parentFlags.storage
|
|
@@ -23,13 +23,13 @@ function createClient (parentFlags) {
|
|
|
23
23
|
return new QVACRegistryClient(opts)
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
function getRootFlags
|
|
26
|
+
function getRootFlags(cmd) {
|
|
27
27
|
let current = cmd.command || cmd
|
|
28
28
|
while (current.parent) current = current.parent
|
|
29
29
|
return current.flags || {}
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
function formatSize
|
|
32
|
+
function formatSize(bytes) {
|
|
33
33
|
if (!bytes) return 'N/A'
|
|
34
34
|
if (bytes >= 1024 * 1024 * 1024) return (bytes / (1024 * 1024 * 1024)).toFixed(2) + ' GB'
|
|
35
35
|
if (bytes >= 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(2) + ' MB'
|
|
@@ -37,14 +37,14 @@ function formatSize (bytes) {
|
|
|
37
37
|
return bytes + ' B'
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
function printModelCompact
|
|
40
|
+
function printModelCompact(model) {
|
|
41
41
|
const parts = [model.path, model.source]
|
|
42
42
|
if (model.quantization) parts.push(model.quantization)
|
|
43
43
|
if (model.params) parts.push(model.params)
|
|
44
44
|
console.log(parts.join('\t'))
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
function printModelFull
|
|
47
|
+
function printModelFull(model, opts = {}) {
|
|
48
48
|
const blob = model.blobBinding || {}
|
|
49
49
|
const coreKey = blob.coreKey ? IdEnc.normalize(blob.coreKey) : 'N/A'
|
|
50
50
|
|
|
@@ -55,7 +55,9 @@ function printModelFull (model, opts = {}) {
|
|
|
55
55
|
if (model.quantization) console.log(` quantization: ${model.quantization}`)
|
|
56
56
|
if (model.params) console.log(` params: ${model.params}`)
|
|
57
57
|
console.log(` size: ${formatSize(blob.byteLength || model.sizeBytes)}`)
|
|
58
|
-
if (model.licenseId || model.license)
|
|
58
|
+
if (model.licenseId || model.license) {
|
|
59
|
+
console.log(` license: ${model.licenseId || model.license}`)
|
|
60
|
+
}
|
|
59
61
|
if (blob.sha256 || model.sha256) console.log(` sha256: ${blob.sha256 || model.sha256}`)
|
|
60
62
|
if (opts.verbose && model.description) console.log(` description: ${model.description}`)
|
|
61
63
|
if (opts.verbose) console.log(` blob core: ${coreKey}`)
|
|
@@ -63,7 +65,7 @@ function printModelFull (model, opts = {}) {
|
|
|
63
65
|
console.log()
|
|
64
66
|
}
|
|
65
67
|
|
|
66
|
-
async function withClient
|
|
68
|
+
async function withClient(cmd, fn) {
|
|
67
69
|
const rootFlags = getRootFlags(cmd)
|
|
68
70
|
const client = createClient(rootFlags)
|
|
69
71
|
try {
|
|
@@ -75,7 +77,8 @@ async function withClient (cmd, fn) {
|
|
|
75
77
|
|
|
76
78
|
// --- Commands ---
|
|
77
79
|
|
|
78
|
-
const listCmd = command(
|
|
80
|
+
const listCmd = command(
|
|
81
|
+
'list',
|
|
79
82
|
summary('List models from the registry'),
|
|
80
83
|
description('List all models or filter by name, engine, or quantization.'),
|
|
81
84
|
flag('--name|-n [name]', 'Filter by model name (partial match)'),
|
|
@@ -103,10 +106,13 @@ const listCmd = command('list',
|
|
|
103
106
|
}
|
|
104
107
|
|
|
105
108
|
if (flags.json) {
|
|
106
|
-
const out = models.map(m => ({
|
|
109
|
+
const out = models.map((m) => ({
|
|
107
110
|
...m,
|
|
108
111
|
blobBinding: m.blobBinding
|
|
109
|
-
? {
|
|
112
|
+
? {
|
|
113
|
+
...m.blobBinding,
|
|
114
|
+
coreKey: m.blobBinding.coreKey ? IdEnc.normalize(m.blobBinding.coreKey) : undefined
|
|
115
|
+
}
|
|
110
116
|
: undefined
|
|
111
117
|
}))
|
|
112
118
|
console.log(JSON.stringify(out, null, 2))
|
|
@@ -129,7 +135,8 @@ const listCmd = command('list',
|
|
|
129
135
|
}
|
|
130
136
|
)
|
|
131
137
|
|
|
132
|
-
const getCmd = command(
|
|
138
|
+
const getCmd = command(
|
|
139
|
+
'get',
|
|
133
140
|
summary('Get a specific model by path and source'),
|
|
134
141
|
arg('<path>', 'Model path (e.g. hf/org/repo/file.gguf)'),
|
|
135
142
|
arg('<source>', 'Model source (e.g. hf, s3)'),
|
|
@@ -149,7 +156,12 @@ const getCmd = command('get',
|
|
|
149
156
|
const out = {
|
|
150
157
|
...model,
|
|
151
158
|
blobBinding: model.blobBinding
|
|
152
|
-
? {
|
|
159
|
+
? {
|
|
160
|
+
...model.blobBinding,
|
|
161
|
+
coreKey: model.blobBinding.coreKey
|
|
162
|
+
? IdEnc.normalize(model.blobBinding.coreKey)
|
|
163
|
+
: undefined
|
|
164
|
+
}
|
|
153
165
|
: undefined
|
|
154
166
|
}
|
|
155
167
|
console.log(JSON.stringify(out, null, 2))
|
|
@@ -161,7 +173,8 @@ const getCmd = command('get',
|
|
|
161
173
|
}
|
|
162
174
|
)
|
|
163
175
|
|
|
164
|
-
const downloadCmd = command(
|
|
176
|
+
const downloadCmd = command(
|
|
177
|
+
'download',
|
|
165
178
|
summary('Download a model from the registry'),
|
|
166
179
|
arg('<path>', 'Model path'),
|
|
167
180
|
arg('<source>', 'Model source (e.g. hf, s3)'),
|
|
@@ -192,9 +205,12 @@ const downloadCmd = command('download',
|
|
|
192
205
|
}
|
|
193
206
|
)
|
|
194
207
|
|
|
195
|
-
const profileCmd = command(
|
|
208
|
+
const profileCmd = command(
|
|
209
|
+
'profile',
|
|
196
210
|
summary('Profile download performance for a model'),
|
|
197
|
-
description(
|
|
211
|
+
description(
|
|
212
|
+
'Downloads a model to a temp directory while collecting UDX network stats, connection info, and hypercore metrics. Similar to hyperdrive-profiler but for registry blobs.'
|
|
213
|
+
),
|
|
198
214
|
arg('<path>', 'Model path'),
|
|
199
215
|
arg('[source]', 'Model source filter (e.g. hf, s3)'),
|
|
200
216
|
flag('--interval|-i [seconds]', 'Stats print interval in seconds (default: 5)'),
|
|
@@ -232,7 +248,8 @@ const profileCmd = command('profile',
|
|
|
232
248
|
|
|
233
249
|
// --- Root command ---
|
|
234
250
|
|
|
235
|
-
const cmd = command(
|
|
251
|
+
const cmd = command(
|
|
252
|
+
'qvac-registry',
|
|
236
253
|
header(`qvac-registry v${VERSION}`),
|
|
237
254
|
summary('QVAC Model Registry CLI'),
|
|
238
255
|
footer('Set QVAC_REGISTRY_CORE_KEY env var or use --key to specify the registry core key.'),
|
package/index.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ export interface QVACBlobBinding {
|
|
|
10
10
|
blockLength: number
|
|
11
11
|
byteOffset: number
|
|
12
12
|
byteLength: number
|
|
13
|
+
sha256: string
|
|
13
14
|
}
|
|
14
15
|
|
|
15
16
|
export interface QVACModelEntry {
|
|
@@ -26,6 +27,29 @@ export interface QVACModelEntry {
|
|
|
26
27
|
notes?: string
|
|
27
28
|
tags?: string[]
|
|
28
29
|
blobBinding: QVACBlobBinding
|
|
30
|
+
/**
|
|
31
|
+
* Raw GGUF key-value metadata extracted at ingest, serialised as a JSON string.
|
|
32
|
+
*
|
|
33
|
+
* - Only present for GGUF artifacts; absent for other formats and when
|
|
34
|
+
* extraction failed (e.g. some GGUF v3 files).
|
|
35
|
+
* - Tensor infos are discarded and every `tokenizer.*` key is stripped.
|
|
36
|
+
* - For sharded models only the first shard (`-00001-of-`) is described.
|
|
37
|
+
* - Numeric values that exceeded the safe integer range are stringified.
|
|
38
|
+
*/
|
|
39
|
+
ggufMetadata?: string
|
|
40
|
+
/**
|
|
41
|
+
* Points at a weightless description of this artifact: the tensor list an
|
|
42
|
+
* engine's dry-run fitter reads, with none of the weights.
|
|
43
|
+
*
|
|
44
|
+
* - For a GGUF it is a short GGUF holding the tensor list and the settings
|
|
45
|
+
* without the tokenizer tables; for a safetensors it is the JSON header.
|
|
46
|
+
* - Present for GGUF and safetensors artifacts, every shard of a split model
|
|
47
|
+
* included; absent for other formats and when it could not be built.
|
|
48
|
+
* - Written to the writer's active blob core, which is not always the one
|
|
49
|
+
* holding the weights; `downloadBlob` takes the binding directly.
|
|
50
|
+
* - `sha256` covers the description, not the artifact it came from.
|
|
51
|
+
*/
|
|
52
|
+
fitBlobBinding?: QVACBlobBinding
|
|
29
53
|
}
|
|
30
54
|
|
|
31
55
|
export interface QVACDownloadedArtifactStream {
|
|
@@ -48,7 +72,12 @@ export interface QVACDownloadOptions {
|
|
|
48
72
|
peerTimeout?: number
|
|
49
73
|
maxRetries?: number
|
|
50
74
|
outputFile?: string
|
|
51
|
-
onProgress?: (progress: {
|
|
75
|
+
onProgress?: (progress: {
|
|
76
|
+
downloaded: number
|
|
77
|
+
total: number
|
|
78
|
+
cachedBlocks: number
|
|
79
|
+
totalBlocks: number
|
|
80
|
+
}) => void
|
|
52
81
|
signal?: AbortSignal
|
|
53
82
|
}
|
|
54
83
|
|
|
@@ -56,7 +85,12 @@ export interface QVACBlobDownloadOptions {
|
|
|
56
85
|
timeout?: number
|
|
57
86
|
maxRetries?: number
|
|
58
87
|
outputFile?: string
|
|
59
|
-
onProgress?: (progress: {
|
|
88
|
+
onProgress?: (progress: {
|
|
89
|
+
downloaded: number
|
|
90
|
+
total: number
|
|
91
|
+
cachedBlocks: number
|
|
92
|
+
totalBlocks: number
|
|
93
|
+
}) => void
|
|
60
94
|
signal?: AbortSignal
|
|
61
95
|
}
|
|
62
96
|
|
|
@@ -91,35 +125,42 @@ export interface FindByParams {
|
|
|
91
125
|
|
|
92
126
|
export interface LifecycleSwarmHandle {
|
|
93
127
|
readonly suspended: boolean
|
|
94
|
-
suspend
|
|
95
|
-
resume
|
|
128
|
+
suspend(opts?: LifecycleLogOptions): Promise<void>
|
|
129
|
+
resume(opts?: LifecycleLogOptions): Promise<void>
|
|
96
130
|
}
|
|
97
131
|
|
|
98
132
|
export interface LifecycleStoreHandle {
|
|
99
|
-
suspend
|
|
100
|
-
resume
|
|
133
|
+
suspend(opts?: LifecycleLogOptions): Promise<void>
|
|
134
|
+
resume(): Promise<void>
|
|
101
135
|
}
|
|
102
136
|
|
|
103
137
|
export class QVACRegistryClient extends ReadyResource {
|
|
104
|
-
constructor
|
|
138
|
+
constructor(opts?: QVACRegistryClientOptions)
|
|
105
139
|
|
|
106
140
|
/** Valid only while the client remains open. Cached handles become stale after close(). */
|
|
107
141
|
readonly corestore: LifecycleStoreHandle | null
|
|
108
142
|
/** Valid only while the client remains open. Cached handles become stale after close(). */
|
|
109
143
|
readonly hyperswarm: LifecycleSwarmHandle | null
|
|
110
144
|
|
|
111
|
-
ready
|
|
112
|
-
close
|
|
113
|
-
suspend
|
|
114
|
-
resume
|
|
115
|
-
|
|
116
|
-
getModel
|
|
117
|
-
downloadModel
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
145
|
+
ready(): Promise<void>
|
|
146
|
+
close(): Promise<void>
|
|
147
|
+
suspend(opts?: LifecycleLogOptions): Promise<void>
|
|
148
|
+
resume(opts?: LifecycleLogOptions): Promise<void>
|
|
149
|
+
|
|
150
|
+
getModel(path: string, source: string): Promise<QVACModelEntry | null>
|
|
151
|
+
downloadModel(
|
|
152
|
+
path: string,
|
|
153
|
+
source: string,
|
|
154
|
+
options?: QVACDownloadOptions
|
|
155
|
+
): Promise<QVACDownloadResult>
|
|
156
|
+
downloadBlob(
|
|
157
|
+
blobBinding: QVACBlobBinding,
|
|
158
|
+
options?: QVACBlobDownloadOptions
|
|
159
|
+
): Promise<QVACBlobDownloadResult>
|
|
160
|
+
|
|
161
|
+
findBy(params?: FindByParams): Promise<QVACModelEntry[]>
|
|
162
|
+
findModels(query?: QVACModelQuery): Promise<QVACModelEntry[]>
|
|
163
|
+
findModelsByEngine(query?: QVACModelQuery): Promise<QVACModelEntry[]>
|
|
164
|
+
findModelsByName(query?: QVACModelQuery): Promise<QVACModelEntry[]>
|
|
165
|
+
findModelsByQuantization(query?: QVACModelQuery): Promise<QVACModelEntry[]>
|
|
125
166
|
}
|
package/lib/client.js
CHANGED
|
@@ -15,6 +15,8 @@ const fs = require('#fs')
|
|
|
15
15
|
|
|
16
16
|
const DEFAULT_DOWNLOAD_MAX_RETRIES = 3
|
|
17
17
|
const RETRIABLE_DOWNLOAD_CODES = ['REQUEST_TIMEOUT']
|
|
18
|
+
const INFLIGHT_DRAIN_TIMEOUT_MS = 5000
|
|
19
|
+
const INFLIGHT_DRAIN_POLL_MS = 10
|
|
18
20
|
|
|
19
21
|
// While the app is backgrounded the swarm is suspended; a retry must wait for
|
|
20
22
|
// resume rather than burn its (small) retry budget timing out against a dead
|
|
@@ -23,7 +25,7 @@ const RESUME_WAIT_MAX_MS = 5 * 60 * 1000
|
|
|
23
25
|
const RESUME_WAIT_POLL_MS = 200
|
|
24
26
|
|
|
25
27
|
class QVACRegistryClient extends ReadyResource {
|
|
26
|
-
constructor
|
|
28
|
+
constructor(opts = {}) {
|
|
27
29
|
super()
|
|
28
30
|
|
|
29
31
|
this.logger = new Logger(opts.logger)
|
|
@@ -46,7 +48,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
46
48
|
this.ready()
|
|
47
49
|
}
|
|
48
50
|
|
|
49
|
-
async _open
|
|
51
|
+
async _open() {
|
|
50
52
|
this.logger.debug('_open called')
|
|
51
53
|
|
|
52
54
|
this.logger.debug('Opening corestore')
|
|
@@ -65,10 +67,13 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
65
67
|
await this._metadataReady
|
|
66
68
|
}
|
|
67
69
|
|
|
68
|
-
async _connectMetadataCore
|
|
70
|
+
async _connectMetadataCore() {
|
|
69
71
|
if (!this.registryCoreKey) {
|
|
70
72
|
this.logger.error('Missing registry core key for read mode')
|
|
71
|
-
throw new QvacErrorRegistryClient({
|
|
73
|
+
throw new QvacErrorRegistryClient({
|
|
74
|
+
code: ERR_CODES.FAILED_TO_CONNECT,
|
|
75
|
+
adds: 'Missing registry core key. Set QVAC_REGISTRY_CORE_KEY environment variable.'
|
|
76
|
+
})
|
|
72
77
|
}
|
|
73
78
|
|
|
74
79
|
const viewKey = IdEnc.decode(this.registryCoreKey)
|
|
@@ -96,15 +101,18 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
96
101
|
})
|
|
97
102
|
}
|
|
98
103
|
|
|
99
|
-
async _ensureMetadata
|
|
104
|
+
async _ensureMetadata() {
|
|
100
105
|
await this.ready()
|
|
101
106
|
await this._metadataReady
|
|
102
107
|
if (!this.db) {
|
|
103
|
-
throw new QvacErrorRegistryClient({
|
|
108
|
+
throw new QvacErrorRegistryClient({
|
|
109
|
+
code: ERR_CODES.FAILED_TO_CONNECT,
|
|
110
|
+
adds: 'Registry database not available.'
|
|
111
|
+
})
|
|
104
112
|
}
|
|
105
113
|
}
|
|
106
114
|
|
|
107
|
-
async getModel
|
|
115
|
+
async getModel(path, source) {
|
|
108
116
|
this._validateString(path, 'path')
|
|
109
117
|
this._validateString(source, 'source')
|
|
110
118
|
|
|
@@ -121,7 +129,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
121
129
|
}
|
|
122
130
|
}
|
|
123
131
|
|
|
124
|
-
async findModels
|
|
132
|
+
async findModels(query = {}, opts = {}) {
|
|
125
133
|
await this._ensureMetadata()
|
|
126
134
|
const { includeDeprecated = false } = opts
|
|
127
135
|
this.logger.debug('findModels called', { query, includeDeprecated })
|
|
@@ -129,25 +137,25 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
129
137
|
let models = await this.db.findModelsByPath(query).toArray()
|
|
130
138
|
|
|
131
139
|
if (!includeDeprecated) {
|
|
132
|
-
models = models.filter(m => !m.deprecated)
|
|
140
|
+
models = models.filter((m) => !m.deprecated)
|
|
133
141
|
}
|
|
134
142
|
|
|
135
143
|
return models
|
|
136
144
|
}
|
|
137
145
|
|
|
138
|
-
async findModelsByEngine
|
|
146
|
+
async findModelsByEngine(query = {}) {
|
|
139
147
|
await this._ensureMetadata()
|
|
140
148
|
this.logger.debug('findModelsByEngine called', { query })
|
|
141
149
|
return this.db.findModelsByEngine(query).toArray()
|
|
142
150
|
}
|
|
143
151
|
|
|
144
|
-
async findModelsByName
|
|
152
|
+
async findModelsByName(query = {}) {
|
|
145
153
|
await this._ensureMetadata()
|
|
146
154
|
this.logger.debug('findModelsByName called', { query })
|
|
147
155
|
return this.db.findModelsByName(query).toArray()
|
|
148
156
|
}
|
|
149
157
|
|
|
150
|
-
async findModelsByQuantization
|
|
158
|
+
async findModelsByQuantization(query = {}) {
|
|
151
159
|
await this._ensureMetadata()
|
|
152
160
|
this.logger.debug('findModelsByQuantization called', { query })
|
|
153
161
|
return this.db.findModelsByQuantization(query).toArray()
|
|
@@ -163,19 +171,19 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
163
171
|
* @param {boolean} [params.includeDeprecated=false] - Include deprecated models
|
|
164
172
|
* @returns {Promise<Array>} Array of matching models
|
|
165
173
|
*/
|
|
166
|
-
async findBy
|
|
174
|
+
async findBy(params = {}) {
|
|
167
175
|
await this._ensureMetadata()
|
|
168
176
|
this.logger.debug('findBy called', { params })
|
|
169
177
|
return this.db.findBy(params)
|
|
170
178
|
}
|
|
171
179
|
|
|
172
|
-
_validateString
|
|
180
|
+
_validateString(value, name) {
|
|
173
181
|
if (typeof value !== 'string' || value.length === 0) {
|
|
174
182
|
throw new Error(`Invalid ${name}: ${value}`)
|
|
175
183
|
}
|
|
176
184
|
}
|
|
177
185
|
|
|
178
|
-
async _checkBlobProgress
|
|
186
|
+
async _checkBlobProgress(core, blobPointer) {
|
|
179
187
|
const totalBlocks = blobPointer.blockLength
|
|
180
188
|
const totalBytes = blobPointer.byteLength
|
|
181
189
|
|
|
@@ -192,7 +200,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
192
200
|
return { cachedBlocks, totalBlocks, totalBytes }
|
|
193
201
|
}
|
|
194
202
|
|
|
195
|
-
async _getBlobsCore
|
|
203
|
+
async _getBlobsCore(blobsCoreKey) {
|
|
196
204
|
let keyBuffer
|
|
197
205
|
if (Buffer.isBuffer(blobsCoreKey)) {
|
|
198
206
|
keyBuffer = blobsCoreKey
|
|
@@ -221,7 +229,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
221
229
|
* Bounded by RESUME_WAIT_MAX_MS; returns (and lets the retry proceed/fail) if
|
|
222
230
|
* the runtime never resumes.
|
|
223
231
|
*/
|
|
224
|
-
async _waitForSwarmResumed
|
|
232
|
+
async _waitForSwarmResumed(signal) {
|
|
225
233
|
if (!this.hyperswarm || !this.hyperswarm.suspended) return
|
|
226
234
|
|
|
227
235
|
const start = Date.now()
|
|
@@ -231,7 +239,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
231
239
|
this.logger.warn('Swarm still suspended after resume wait; retrying anyway')
|
|
232
240
|
return
|
|
233
241
|
}
|
|
234
|
-
await new Promise(resolve => setTimeout(resolve, RESUME_WAIT_POLL_MS))
|
|
242
|
+
await new Promise((resolve) => setTimeout(resolve, RESUME_WAIT_POLL_MS))
|
|
235
243
|
}
|
|
236
244
|
}
|
|
237
245
|
|
|
@@ -241,7 +249,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
241
249
|
* (and timing out) against zero peers and burning the retry budget. Bounded
|
|
242
250
|
* by RESUME_WAIT_MAX_MS. No-op when peer info is unavailable.
|
|
243
251
|
*/
|
|
244
|
-
async _waitForPeers
|
|
252
|
+
async _waitForPeers(core, signal) {
|
|
245
253
|
if (!core || !Array.isArray(core.peers)) return
|
|
246
254
|
if (core.peers.length > 0) return
|
|
247
255
|
|
|
@@ -252,7 +260,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
252
260
|
this.logger.warn('No peers after reconnect wait; retrying anyway')
|
|
253
261
|
return
|
|
254
262
|
}
|
|
255
|
-
await new Promise(resolve => setTimeout(resolve, RESUME_WAIT_POLL_MS))
|
|
263
|
+
await new Promise((resolve) => setTimeout(resolve, RESUME_WAIT_POLL_MS))
|
|
256
264
|
}
|
|
257
265
|
}
|
|
258
266
|
|
|
@@ -269,7 +277,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
269
277
|
* RESUME_WAIT_MAX_MS) swarm/peer waits aborts promptly instead of blocking
|
|
270
278
|
* until peers return or the cap elapses.
|
|
271
279
|
*/
|
|
272
|
-
async _reconnectCore
|
|
280
|
+
async _reconnectCore(core, signal) {
|
|
273
281
|
if (!core || !this.hyperswarm) return
|
|
274
282
|
if (signal && signal.aborted) throw new Error('Download cancelled')
|
|
275
283
|
|
|
@@ -290,7 +298,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
290
298
|
await core.update()
|
|
291
299
|
}
|
|
292
300
|
|
|
293
|
-
async downloadModel
|
|
301
|
+
async downloadModel(path, source, options = {}) {
|
|
294
302
|
this._validateString(path, 'path')
|
|
295
303
|
this._validateString(source, 'source')
|
|
296
304
|
|
|
@@ -298,7 +306,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
298
306
|
throw new Error(`Invalid options: ${typeof options}`)
|
|
299
307
|
}
|
|
300
308
|
|
|
301
|
-
let core, blobs
|
|
309
|
+
let core, blobs, blockStart, blockEnd, rangeDownload
|
|
302
310
|
|
|
303
311
|
try {
|
|
304
312
|
this.logger.info('Downloading model', { path, source })
|
|
@@ -306,11 +314,17 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
306
314
|
|
|
307
315
|
const model = await this.getModel(path, source)
|
|
308
316
|
if (!model) {
|
|
309
|
-
throw new QvacErrorRegistryClient({
|
|
317
|
+
throw new QvacErrorRegistryClient({
|
|
318
|
+
code: ERR_CODES.MODEL_NOT_FOUND,
|
|
319
|
+
adds: `Model not found: ${path} (source: ${source})`
|
|
320
|
+
})
|
|
310
321
|
}
|
|
311
322
|
|
|
312
323
|
if (!model.blobBinding || !model.blobBinding.coreKey) {
|
|
313
|
-
throw new QvacErrorRegistryClient({
|
|
324
|
+
throw new QvacErrorRegistryClient({
|
|
325
|
+
code: ERR_CODES.MODEL_NOT_FOUND,
|
|
326
|
+
adds: 'Model missing blob binding'
|
|
327
|
+
})
|
|
314
328
|
}
|
|
315
329
|
|
|
316
330
|
this.logger.debug('Model metadata retrieved', { model })
|
|
@@ -332,20 +346,20 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
332
346
|
|
|
333
347
|
const totalSize = model.blobBinding.byteLength
|
|
334
348
|
|
|
335
|
-
|
|
349
|
+
rangeDownload = core.download({
|
|
336
350
|
start: model.blobBinding.blockOffset,
|
|
337
351
|
length: model.blobBinding.blockLength
|
|
338
352
|
})
|
|
339
353
|
|
|
340
|
-
|
|
341
|
-
|
|
354
|
+
blockStart = model.blobBinding.blockOffset
|
|
355
|
+
blockEnd = blockStart + model.blobBinding.blockLength
|
|
342
356
|
|
|
343
357
|
let artifact
|
|
344
358
|
if (options.outputFile) {
|
|
345
359
|
await withRetry(
|
|
346
360
|
() => this._streamBlobToFile(blobs, core, model.blobBinding, options.outputFile, options),
|
|
347
361
|
{
|
|
348
|
-
maxRetries: options.maxRetries
|
|
362
|
+
maxRetries: options.maxRetries ?? DEFAULT_DOWNLOAD_MAX_RETRIES,
|
|
349
363
|
retryCodes: RETRIABLE_DOWNLOAD_CODES,
|
|
350
364
|
// Wait for the swarm to resume + reconnect peers before retrying,
|
|
351
365
|
// so the retry doesn't immediately time out again against a dead
|
|
@@ -357,10 +371,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
357
371
|
)
|
|
358
372
|
artifact = { path: options.outputFile, totalSize }
|
|
359
373
|
|
|
360
|
-
|
|
361
|
-
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
362
|
-
if (blobs) await blobs.close()
|
|
363
|
-
if (core) await core.close()
|
|
374
|
+
await this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd)
|
|
364
375
|
} else {
|
|
365
376
|
const stream = blobs.createReadStream(model.blobBinding, {
|
|
366
377
|
wait: true,
|
|
@@ -368,27 +379,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
368
379
|
})
|
|
369
380
|
artifact = { stream, totalSize }
|
|
370
381
|
|
|
371
|
-
|
|
372
|
-
rangeDownload.destroy()
|
|
373
|
-
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
374
|
-
if (blobs) {
|
|
375
|
-
try {
|
|
376
|
-
await blobs.close()
|
|
377
|
-
} catch (cleanupError) {
|
|
378
|
-
this.logger.warn('Error closing blob instance', { error: cleanupError.message })
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
if (core) {
|
|
382
|
-
try {
|
|
383
|
-
await core.close()
|
|
384
|
-
} catch (cleanupError) {
|
|
385
|
-
this.logger.warn('Error closing blob core', { error: cleanupError.message })
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
this.logger.debug('Blob resources closed after stream end')
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
stream.once('end', cleanup)
|
|
382
|
+
this._releaseOnStreamEnd(stream, core, blobs, rangeDownload, blockStart, blockEnd)
|
|
392
383
|
}
|
|
393
384
|
|
|
394
385
|
this.logger.info('Model downloaded successfully')
|
|
@@ -400,44 +391,38 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
400
391
|
} catch (error) {
|
|
401
392
|
this.logger.error('Error downloading model', error)
|
|
402
393
|
|
|
403
|
-
|
|
404
|
-
try {
|
|
405
|
-
await blobs.close()
|
|
406
|
-
} catch (cleanupError) {
|
|
407
|
-
this.logger.warn('Error closing blob instance on error', { error: cleanupError.message })
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
if (core) {
|
|
411
|
-
try {
|
|
412
|
-
await core.close()
|
|
413
|
-
} catch (cleanupError) {
|
|
414
|
-
this.logger.warn('Error closing blob core on error', { error: cleanupError.message })
|
|
415
|
-
}
|
|
416
|
-
}
|
|
394
|
+
await this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd)
|
|
417
395
|
|
|
418
396
|
throw error
|
|
419
397
|
}
|
|
420
398
|
}
|
|
421
399
|
|
|
422
|
-
async downloadBlob
|
|
400
|
+
async downloadBlob(blobBinding, options = {}) {
|
|
423
401
|
if (!blobBinding || !blobBinding.coreKey) {
|
|
424
402
|
throw new Error('Invalid blobBinding: coreKey is required')
|
|
425
403
|
}
|
|
426
|
-
if (
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
404
|
+
if (
|
|
405
|
+
typeof blobBinding.blockOffset !== 'number' ||
|
|
406
|
+
typeof blobBinding.blockLength !== 'number' ||
|
|
407
|
+
typeof blobBinding.byteLength !== 'number'
|
|
408
|
+
) {
|
|
409
|
+
throw new Error(
|
|
410
|
+
'Invalid blobBinding: blockOffset, blockLength, and byteLength are required numbers'
|
|
411
|
+
)
|
|
430
412
|
}
|
|
431
413
|
|
|
432
414
|
if (options && typeof options !== 'object') {
|
|
433
415
|
throw new Error(`Invalid options: ${typeof options}`)
|
|
434
416
|
}
|
|
435
417
|
|
|
436
|
-
let core, blobs
|
|
418
|
+
let core, blobs, blockStart, blockEnd, rangeDownload
|
|
437
419
|
|
|
438
420
|
try {
|
|
439
421
|
this.logger.info('Downloading blob directly', {
|
|
440
|
-
coreKey:
|
|
422
|
+
coreKey:
|
|
423
|
+
typeof blobBinding.coreKey === 'string'
|
|
424
|
+
? blobBinding.coreKey.slice(0, 12) + '...'
|
|
425
|
+
: '(buffer)',
|
|
441
426
|
blockOffset: blobBinding.blockOffset,
|
|
442
427
|
blockLength: blobBinding.blockLength,
|
|
443
428
|
byteLength: blobBinding.byteLength
|
|
@@ -468,10 +453,10 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
468
453
|
}
|
|
469
454
|
const totalSize = blobBinding.byteLength
|
|
470
455
|
|
|
471
|
-
|
|
472
|
-
|
|
456
|
+
blockStart = pointer.blockOffset
|
|
457
|
+
blockEnd = blockStart + pointer.blockLength
|
|
473
458
|
|
|
474
|
-
|
|
459
|
+
rangeDownload = core.download({
|
|
475
460
|
start: pointer.blockOffset,
|
|
476
461
|
length: pointer.blockLength
|
|
477
462
|
})
|
|
@@ -481,7 +466,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
481
466
|
await withRetry(
|
|
482
467
|
() => this._streamBlobToFile(blobs, core, pointer, options.outputFile, options),
|
|
483
468
|
{
|
|
484
|
-
maxRetries: options.maxRetries
|
|
469
|
+
maxRetries: options.maxRetries ?? DEFAULT_DOWNLOAD_MAX_RETRIES,
|
|
485
470
|
retryCodes: RETRIABLE_DOWNLOAD_CODES,
|
|
486
471
|
// Wait for swarm resume + peer reconnect before retrying (see
|
|
487
472
|
// downloadModel). Cached blocks are reused; the file is re-streamed.
|
|
@@ -491,10 +476,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
491
476
|
)
|
|
492
477
|
artifact = { path: options.outputFile, totalSize }
|
|
493
478
|
|
|
494
|
-
|
|
495
|
-
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
496
|
-
if (blobs) await blobs.close()
|
|
497
|
-
if (core) await core.close()
|
|
479
|
+
await this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd)
|
|
498
480
|
} else {
|
|
499
481
|
const stream = blobs.createReadStream(pointer, {
|
|
500
482
|
wait: true,
|
|
@@ -502,23 +484,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
502
484
|
})
|
|
503
485
|
artifact = { stream, totalSize }
|
|
504
486
|
|
|
505
|
-
|
|
506
|
-
rangeDownload.destroy()
|
|
507
|
-
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
508
|
-
if (blobs) {
|
|
509
|
-
try { await blobs.close() } catch (e) {
|
|
510
|
-
this.logger.warn('Error closing blob instance', { error: e.message })
|
|
511
|
-
}
|
|
512
|
-
}
|
|
513
|
-
if (core) {
|
|
514
|
-
try { await core.close() } catch (e) {
|
|
515
|
-
this.logger.warn('Error closing blob core', { error: e.message })
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
this.logger.debug('Blob resources closed after stream end')
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
stream.once('end', cleanup)
|
|
487
|
+
this._releaseOnStreamEnd(stream, core, blobs, rangeDownload, blockStart, blockEnd)
|
|
522
488
|
}
|
|
523
489
|
|
|
524
490
|
this.logger.info('Blob download complete (direct)')
|
|
@@ -527,33 +493,96 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
527
493
|
} catch (error) {
|
|
528
494
|
this.logger.error('Error downloading blob directly', error)
|
|
529
495
|
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
496
|
+
await this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd)
|
|
497
|
+
|
|
498
|
+
throw error
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
_releaseOnStreamEnd(stream, core, blobs, rangeDownload, blockStart, blockEnd) {
|
|
503
|
+
let released = false
|
|
504
|
+
|
|
505
|
+
// 'close' also covers a destroyed or errored stream; on 'end' alone a
|
|
506
|
+
// cancelled stream download would never free its blocks.
|
|
507
|
+
const release = () => {
|
|
508
|
+
if (released) return
|
|
509
|
+
released = true
|
|
510
|
+
return this._releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd).catch((e) =>
|
|
511
|
+
this.logger.warn('Error releasing blob resources', { error: e.message })
|
|
512
|
+
)
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
stream.once('end', release)
|
|
516
|
+
stream.once('close', release)
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
async _releaseDownload(core, blobs, rangeDownload, blockStart, blockEnd) {
|
|
520
|
+
// Hypercore can commit in-flight responses after a range is destroyed.
|
|
521
|
+
if (rangeDownload) rangeDownload.destroy()
|
|
522
|
+
|
|
523
|
+
if (core && blockStart !== undefined) {
|
|
524
|
+
if (rangeDownload) await this._waitForInflightBlocks(core)
|
|
525
|
+
await this._clearBlobBlocks(core, blockStart, blockEnd)
|
|
526
|
+
}
|
|
527
|
+
if (blobs) {
|
|
528
|
+
try {
|
|
529
|
+
await blobs.close()
|
|
530
|
+
} catch (e) {
|
|
531
|
+
this.logger.warn('Error closing blob instance', { error: e.message })
|
|
534
532
|
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
533
|
+
}
|
|
534
|
+
if (core) {
|
|
535
|
+
try {
|
|
536
|
+
await core.close()
|
|
537
|
+
} catch (e) {
|
|
538
|
+
this.logger.warn('Error closing blob core', { error: e.message })
|
|
539
539
|
}
|
|
540
|
+
}
|
|
540
541
|
|
|
541
|
-
|
|
542
|
+
this.logger.debug('Blob resources released')
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
async _waitForInflightBlocks(core) {
|
|
546
|
+
if (!Array.isArray(core.peers)) return
|
|
547
|
+
|
|
548
|
+
const timeoutMs = this._inflightDrainTimeoutMs ?? INFLIGHT_DRAIN_TIMEOUT_MS
|
|
549
|
+
const pollMs = this._inflightDrainPollMs ?? INFLIGHT_DRAIN_POLL_MS
|
|
550
|
+
const deadline = Date.now() + timeoutMs
|
|
551
|
+
while (core.peers.some((peer) => peer.inflight > 0 || peer.dataProcessing > 0)) {
|
|
552
|
+
if (Date.now() >= deadline) {
|
|
553
|
+
this.logger.warn('Timed out waiting for in-flight blob blocks before clearing', {
|
|
554
|
+
timeoutMs,
|
|
555
|
+
peers: core.peers.map((peer, index) => ({
|
|
556
|
+
peer: index,
|
|
557
|
+
inflight: peer.inflight,
|
|
558
|
+
dataProcessing: peer.dataProcessing
|
|
559
|
+
}))
|
|
560
|
+
})
|
|
561
|
+
return
|
|
562
|
+
}
|
|
563
|
+
await new Promise((resolve) => setTimeout(resolve, pollMs))
|
|
542
564
|
}
|
|
543
565
|
}
|
|
544
566
|
|
|
545
|
-
async _clearBlobBlocks
|
|
567
|
+
async _clearBlobBlocks(core, start, end) {
|
|
546
568
|
try {
|
|
547
569
|
const cleared = await core.clear(start, end, { diff: true })
|
|
548
570
|
await core.compact()
|
|
549
|
-
this.logger.info('Cleared blob blocks from corestore', {
|
|
571
|
+
this.logger.info('Cleared blob blocks from corestore', {
|
|
572
|
+
start,
|
|
573
|
+
end,
|
|
574
|
+
blocks: cleared ? cleared.blocks : end - start
|
|
575
|
+
})
|
|
550
576
|
} catch (err) {
|
|
551
577
|
this.logger.warn('Failed to clear blob blocks from corestore', { error: err.message })
|
|
552
578
|
}
|
|
553
579
|
}
|
|
554
580
|
|
|
555
|
-
async _streamBlobToFile
|
|
556
|
-
const { cachedBlocks, totalBlocks, totalBytes } = await this._checkBlobProgress(
|
|
581
|
+
async _streamBlobToFile(blobs, core, blobPointer, filePath, options) {
|
|
582
|
+
const { cachedBlocks, totalBlocks, totalBytes } = await this._checkBlobProgress(
|
|
583
|
+
core,
|
|
584
|
+
blobPointer
|
|
585
|
+
)
|
|
557
586
|
|
|
558
587
|
this.logger.debug('Blob progress before download', {
|
|
559
588
|
cachedBlocks,
|
|
@@ -575,7 +604,10 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
575
604
|
}
|
|
576
605
|
|
|
577
606
|
const progressHandler = (index, bytes) => {
|
|
578
|
-
if (
|
|
607
|
+
if (
|
|
608
|
+
index >= blobPointer.blockOffset &&
|
|
609
|
+
index < blobPointer.blockOffset + blobPointer.blockLength
|
|
610
|
+
) {
|
|
579
611
|
downloadedBytes += bytes
|
|
580
612
|
const capped = Math.min(downloadedBytes, totalBytes)
|
|
581
613
|
if (options.onProgress) {
|
|
@@ -620,11 +652,15 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
620
652
|
reject(new Error('Download cancelled'))
|
|
621
653
|
return
|
|
622
654
|
}
|
|
623
|
-
options.signal.addEventListener(
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
655
|
+
options.signal.addEventListener(
|
|
656
|
+
'abort',
|
|
657
|
+
() => {
|
|
658
|
+
stream.destroy()
|
|
659
|
+
writeStream.destroy()
|
|
660
|
+
reject(new Error('Download cancelled'))
|
|
661
|
+
},
|
|
662
|
+
{ once: true }
|
|
663
|
+
)
|
|
628
664
|
}
|
|
629
665
|
})
|
|
630
666
|
} finally {
|
|
@@ -633,7 +669,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
633
669
|
}
|
|
634
670
|
}
|
|
635
671
|
|
|
636
|
-
async suspend
|
|
672
|
+
async suspend(opts = {}) {
|
|
637
673
|
this.logger.debug('suspend called')
|
|
638
674
|
|
|
639
675
|
if (!this.opened || this.closing) {
|
|
@@ -651,7 +687,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
651
687
|
this.logger.debug('QVACRegistryClient suspended')
|
|
652
688
|
}
|
|
653
689
|
|
|
654
|
-
async resume
|
|
690
|
+
async resume(opts = {}) {
|
|
655
691
|
this.logger.debug('resume called')
|
|
656
692
|
|
|
657
693
|
if (!this.opened || this.closing) {
|
|
@@ -669,7 +705,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
669
705
|
this.logger.debug('QVACRegistryClient resumed')
|
|
670
706
|
}
|
|
671
707
|
|
|
672
|
-
async _close
|
|
708
|
+
async _close() {
|
|
673
709
|
this.logger.debug('_close called')
|
|
674
710
|
|
|
675
711
|
if (this._metadataReady) {
|
package/lib/config.js
CHANGED
|
@@ -9,12 +9,12 @@ const path = require('#path')
|
|
|
9
9
|
const DEFAULT_REGISTRY_CORE_KEY = 'uf1fm44uzockp6azhcdiqt1esjgm65fwtimsh946e8kwysdes9ko'
|
|
10
10
|
|
|
11
11
|
class RegistryConfig {
|
|
12
|
-
constructor
|
|
12
|
+
constructor(opts = {}) {
|
|
13
13
|
this.logger = new Logger(opts.logger)
|
|
14
14
|
this.logger.debug('RegistryConfig initialized', { opts })
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
getRegistryStorage
|
|
17
|
+
getRegistryStorage(providedPath) {
|
|
18
18
|
if (providedPath) {
|
|
19
19
|
this.logger.debug('getRegistryStorage called with providedPath', { providedPath })
|
|
20
20
|
return providedPath
|
|
@@ -26,14 +26,17 @@ class RegistryConfig {
|
|
|
26
26
|
return result
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
_createTempStoragePath
|
|
29
|
+
_createTempStoragePath() {
|
|
30
30
|
const tmpBase = os.tmpdir()
|
|
31
|
-
const storageDir = path.join(
|
|
31
|
+
const storageDir = path.join(
|
|
32
|
+
tmpBase,
|
|
33
|
+
`qvac-registry-client-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
|
|
34
|
+
)
|
|
32
35
|
this.logger.debug('Created temp storage path', { storageDir })
|
|
33
36
|
return storageDir
|
|
34
37
|
}
|
|
35
38
|
|
|
36
|
-
getRegistryCoreKey
|
|
39
|
+
getRegistryCoreKey(providedKey) {
|
|
37
40
|
if (providedKey) {
|
|
38
41
|
this.logger.debug('getRegistryCoreKey called with providedKey', { providedKey })
|
|
39
42
|
return providedKey
|
package/lib/logger.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
class Logger {
|
|
4
|
-
constructor
|
|
4
|
+
constructor(opts = {}) {
|
|
5
5
|
this.level = opts.level || 'info'
|
|
6
6
|
this.name = opts.name || 'QVACRegistryClient'
|
|
7
7
|
this.enabled = opts.enabled !== false
|
|
@@ -14,30 +14,30 @@ class Logger {
|
|
|
14
14
|
}
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
_shouldLog
|
|
17
|
+
_shouldLog(level) {
|
|
18
18
|
if (!this.enabled) return false
|
|
19
19
|
return this.levels[level] >= this.levels[this.level]
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
_log
|
|
22
|
+
_log(level, ...args) {
|
|
23
23
|
if (!this._shouldLog(level)) return
|
|
24
24
|
const prefix = `[${this.name}] [${level.toUpperCase()}]`
|
|
25
25
|
console[level](prefix, ...args)
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
debug
|
|
28
|
+
debug(...args) {
|
|
29
29
|
this._log('debug', ...args)
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
info
|
|
32
|
+
info(...args) {
|
|
33
33
|
this._log('info', ...args)
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
warn
|
|
36
|
+
warn(...args) {
|
|
37
37
|
this._log('warn', ...args)
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
error
|
|
40
|
+
error(...args) {
|
|
41
41
|
this._log('error', ...args)
|
|
42
42
|
}
|
|
43
43
|
}
|
package/lib/profiler.js
CHANGED
|
@@ -13,7 +13,7 @@ const { RegistryDatabase } = require('@qvac/registry-schema')
|
|
|
13
13
|
* Normalize a blob core key into a Buffer regardless of input format.
|
|
14
14
|
* Handles raw Buffer, { data: [...] } objects from HyperDB, and z32/hex strings.
|
|
15
15
|
*/
|
|
16
|
-
function decodeCoreKey
|
|
16
|
+
function decodeCoreKey(key) {
|
|
17
17
|
if (Buffer.isBuffer(key)) return key
|
|
18
18
|
if (typeof key === 'object' && key !== null && key.data) {
|
|
19
19
|
return Buffer.from(key.data)
|
|
@@ -25,7 +25,7 @@ function decodeCoreKey (key) {
|
|
|
25
25
|
* Collect a snapshot of network, connection, and hypercore stats.
|
|
26
26
|
* All fields are plain values suitable for logging or formatting.
|
|
27
27
|
*/
|
|
28
|
-
function collectStats
|
|
28
|
+
function collectStats(swarmStats, hypercoreStats, blobCore, elapsedSec) {
|
|
29
29
|
const bytesRx = swarmStats.dhtStats.udxBytesReceived
|
|
30
30
|
const bytesTx = swarmStats.dhtStats.udxBytesTransmitted
|
|
31
31
|
|
|
@@ -71,7 +71,7 @@ function collectStats (swarmStats, hypercoreStats, blobCore, elapsedSec) {
|
|
|
71
71
|
/**
|
|
72
72
|
* Format a stats snapshot into a human-readable string.
|
|
73
73
|
*/
|
|
74
|
-
function formatStats
|
|
74
|
+
function formatStats(stats) {
|
|
75
75
|
const n = stats.network
|
|
76
76
|
const c = stats.connection
|
|
77
77
|
const h = stats.hypercore
|
|
@@ -88,7 +88,14 @@ function formatStats (stats) {
|
|
|
88
88
|
lines += ' Attempted: ' + c.attempted + '\n'
|
|
89
89
|
lines += ' Opened: ' + c.opened + '\n'
|
|
90
90
|
lines += ' Closed: ' + c.closed + '\n'
|
|
91
|
-
lines +=
|
|
91
|
+
lines +=
|
|
92
|
+
' Issues: rto=' +
|
|
93
|
+
c.rtos +
|
|
94
|
+
' fast-recoveries=' +
|
|
95
|
+
c.fastRecoveries +
|
|
96
|
+
' retransmits=' +
|
|
97
|
+
c.retransmits +
|
|
98
|
+
'\n'
|
|
92
99
|
lines += 'Hypercore\n'
|
|
93
100
|
lines += ' Blob core: ' + h.contiguousLength + ' / ' + h.length + ' (contiguous / length)\n'
|
|
94
101
|
lines += ' Hotswaps: ' + h.hotswaps + '\n'
|
|
@@ -106,7 +113,7 @@ function formatStats (stats) {
|
|
|
106
113
|
/**
|
|
107
114
|
* Format a final download summary into a human-readable string.
|
|
108
115
|
*/
|
|
109
|
-
function formatSummary
|
|
116
|
+
function formatSummary(opts) {
|
|
110
117
|
let lines = '='.repeat(50) + '\n'
|
|
111
118
|
lines += 'FINAL SUMMARY\n'
|
|
112
119
|
lines += '='.repeat(50) + '\n'
|
|
@@ -133,7 +140,7 @@ function formatSummary (opts) {
|
|
|
133
140
|
* @param {function} [opts.onLog] - Called with (message) for log output; defaults to console.log
|
|
134
141
|
* @returns {Promise<object>} Summary with timing and stats
|
|
135
142
|
*/
|
|
136
|
-
async function profileDownload
|
|
143
|
+
async function profileDownload(opts) {
|
|
137
144
|
// Lazy-loaded: these Node builtins don't exist in Bare runtime,
|
|
138
145
|
// so they must not be required at the top level or unit tests break.
|
|
139
146
|
const os = require('#os')
|
|
@@ -177,7 +184,9 @@ async function profileDownload (opts) {
|
|
|
177
184
|
const cleanupResources = async () => {
|
|
178
185
|
await swarm.destroy()
|
|
179
186
|
await store.close()
|
|
180
|
-
try {
|
|
187
|
+
try {
|
|
188
|
+
fs.rmSync(tmpdir, { recursive: true, force: true })
|
|
189
|
+
} catch {}
|
|
181
190
|
}
|
|
182
191
|
|
|
183
192
|
try {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/registry-client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "QVAC Registry client library for read-only queries via Hyperswarm",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -47,13 +47,15 @@
|
|
|
47
47
|
"test:unit": "brittle tests/unit/*.test.js",
|
|
48
48
|
"test:unit:bare": "brittle-bare tests/unit/*.test.js",
|
|
49
49
|
"test": "npm run test:unit && npm run test:integration",
|
|
50
|
-
"
|
|
51
|
-
"
|
|
50
|
+
"format": "prettier --check .",
|
|
51
|
+
"format:fix": "prettier --write .",
|
|
52
|
+
"lint": "lunte",
|
|
53
|
+
"lint:fix": "lunte --fix",
|
|
52
54
|
"typecheck": "npx tsc --noEmit"
|
|
53
55
|
},
|
|
54
56
|
"dependencies": {
|
|
55
57
|
"@qvac/error": "^0.1.0",
|
|
56
|
-
"@qvac/registry-schema": "^0.
|
|
58
|
+
"@qvac/registry-schema": "^0.4.0",
|
|
57
59
|
"b4a": "^1.6.7",
|
|
58
60
|
"bare-fs": "^4.5.2",
|
|
59
61
|
"bare-os": "^3.6.2",
|
|
@@ -72,7 +74,9 @@
|
|
|
72
74
|
"devDependencies": {
|
|
73
75
|
"brittle": "^3.4.0",
|
|
74
76
|
"dotenv": "^17.2.3",
|
|
75
|
-
"
|
|
77
|
+
"lunte": "^1.8.4",
|
|
78
|
+
"prettier": "^3.9.6",
|
|
79
|
+
"prettier-config-holepunch": "^2.0.0",
|
|
76
80
|
"test-tmp": "^1.4.0",
|
|
77
81
|
"typescript": "^5.9.3"
|
|
78
82
|
}
|
package/utils/env.js
CHANGED
package/utils/error.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
const { QvacErrorBase, addCodes } = require('@qvac/error')
|
|
4
4
|
const { name, version } = require('../package.json')
|
|
5
5
|
|
|
6
|
-
class QvacErrorRegistryClient extends QvacErrorBase {
|
|
6
|
+
class QvacErrorRegistryClient extends QvacErrorBase {}
|
|
7
7
|
|
|
8
8
|
// This library has error code range from 19,001 to 20,000
|
|
9
9
|
const ERR_CODES = Object.freeze({
|
|
@@ -12,23 +12,26 @@ const ERR_CODES = Object.freeze({
|
|
|
12
12
|
MODEL_NOT_FOUND: 19003
|
|
13
13
|
})
|
|
14
14
|
|
|
15
|
-
addCodes(
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
15
|
+
addCodes(
|
|
16
|
+
{
|
|
17
|
+
[ERR_CODES.FAILED_TO_CONNECT]: {
|
|
18
|
+
name: 'FAILED_TO_CONNECT',
|
|
19
|
+
message: (message) => `Failed to connect to registry, error: ${message}`
|
|
20
|
+
},
|
|
21
|
+
[ERR_CODES.FAILED_TO_CLOSE]: {
|
|
22
|
+
name: 'FAILED_TO_CLOSE',
|
|
23
|
+
message: (message) => `Failed to close registry, error: ${message}`
|
|
24
|
+
},
|
|
25
|
+
[ERR_CODES.MODEL_NOT_FOUND]: {
|
|
26
|
+
name: 'MODEL_NOT_FOUND',
|
|
27
|
+
message: (message) => `Model not found, error: ${message}`
|
|
28
|
+
}
|
|
19
29
|
},
|
|
20
|
-
|
|
21
|
-
name
|
|
22
|
-
|
|
23
|
-
},
|
|
24
|
-
[ERR_CODES.MODEL_NOT_FOUND]: {
|
|
25
|
-
name: 'MODEL_NOT_FOUND',
|
|
26
|
-
message: (message) => `Model not found, error: ${message}`
|
|
30
|
+
{
|
|
31
|
+
name,
|
|
32
|
+
version
|
|
27
33
|
}
|
|
28
|
-
|
|
29
|
-
name,
|
|
30
|
-
version
|
|
31
|
-
})
|
|
34
|
+
)
|
|
32
35
|
|
|
33
36
|
module.exports = {
|
|
34
37
|
ERR_CODES,
|
package/utils/retry.js
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* @param {{ warn: Function }} [opts.logger] - Logger instance for retry warnings
|
|
15
15
|
* @returns {Promise<unknown>}
|
|
16
16
|
*/
|
|
17
|
-
async function withRetry
|
|
17
|
+
async function withRetry(fn, opts = {}) {
|
|
18
18
|
const { maxRetries = 3, retryCodes = [], onRetry, beforeRetry, logger } = opts
|
|
19
19
|
|
|
20
20
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
@@ -23,7 +23,8 @@ async function withRetry (fn, opts = {}) {
|
|
|
23
23
|
} catch (err) {
|
|
24
24
|
const isRetriable = retryCodes.length > 0 && retryCodes.includes(err && err.code)
|
|
25
25
|
if (!isRetriable || attempt >= maxRetries) throw err
|
|
26
|
-
logger &&
|
|
26
|
+
logger &&
|
|
27
|
+
logger.warn(`Retrying after ${err.code} (attempt ${attempt}/${maxRetries}): ${err.message}`)
|
|
27
28
|
if (onRetry) await onRetry()
|
|
28
29
|
if (beforeRetry) await beforeRetry()
|
|
29
30
|
}
|